1fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall//===--- CGAtomic.cpp - Emit LLVM IR for atomic operations ----------------===// 2fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall// 3fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall// The LLVM Compiler Infrastructure 4fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall// 5fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall// This file is distributed under the University of Illinois Open Source 6fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall// License. See LICENSE.TXT for details. 7fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall// 8fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall//===----------------------------------------------------------------------===// 9fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall// 10fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall// This file contains the code for emitting atomic operations. 11fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall// 12fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall//===----------------------------------------------------------------------===// 13fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall 14fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall#include "CodeGenFunction.h" 15fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall#include "CGCall.h" 16fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall#include "CodeGenModule.h" 17fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall#include "clang/AST/ASTContext.h" 18fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall#include "llvm/IR/DataLayout.h" 19fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall#include "llvm/IR/Intrinsics.h" 209eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall#include "llvm/IR/Operator.h" 21fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall 22fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCallusing namespace clang; 23fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCallusing namespace CodeGen; 24fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall 259eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall// The ABI values for various atomic memory orderings. 269eda3abe7e183b05834947391c0cdc291f4ee0d8John McCallenum AtomicOrderingKind { 279eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall AO_ABI_memory_order_relaxed = 0, 289eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall AO_ABI_memory_order_consume = 1, 299eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall AO_ABI_memory_order_acquire = 2, 309eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall AO_ABI_memory_order_release = 3, 319eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall AO_ABI_memory_order_acq_rel = 4, 329eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall AO_ABI_memory_order_seq_cst = 5 339eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall}; 349eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 359eda3abe7e183b05834947391c0cdc291f4ee0d8John McCallnamespace { 369eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall class AtomicInfo { 379eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall CodeGenFunction &CGF; 389eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall QualType AtomicTy; 399eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall QualType ValueTy; 409eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall uint64_t AtomicSizeInBits; 419eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall uint64_t ValueSizeInBits; 429eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall CharUnits AtomicAlign; 439eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall CharUnits ValueAlign; 449eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall CharUnits LValueAlign; 459eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall TypeEvaluationKind EvaluationKind; 469eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall bool UseLibcall; 479eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall public: 489eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall AtomicInfo(CodeGenFunction &CGF, LValue &lvalue) : CGF(CGF) { 499eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall assert(lvalue.isSimple()); 509eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 519eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall AtomicTy = lvalue.getType(); 529eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall ValueTy = AtomicTy->castAs<AtomicType>()->getValueType(); 539eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall EvaluationKind = CGF.getEvaluationKind(ValueTy); 549eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 559eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall ASTContext &C = CGF.getContext(); 569eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 579eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall uint64_t valueAlignInBits; 589eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall llvm::tie(ValueSizeInBits, valueAlignInBits) = C.getTypeInfo(ValueTy); 599eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 609eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall uint64_t atomicAlignInBits; 619eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall llvm::tie(AtomicSizeInBits, atomicAlignInBits) = C.getTypeInfo(AtomicTy); 629eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 639eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall assert(ValueSizeInBits <= AtomicSizeInBits); 649eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall assert(valueAlignInBits <= atomicAlignInBits); 659eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 669eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall AtomicAlign = C.toCharUnitsFromBits(atomicAlignInBits); 679eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall ValueAlign = C.toCharUnitsFromBits(valueAlignInBits); 689eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall if (lvalue.getAlignment().isZero()) 699eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall lvalue.setAlignment(AtomicAlign); 709eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 719eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall UseLibcall = 729eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall (AtomicSizeInBits > uint64_t(C.toBits(lvalue.getAlignment())) || 739eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall AtomicSizeInBits > C.getTargetInfo().getMaxAtomicInlineWidth()); 749eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall } 759eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 769eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall QualType getAtomicType() const { return AtomicTy; } 779eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall QualType getValueType() const { return ValueTy; } 789eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall CharUnits getAtomicAlignment() const { return AtomicAlign; } 799eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall CharUnits getValueAlignment() const { return ValueAlign; } 809eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall uint64_t getAtomicSizeInBits() const { return AtomicSizeInBits; } 819eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall uint64_t getValueSizeInBits() const { return AtomicSizeInBits; } 829eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall TypeEvaluationKind getEvaluationKind() const { return EvaluationKind; } 839eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall bool shouldUseLibcall() const { return UseLibcall; } 849eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 859eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall /// Is the atomic size larger than the underlying value type? 869eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall /// 879eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall /// Note that the absence of padding does not mean that atomic 889eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall /// objects are completely interchangeable with non-atomic 899eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall /// objects: we might have promoted the alignment of a type 909eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall /// without making it bigger. 919eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall bool hasPadding() const { 929eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall return (ValueSizeInBits != AtomicSizeInBits); 939eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall } 949eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 959eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall void emitMemSetZeroIfNecessary(LValue dest) const; 969eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 979eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall llvm::Value *getAtomicSizeValue() const { 989eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall CharUnits size = CGF.getContext().toCharUnitsFromBits(AtomicSizeInBits); 999eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall return CGF.CGM.getSize(size); 1009eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall } 1019eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 1029eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall /// Cast the given pointer to an integer pointer suitable for 1039eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall /// atomic operations. 1049eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall llvm::Value *emitCastToAtomicIntPointer(llvm::Value *addr) const; 1059eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 1069eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall /// Turn an atomic-layout object into an r-value. 1079eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall RValue convertTempToRValue(llvm::Value *addr, 1089eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall AggValueSlot resultSlot) const; 1099eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 1109eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall /// Copy an atomic r-value into atomic-layout memory. 1119eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall void emitCopyIntoMemory(RValue rvalue, LValue lvalue) const; 1129eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 1139eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall /// Project an l-value down to the value field. 1149eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall LValue projectValue(LValue lvalue) const { 1159eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall llvm::Value *addr = lvalue.getAddress(); 1169eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall if (hasPadding()) 1179eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall addr = CGF.Builder.CreateStructGEP(addr, 0); 1189eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 1199eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall return LValue::MakeAddr(addr, getValueType(), lvalue.getAlignment(), 1209eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall CGF.getContext(), lvalue.getTBAAInfo()); 1219eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall } 1229eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 1239eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall /// Materialize an atomic r-value in atomic-layout memory. 1249eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall llvm::Value *materializeRValue(RValue rvalue) const; 1259eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 1269eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall private: 1279eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall bool requiresMemSetZero(llvm::Type *type) const; 1289eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall }; 1299eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall} 1309eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 1319eda3abe7e183b05834947391c0cdc291f4ee0d8John McCallstatic RValue emitAtomicLibcall(CodeGenFunction &CGF, 1329eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall StringRef fnName, 1339eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall QualType resultType, 1349eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall CallArgList &args) { 1359eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall const CGFunctionInfo &fnInfo = 1369eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall CGF.CGM.getTypes().arrangeFreeFunctionCall(resultType, args, 1379eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall FunctionType::ExtInfo(), RequiredArgs::All); 1389eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall llvm::FunctionType *fnTy = CGF.CGM.getTypes().GetFunctionType(fnInfo); 1399eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall llvm::Constant *fn = CGF.CGM.CreateRuntimeFunction(fnTy, fnName); 1409eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall return CGF.EmitCall(fnInfo, fn, ReturnValueSlot(), args); 1419eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall} 1429eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 1439eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall/// Does a store of the given IR type modify the full expected width? 1449eda3abe7e183b05834947391c0cdc291f4ee0d8John McCallstatic bool isFullSizeType(CodeGenModule &CGM, llvm::Type *type, 1459eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall uint64_t expectedSize) { 1469eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall return (CGM.getDataLayout().getTypeStoreSize(type) * 8 == expectedSize); 1479eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall} 1489eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 1499eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall/// Does the atomic type require memsetting to zero before initialization? 1509eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall/// 1519eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall/// The IR type is provided as a way of making certain queries faster. 1529eda3abe7e183b05834947391c0cdc291f4ee0d8John McCallbool AtomicInfo::requiresMemSetZero(llvm::Type *type) const { 1539eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall // If the atomic type has size padding, we definitely need a memset. 1549eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall if (hasPadding()) return true; 1559eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 1569eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall // Otherwise, do some simple heuristics to try to avoid it: 1579eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall switch (getEvaluationKind()) { 1589eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall // For scalars and complexes, check whether the store size of the 1599eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall // type uses the full size. 1609eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall case TEK_Scalar: 1619eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall return !isFullSizeType(CGF.CGM, type, AtomicSizeInBits); 1629eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall case TEK_Complex: 1639eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall return !isFullSizeType(CGF.CGM, type->getStructElementType(0), 1649eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall AtomicSizeInBits / 2); 1659eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 1669eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall // Just be pessimistic about aggregates. 1679eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall case TEK_Aggregate: 1689eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall return true; 1699eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall } 1709eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall llvm_unreachable("bad evaluation kind"); 1719eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall} 1729eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 1739eda3abe7e183b05834947391c0cdc291f4ee0d8John McCallvoid AtomicInfo::emitMemSetZeroIfNecessary(LValue dest) const { 1749eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall llvm::Value *addr = dest.getAddress(); 1759eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall if (!requiresMemSetZero(addr->getType()->getPointerElementType())) 1769eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall return; 1779eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 1789eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall CGF.Builder.CreateMemSet(addr, llvm::ConstantInt::get(CGF.Int8Ty, 0), 1799eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall AtomicSizeInBits / 8, 1809eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall dest.getAlignment().getQuantity()); 1819eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall} 1829eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 183fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCallstatic void 184fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCallEmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *E, llvm::Value *Dest, 185fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall llvm::Value *Ptr, llvm::Value *Val1, llvm::Value *Val2, 186fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall uint64_t Size, unsigned Align, llvm::AtomicOrdering Order) { 187fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall llvm::AtomicRMWInst::BinOp Op = llvm::AtomicRMWInst::Add; 188fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall llvm::Instruction::BinaryOps PostOp = (llvm::Instruction::BinaryOps)0; 189fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall 190fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall switch (E->getOp()) { 191fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__c11_atomic_init: 192fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall llvm_unreachable("Already handled!"); 193fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall 194fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__c11_atomic_compare_exchange_strong: 195fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__c11_atomic_compare_exchange_weak: 196fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__atomic_compare_exchange: 197fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__atomic_compare_exchange_n: { 198fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall // Note that cmpxchg only supports specifying one ordering and 199fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall // doesn't support weak cmpxchg, at least at the moment. 200fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall llvm::LoadInst *LoadVal1 = CGF.Builder.CreateLoad(Val1); 201fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall LoadVal1->setAlignment(Align); 202fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall llvm::LoadInst *LoadVal2 = CGF.Builder.CreateLoad(Val2); 203fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall LoadVal2->setAlignment(Align); 204fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall llvm::AtomicCmpXchgInst *CXI = 205fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall CGF.Builder.CreateAtomicCmpXchg(Ptr, LoadVal1, LoadVal2, Order); 206fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall CXI->setVolatile(E->isVolatile()); 207fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall llvm::StoreInst *StoreVal1 = CGF.Builder.CreateStore(CXI, Val1); 208fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall StoreVal1->setAlignment(Align); 209fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall llvm::Value *Cmp = CGF.Builder.CreateICmpEQ(CXI, LoadVal1); 210fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall CGF.EmitStoreOfScalar(Cmp, CGF.MakeAddrLValue(Dest, E->getType())); 211fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall return; 212fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall } 213fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall 214fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__c11_atomic_load: 215fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__atomic_load_n: 216fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__atomic_load: { 217fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall llvm::LoadInst *Load = CGF.Builder.CreateLoad(Ptr); 218fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall Load->setAtomic(Order); 219fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall Load->setAlignment(Size); 220fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall Load->setVolatile(E->isVolatile()); 221fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall llvm::StoreInst *StoreDest = CGF.Builder.CreateStore(Load, Dest); 222fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall StoreDest->setAlignment(Align); 223fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall return; 224fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall } 225fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall 226fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__c11_atomic_store: 227fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__atomic_store: 228fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__atomic_store_n: { 229fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall assert(!Dest && "Store does not return a value"); 230fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall llvm::LoadInst *LoadVal1 = CGF.Builder.CreateLoad(Val1); 231fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall LoadVal1->setAlignment(Align); 232fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall llvm::StoreInst *Store = CGF.Builder.CreateStore(LoadVal1, Ptr); 233fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall Store->setAtomic(Order); 234fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall Store->setAlignment(Size); 235fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall Store->setVolatile(E->isVolatile()); 236fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall return; 237fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall } 238fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall 239fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__c11_atomic_exchange: 240fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__atomic_exchange_n: 241fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__atomic_exchange: 242fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall Op = llvm::AtomicRMWInst::Xchg; 243fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall break; 244fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall 245fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__atomic_add_fetch: 246fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall PostOp = llvm::Instruction::Add; 247fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall // Fall through. 248fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__c11_atomic_fetch_add: 249fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__atomic_fetch_add: 250fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall Op = llvm::AtomicRMWInst::Add; 251fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall break; 252fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall 253fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__atomic_sub_fetch: 254fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall PostOp = llvm::Instruction::Sub; 255fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall // Fall through. 256fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__c11_atomic_fetch_sub: 257fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__atomic_fetch_sub: 258fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall Op = llvm::AtomicRMWInst::Sub; 259fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall break; 260fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall 261fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__atomic_and_fetch: 262fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall PostOp = llvm::Instruction::And; 263fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall // Fall through. 264fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__c11_atomic_fetch_and: 265fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__atomic_fetch_and: 266fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall Op = llvm::AtomicRMWInst::And; 267fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall break; 268fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall 269fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__atomic_or_fetch: 270fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall PostOp = llvm::Instruction::Or; 271fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall // Fall through. 272fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__c11_atomic_fetch_or: 273fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__atomic_fetch_or: 274fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall Op = llvm::AtomicRMWInst::Or; 275fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall break; 276fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall 277fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__atomic_xor_fetch: 278fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall PostOp = llvm::Instruction::Xor; 279fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall // Fall through. 280fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__c11_atomic_fetch_xor: 281fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__atomic_fetch_xor: 282fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall Op = llvm::AtomicRMWInst::Xor; 283fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall break; 284fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall 285fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__atomic_nand_fetch: 286fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall PostOp = llvm::Instruction::And; 287fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall // Fall through. 288fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__atomic_fetch_nand: 289fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall Op = llvm::AtomicRMWInst::Nand; 290fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall break; 291fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall } 292fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall 293fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall llvm::LoadInst *LoadVal1 = CGF.Builder.CreateLoad(Val1); 294fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall LoadVal1->setAlignment(Align); 295fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall llvm::AtomicRMWInst *RMWI = 296fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall CGF.Builder.CreateAtomicRMW(Op, Ptr, LoadVal1, Order); 297fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall RMWI->setVolatile(E->isVolatile()); 298fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall 299fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall // For __atomic_*_fetch operations, perform the operation again to 300fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall // determine the value which was written. 301fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall llvm::Value *Result = RMWI; 302fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall if (PostOp) 303fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall Result = CGF.Builder.CreateBinOp(PostOp, RMWI, LoadVal1); 304fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall if (E->getOp() == AtomicExpr::AO__atomic_nand_fetch) 305fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall Result = CGF.Builder.CreateNot(Result); 306fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall llvm::StoreInst *StoreDest = CGF.Builder.CreateStore(Result, Dest); 307fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall StoreDest->setAlignment(Align); 308fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall} 309fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall 310fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall// This function emits any expression (scalar, complex, or aggregate) 311fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall// into a temporary alloca. 312fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCallstatic llvm::Value * 313fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCallEmitValToTemp(CodeGenFunction &CGF, Expr *E) { 314fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall llvm::Value *DeclPtr = CGF.CreateMemTemp(E->getType(), ".atomictmp"); 315fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall CGF.EmitAnyExprToMem(E, DeclPtr, E->getType().getQualifiers(), 316fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall /*Init*/ true); 317fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall return DeclPtr; 318fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall} 319fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall 320fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCallRValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E, llvm::Value *Dest) { 321fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall QualType AtomicTy = E->getPtr()->getType()->getPointeeType(); 322fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall QualType MemTy = AtomicTy; 323fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall if (const AtomicType *AT = AtomicTy->getAs<AtomicType>()) 324fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall MemTy = AT->getValueType(); 325fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall CharUnits sizeChars = getContext().getTypeSizeInChars(AtomicTy); 326fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall uint64_t Size = sizeChars.getQuantity(); 327fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall CharUnits alignChars = getContext().getTypeAlignInChars(AtomicTy); 328fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall unsigned Align = alignChars.getQuantity(); 329fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall unsigned MaxInlineWidthInBits = 330fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall getContext().getTargetInfo().getMaxAtomicInlineWidth(); 331fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall bool UseLibcall = (Size != Align || 332fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall getContext().toBits(sizeChars) > MaxInlineWidthInBits); 333fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall 334fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall llvm::Value *Ptr, *Order, *OrderFail = 0, *Val1 = 0, *Val2 = 0; 335fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall Ptr = EmitScalarExpr(E->getPtr()); 336fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall 337fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall if (E->getOp() == AtomicExpr::AO__c11_atomic_init) { 338fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall assert(!Dest && "Init does not return a value"); 3399eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall LValue lvalue = LValue::MakeAddr(Ptr, AtomicTy, alignChars, getContext()); 3409eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall EmitAtomicInit(E->getVal1(), lvalue); 3419eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall return RValue::get(0); 342fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall } 343fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall 344fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall Order = EmitScalarExpr(E->getOrder()); 345fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall 346fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall switch (E->getOp()) { 347fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__c11_atomic_init: 348fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall llvm_unreachable("Already handled!"); 349fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall 350fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__c11_atomic_load: 351fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__atomic_load_n: 352fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall break; 353fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall 354fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__atomic_load: 355fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall Dest = EmitScalarExpr(E->getVal1()); 356fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall break; 357fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall 358fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__atomic_store: 359fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall Val1 = EmitScalarExpr(E->getVal1()); 360fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall break; 361fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall 362fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__atomic_exchange: 363fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall Val1 = EmitScalarExpr(E->getVal1()); 364fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall Dest = EmitScalarExpr(E->getVal2()); 365fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall break; 366fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall 367fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__c11_atomic_compare_exchange_strong: 368fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__c11_atomic_compare_exchange_weak: 369fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__atomic_compare_exchange_n: 370fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__atomic_compare_exchange: 371fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall Val1 = EmitScalarExpr(E->getVal1()); 372fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall if (E->getOp() == AtomicExpr::AO__atomic_compare_exchange) 373fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall Val2 = EmitScalarExpr(E->getVal2()); 374fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall else 375fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall Val2 = EmitValToTemp(*this, E->getVal2()); 376fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall OrderFail = EmitScalarExpr(E->getOrderFail()); 377fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall // Evaluate and discard the 'weak' argument. 378fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall if (E->getNumSubExprs() == 6) 379fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall EmitScalarExpr(E->getWeak()); 380fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall break; 381fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall 382fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__c11_atomic_fetch_add: 383fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__c11_atomic_fetch_sub: 384fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall if (MemTy->isPointerType()) { 385fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall // For pointer arithmetic, we're required to do a bit of math: 386fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall // adding 1 to an int* is not the same as adding 1 to a uintptr_t. 387fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall // ... but only for the C11 builtins. The GNU builtins expect the 388fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall // user to multiply by sizeof(T). 389fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall QualType Val1Ty = E->getVal1()->getType(); 390fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall llvm::Value *Val1Scalar = EmitScalarExpr(E->getVal1()); 391fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall CharUnits PointeeIncAmt = 392fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall getContext().getTypeSizeInChars(MemTy->getPointeeType()); 393fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall Val1Scalar = Builder.CreateMul(Val1Scalar, CGM.getSize(PointeeIncAmt)); 394fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall Val1 = CreateMemTemp(Val1Ty, ".atomictmp"); 395fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall EmitStoreOfScalar(Val1Scalar, MakeAddrLValue(Val1, Val1Ty)); 396fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall break; 397fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall } 398fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall // Fall through. 399fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__atomic_fetch_add: 400fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__atomic_fetch_sub: 401fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__atomic_add_fetch: 402fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__atomic_sub_fetch: 403fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__c11_atomic_store: 404fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__c11_atomic_exchange: 405fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__atomic_store_n: 406fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__atomic_exchange_n: 407fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__c11_atomic_fetch_and: 408fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__c11_atomic_fetch_or: 409fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__c11_atomic_fetch_xor: 410fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__atomic_fetch_and: 411fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__atomic_fetch_or: 412fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__atomic_fetch_xor: 413fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__atomic_fetch_nand: 414fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__atomic_and_fetch: 415fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__atomic_or_fetch: 416fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__atomic_xor_fetch: 417fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__atomic_nand_fetch: 418fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall Val1 = EmitValToTemp(*this, E->getVal1()); 419fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall break; 420fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall } 421fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall 422fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall if (!E->getType()->isVoidType() && !Dest) 423fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall Dest = CreateMemTemp(E->getType(), ".atomicdst"); 424fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall 425fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall // Use a library call. See: http://gcc.gnu.org/wiki/Atomic/GCCMM/LIbrary . 426fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall if (UseLibcall) { 427fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall 428fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall SmallVector<QualType, 5> Params; 429fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall CallArgList Args; 430fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall // Size is always the first parameter 431fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall Args.add(RValue::get(llvm::ConstantInt::get(SizeTy, Size)), 432fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall getContext().getSizeType()); 433fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall // Atomic address is always the second parameter 434fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall Args.add(RValue::get(EmitCastToVoidPtr(Ptr)), 435fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall getContext().VoidPtrTy); 436fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall 437fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall const char* LibCallName; 438fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall QualType RetTy = getContext().VoidTy; 439fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall switch (E->getOp()) { 440fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall // There is only one libcall for compare an exchange, because there is no 441fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall // optimisation benefit possible from a libcall version of a weak compare 442fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall // and exchange. 443fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall // bool __atomic_compare_exchange(size_t size, void *obj, void *expected, 444fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall // void *desired, int success, int failure) 445fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__c11_atomic_compare_exchange_weak: 446fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__c11_atomic_compare_exchange_strong: 447fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__atomic_compare_exchange: 448fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__atomic_compare_exchange_n: 449fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall LibCallName = "__atomic_compare_exchange"; 450fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall RetTy = getContext().BoolTy; 451fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall Args.add(RValue::get(EmitCastToVoidPtr(Val1)), 452fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall getContext().VoidPtrTy); 453fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall Args.add(RValue::get(EmitCastToVoidPtr(Val2)), 454fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall getContext().VoidPtrTy); 455fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall Args.add(RValue::get(Order), 456fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall getContext().IntTy); 457fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall Order = OrderFail; 458fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall break; 459fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall // void __atomic_exchange(size_t size, void *mem, void *val, void *return, 460fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall // int order) 461fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__c11_atomic_exchange: 462fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__atomic_exchange_n: 463fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__atomic_exchange: 464fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall LibCallName = "__atomic_exchange"; 465fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall Args.add(RValue::get(EmitCastToVoidPtr(Val1)), 466fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall getContext().VoidPtrTy); 467fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall Args.add(RValue::get(EmitCastToVoidPtr(Dest)), 468fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall getContext().VoidPtrTy); 469fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall break; 470fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall // void __atomic_store(size_t size, void *mem, void *val, int order) 471fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__c11_atomic_store: 472fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__atomic_store: 473fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__atomic_store_n: 474fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall LibCallName = "__atomic_store"; 475fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall Args.add(RValue::get(EmitCastToVoidPtr(Val1)), 476fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall getContext().VoidPtrTy); 477fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall break; 478fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall // void __atomic_load(size_t size, void *mem, void *return, int order) 479fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__c11_atomic_load: 480fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__atomic_load: 481fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__atomic_load_n: 482fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall LibCallName = "__atomic_load"; 483fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall Args.add(RValue::get(EmitCastToVoidPtr(Dest)), 484fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall getContext().VoidPtrTy); 485fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall break; 486fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall#if 0 487fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall // These are only defined for 1-16 byte integers. It is not clear what 488fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall // their semantics would be on anything else... 489fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::Add: LibCallName = "__atomic_fetch_add_generic"; break; 490fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::Sub: LibCallName = "__atomic_fetch_sub_generic"; break; 491fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::And: LibCallName = "__atomic_fetch_and_generic"; break; 492fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::Or: LibCallName = "__atomic_fetch_or_generic"; break; 493fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::Xor: LibCallName = "__atomic_fetch_xor_generic"; break; 494fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall#endif 495fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall default: return EmitUnsupportedRValue(E, "atomic library call"); 496fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall } 497fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall // order is always the last parameter 498fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall Args.add(RValue::get(Order), 499fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall getContext().IntTy); 500fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall 501fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall const CGFunctionInfo &FuncInfo = 502fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall CGM.getTypes().arrangeFreeFunctionCall(RetTy, Args, 503fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall FunctionType::ExtInfo(), RequiredArgs::All); 504fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(FuncInfo); 505fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall llvm::Constant *Func = CGM.CreateRuntimeFunction(FTy, LibCallName); 506fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall RValue Res = EmitCall(FuncInfo, Func, ReturnValueSlot(), Args); 507fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall if (E->isCmpXChg()) 508fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall return Res; 509fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall if (E->getType()->isVoidType()) 510fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall return RValue::get(0); 511fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall return convertTempToRValue(Dest, E->getType()); 512fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall } 513fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall 514fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall bool IsStore = E->getOp() == AtomicExpr::AO__c11_atomic_store || 515fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall E->getOp() == AtomicExpr::AO__atomic_store || 516fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall E->getOp() == AtomicExpr::AO__atomic_store_n; 517fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall bool IsLoad = E->getOp() == AtomicExpr::AO__c11_atomic_load || 518fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall E->getOp() == AtomicExpr::AO__atomic_load || 519fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall E->getOp() == AtomicExpr::AO__atomic_load_n; 520fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall 521fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall llvm::Type *IPtrTy = 522fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall llvm::IntegerType::get(getLLVMContext(), Size * 8)->getPointerTo(); 523fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall llvm::Value *OrigDest = Dest; 524fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall Ptr = Builder.CreateBitCast(Ptr, IPtrTy); 525fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall if (Val1) Val1 = Builder.CreateBitCast(Val1, IPtrTy); 526fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall if (Val2) Val2 = Builder.CreateBitCast(Val2, IPtrTy); 527fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall if (Dest && !E->isCmpXChg()) Dest = Builder.CreateBitCast(Dest, IPtrTy); 528fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall 529fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall if (isa<llvm::ConstantInt>(Order)) { 530fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall int ord = cast<llvm::ConstantInt>(Order)->getZExtValue(); 531fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall switch (ord) { 5329eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall case AO_ABI_memory_order_relaxed: 533fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, Size, Align, 534fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall llvm::Monotonic); 535fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall break; 5369eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall case AO_ABI_memory_order_consume: 5379eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall case AO_ABI_memory_order_acquire: 538fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall if (IsStore) 539fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall break; // Avoid crashing on code with undefined behavior 540fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, Size, Align, 541fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall llvm::Acquire); 542fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall break; 5439eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall case AO_ABI_memory_order_release: 544fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall if (IsLoad) 545fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall break; // Avoid crashing on code with undefined behavior 546fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, Size, Align, 547fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall llvm::Release); 548fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall break; 5499eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall case AO_ABI_memory_order_acq_rel: 550fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall if (IsLoad || IsStore) 551fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall break; // Avoid crashing on code with undefined behavior 552fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, Size, Align, 553fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall llvm::AcquireRelease); 554fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall break; 5559eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall case AO_ABI_memory_order_seq_cst: 556fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, Size, Align, 557fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall llvm::SequentiallyConsistent); 558fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall break; 559fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall default: // invalid order 560fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall // We should not ever get here normally, but it's hard to 561fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall // enforce that in general. 562fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall break; 563fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall } 564fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall if (E->getType()->isVoidType()) 565fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall return RValue::get(0); 566fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall return convertTempToRValue(OrigDest, E->getType()); 567fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall } 568fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall 569fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall // Long case, when Order isn't obviously constant. 570fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall 571fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall // Create all the relevant BB's 572fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall llvm::BasicBlock *MonotonicBB = 0, *AcquireBB = 0, *ReleaseBB = 0, 573fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall *AcqRelBB = 0, *SeqCstBB = 0; 574fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall MonotonicBB = createBasicBlock("monotonic", CurFn); 575fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall if (!IsStore) 576fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall AcquireBB = createBasicBlock("acquire", CurFn); 577fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall if (!IsLoad) 578fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall ReleaseBB = createBasicBlock("release", CurFn); 579fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall if (!IsLoad && !IsStore) 580fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall AcqRelBB = createBasicBlock("acqrel", CurFn); 581fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall SeqCstBB = createBasicBlock("seqcst", CurFn); 582fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall llvm::BasicBlock *ContBB = createBasicBlock("atomic.continue", CurFn); 583fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall 584fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall // Create the switch for the split 585fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall // MonotonicBB is arbitrarily chosen as the default case; in practice, this 586fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall // doesn't matter unless someone is crazy enough to use something that 587fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall // doesn't fold to a constant for the ordering. 588fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall Order = Builder.CreateIntCast(Order, Builder.getInt32Ty(), false); 589fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall llvm::SwitchInst *SI = Builder.CreateSwitch(Order, MonotonicBB); 590fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall 591fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall // Emit all the different atomics 592fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall Builder.SetInsertPoint(MonotonicBB); 593fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, Size, Align, 594fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall llvm::Monotonic); 595fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall Builder.CreateBr(ContBB); 596fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall if (!IsStore) { 597fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall Builder.SetInsertPoint(AcquireBB); 598fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, Size, Align, 599fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall llvm::Acquire); 600fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall Builder.CreateBr(ContBB); 601fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall SI->addCase(Builder.getInt32(1), AcquireBB); 602fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall SI->addCase(Builder.getInt32(2), AcquireBB); 603fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall } 604fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall if (!IsLoad) { 605fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall Builder.SetInsertPoint(ReleaseBB); 606fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, Size, Align, 607fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall llvm::Release); 608fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall Builder.CreateBr(ContBB); 609fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall SI->addCase(Builder.getInt32(3), ReleaseBB); 610fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall } 611fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall if (!IsLoad && !IsStore) { 612fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall Builder.SetInsertPoint(AcqRelBB); 613fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, Size, Align, 614fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall llvm::AcquireRelease); 615fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall Builder.CreateBr(ContBB); 616fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall SI->addCase(Builder.getInt32(4), AcqRelBB); 617fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall } 618fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall Builder.SetInsertPoint(SeqCstBB); 619fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, Size, Align, 620fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall llvm::SequentiallyConsistent); 621fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall Builder.CreateBr(ContBB); 622fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall SI->addCase(Builder.getInt32(5), SeqCstBB); 623fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall 624fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall // Cleanup and return 625fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall Builder.SetInsertPoint(ContBB); 626fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall if (E->getType()->isVoidType()) 627fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall return RValue::get(0); 628fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall return convertTempToRValue(OrigDest, E->getType()); 629fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall} 6309eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 6319eda3abe7e183b05834947391c0cdc291f4ee0d8John McCallllvm::Value *AtomicInfo::emitCastToAtomicIntPointer(llvm::Value *addr) const { 6329eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall unsigned addrspace = 6339eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall cast<llvm::PointerType>(addr->getType())->getAddressSpace(); 6349eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall llvm::IntegerType *ty = 6359eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall llvm::IntegerType::get(CGF.getLLVMContext(), AtomicSizeInBits); 6369eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall return CGF.Builder.CreateBitCast(addr, ty->getPointerTo(addrspace)); 6379eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall} 6389eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 6399eda3abe7e183b05834947391c0cdc291f4ee0d8John McCallRValue AtomicInfo::convertTempToRValue(llvm::Value *addr, 6409eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall AggValueSlot resultSlot) const { 6419eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall if (EvaluationKind == TEK_Aggregate) { 6429eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall // Nothing to do if the result is ignored. 6439eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall if (resultSlot.isIgnored()) return resultSlot.asRValue(); 6449eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 6459eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall assert(resultSlot.getAddr() == addr || hasPadding()); 6469eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 6479eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall // In these cases, we should have emitted directly into the result slot. 6489eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall if (!hasPadding() || resultSlot.isValueOfAtomic()) 6499eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall return resultSlot.asRValue(); 6509eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 6519eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall // Otherwise, fall into the common path. 6529eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall } 6539eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 6549eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall // Drill into the padding structure if we have one. 6559eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall if (hasPadding()) 6569eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall addr = CGF.Builder.CreateStructGEP(addr, 0); 6579eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 6589eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall // If we're emitting to an aggregate, copy into the result slot. 6599eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall if (EvaluationKind == TEK_Aggregate) { 6609eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall CGF.EmitAggregateCopy(resultSlot.getAddr(), addr, getValueType(), 6619eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall resultSlot.isVolatile()); 6629eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall return resultSlot.asRValue(); 6639eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall } 6649eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 6659eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall // Otherwise, just convert the temporary to an r-value using the 6669eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall // normal conversion routine. 6679eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall return CGF.convertTempToRValue(addr, getValueType()); 6689eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall} 6699eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 6709eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall/// Emit a load from an l-value of atomic type. Note that the r-value 6719eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall/// we produce is an r-value of the atomic *value* type. 6729eda3abe7e183b05834947391c0cdc291f4ee0d8John McCallRValue CodeGenFunction::EmitAtomicLoad(LValue src, AggValueSlot resultSlot) { 6739eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall AtomicInfo atomics(*this, src); 6749eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 6759eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall // Check whether we should use a library call. 6769eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall if (atomics.shouldUseLibcall()) { 6779eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall llvm::Value *tempAddr; 6789eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall if (resultSlot.isValueOfAtomic()) { 6799eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall assert(atomics.getEvaluationKind() == TEK_Aggregate); 6809eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall tempAddr = resultSlot.getPaddedAtomicAddr(); 6819eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall } else if (!resultSlot.isIgnored() && !atomics.hasPadding()) { 6829eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall assert(atomics.getEvaluationKind() == TEK_Aggregate); 6839eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall tempAddr = resultSlot.getAddr(); 6849eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall } else { 6859eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall tempAddr = CreateMemTemp(atomics.getAtomicType(), "atomic-load-temp"); 6869eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall } 6879eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 6889eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall // void __atomic_load(size_t size, void *mem, void *return, int order); 6899eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall CallArgList args; 6909eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall args.add(RValue::get(atomics.getAtomicSizeValue()), 6919eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall getContext().getSizeType()); 6929eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall args.add(RValue::get(EmitCastToVoidPtr(src.getAddress())), 6939eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall getContext().VoidPtrTy); 6949eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall args.add(RValue::get(EmitCastToVoidPtr(tempAddr)), 6959eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall getContext().VoidPtrTy); 6969eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall args.add(RValue::get(llvm::ConstantInt::get(IntTy, 6979eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall AO_ABI_memory_order_seq_cst)), 6989eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall getContext().IntTy); 6999eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall emitAtomicLibcall(*this, "__atomic_load", getContext().VoidTy, args); 7009eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 7019eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall // Produce the r-value. 7029eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall return atomics.convertTempToRValue(tempAddr, resultSlot); 7039eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall } 7049eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 7059eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall // Okay, we're doing this natively. 7069eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall llvm::Value *addr = atomics.emitCastToAtomicIntPointer(src.getAddress()); 7079eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall llvm::LoadInst *load = Builder.CreateLoad(addr, "atomic-load"); 7089eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall load->setAtomic(llvm::SequentiallyConsistent); 7099eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 7109eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall // Other decoration. 7119eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall load->setAlignment(src.getAlignment().getQuantity()); 7129eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall if (src.isVolatileQualified()) 7139eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall load->setVolatile(true); 7149eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall if (src.getTBAAInfo()) 7159eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall CGM.DecorateInstruction(load, src.getTBAAInfo()); 7169eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 7179eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall // Okay, turn that back into the original value type. 7189eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall QualType valueType = atomics.getValueType(); 7199eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall llvm::Value *result = load; 7209eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 7219eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall // If we're ignoring an aggregate return, don't do anything. 7229eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall if (atomics.getEvaluationKind() == TEK_Aggregate && resultSlot.isIgnored()) 7239eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall return RValue::getAggregate(0, false); 7249eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 7259eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall // The easiest way to do this this is to go through memory, but we 7269eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall // try not to in some easy cases. 7279eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall if (atomics.getEvaluationKind() == TEK_Scalar && !atomics.hasPadding()) { 7289eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall llvm::Type *resultTy = CGM.getTypes().ConvertTypeForMem(valueType); 7299eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall if (isa<llvm::IntegerType>(resultTy)) { 7309eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall assert(result->getType() == resultTy); 7319eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall result = EmitFromMemory(result, valueType); 7329eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall } else if (isa<llvm::PointerType>(resultTy)) { 7339eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall result = Builder.CreateIntToPtr(result, resultTy); 7349eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall } else { 7359eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall result = Builder.CreateBitCast(result, resultTy); 7369eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall } 7379eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall return RValue::get(result); 7389eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall } 7399eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 7409eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall // Create a temporary. This needs to be big enough to hold the 7419eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall // atomic integer. 7429eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall llvm::Value *temp; 7439eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall bool tempIsVolatile = false; 7449eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall CharUnits tempAlignment; 7459eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall if (atomics.getEvaluationKind() == TEK_Aggregate && 7469eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall (!atomics.hasPadding() || resultSlot.isValueOfAtomic())) { 7479eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall assert(!resultSlot.isIgnored()); 7489eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall if (resultSlot.isValueOfAtomic()) { 7499eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall temp = resultSlot.getPaddedAtomicAddr(); 7509eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall tempAlignment = atomics.getAtomicAlignment(); 7519eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall } else { 7529eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall temp = resultSlot.getAddr(); 7539eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall tempAlignment = atomics.getValueAlignment(); 7549eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall } 7559eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall tempIsVolatile = resultSlot.isVolatile(); 7569eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall } else { 7579eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall temp = CreateMemTemp(atomics.getAtomicType(), "atomic-load-temp"); 7589eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall tempAlignment = atomics.getAtomicAlignment(); 7599eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall } 7609eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 7619eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall // Slam the integer into the temporary. 7629eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall llvm::Value *castTemp = atomics.emitCastToAtomicIntPointer(temp); 7639eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall Builder.CreateAlignedStore(result, castTemp, tempAlignment.getQuantity()) 7649eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall ->setVolatile(tempIsVolatile); 7659eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 7669eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall return atomics.convertTempToRValue(temp, resultSlot); 7679eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall} 7689eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 7699eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 7709eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 7719eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall/// Copy an r-value into memory as part of storing to an atomic type. 7729eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall/// This needs to create a bit-pattern suitable for atomic operations. 7739eda3abe7e183b05834947391c0cdc291f4ee0d8John McCallvoid AtomicInfo::emitCopyIntoMemory(RValue rvalue, LValue dest) const { 7749eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall // If we have an r-value, the rvalue should be of the atomic type, 7759eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall // which means that the caller is responsible for having zeroed 7769eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall // any padding. Just do an aggregate copy of that type. 7779eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall if (rvalue.isAggregate()) { 7789eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall CGF.EmitAggregateCopy(dest.getAddress(), 7799eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall rvalue.getAggregateAddr(), 7809eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall getAtomicType(), 7819eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall (rvalue.isVolatileQualified() 7829eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall || dest.isVolatileQualified()), 7839eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall dest.getAlignment()); 7849eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall return; 7859eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall } 7869eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 7879eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall // Okay, otherwise we're copying stuff. 7889eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 7899eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall // Zero out the buffer if necessary. 7909eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall emitMemSetZeroIfNecessary(dest); 7919eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 7929eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall // Drill past the padding if present. 7939eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall dest = projectValue(dest); 7949eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 7959eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall // Okay, store the rvalue in. 7969eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall if (rvalue.isScalar()) { 7979eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall CGF.EmitStoreOfScalar(rvalue.getScalarVal(), dest, /*init*/ true); 7989eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall } else { 7999eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall CGF.EmitStoreOfComplex(rvalue.getComplexVal(), dest, /*init*/ true); 8009eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall } 8019eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall} 8029eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 8039eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 8049eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall/// Materialize an r-value into memory for the purposes of storing it 8059eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall/// to an atomic type. 8069eda3abe7e183b05834947391c0cdc291f4ee0d8John McCallllvm::Value *AtomicInfo::materializeRValue(RValue rvalue) const { 8079eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall // Aggregate r-values are already in memory, and EmitAtomicStore 8089eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall // requires them to be values of the atomic type. 8099eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall if (rvalue.isAggregate()) 8109eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall return rvalue.getAggregateAddr(); 8119eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 8129eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall // Otherwise, make a temporary and materialize into it. 8139eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall llvm::Value *temp = CGF.CreateMemTemp(getAtomicType(), "atomic-store-temp"); 8149eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall LValue tempLV = CGF.MakeAddrLValue(temp, getAtomicType(), getAtomicAlignment()); 8159eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall emitCopyIntoMemory(rvalue, tempLV); 8169eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall return temp; 8179eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall} 8189eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 8199eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall/// Emit a store to an l-value of atomic type. 8209eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall/// 8219eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall/// Note that the r-value is expected to be an r-value *of the atomic 8229eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall/// type*; this means that for aggregate r-values, it should include 8239eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall/// storage for any padding that was necessary. 8249eda3abe7e183b05834947391c0cdc291f4ee0d8John McCallvoid CodeGenFunction::EmitAtomicStore(RValue rvalue, LValue dest, 8259eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall bool isInit) { 8269eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall // If this is an aggregate r-value, it should agree in type except 8279eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall // maybe for address-space qualification. 8289eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall assert(!rvalue.isAggregate() || 8299eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall rvalue.getAggregateAddr()->getType()->getPointerElementType() 8309eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall == dest.getAddress()->getType()->getPointerElementType()); 8319eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 8329eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall AtomicInfo atomics(*this, dest); 8339eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 8349eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall // If this is an initialization, just put the value there normally. 8359eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall if (isInit) { 8369eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall atomics.emitCopyIntoMemory(rvalue, dest); 8379eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall return; 8389eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall } 8399eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 8409eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall // Check whether we should use a library call. 8419eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall if (atomics.shouldUseLibcall()) { 8429eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall // Produce a source address. 8439eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall llvm::Value *srcAddr = atomics.materializeRValue(rvalue); 8449eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 8459eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall // void __atomic_store(size_t size, void *mem, void *val, int order) 8469eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall CallArgList args; 8479eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall args.add(RValue::get(atomics.getAtomicSizeValue()), 8489eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall getContext().getSizeType()); 8499eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall args.add(RValue::get(EmitCastToVoidPtr(dest.getAddress())), 8509eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall getContext().VoidPtrTy); 8519eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall args.add(RValue::get(EmitCastToVoidPtr(srcAddr)), 8529eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall getContext().VoidPtrTy); 8539eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall args.add(RValue::get(llvm::ConstantInt::get(IntTy, 8549eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall AO_ABI_memory_order_seq_cst)), 8559eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall getContext().IntTy); 8569eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall emitAtomicLibcall(*this, "__atomic_store", getContext().VoidTy, args); 8579eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall return; 8589eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall } 8599eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 8609eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall // Okay, we're doing this natively. 8619eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall llvm::Value *intValue; 8629eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 8639eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall // If we've got a scalar value of the right size, try to avoid going 8649eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall // through memory. 8659eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall if (rvalue.isScalar() && !atomics.hasPadding()) { 8669eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall llvm::Value *value = rvalue.getScalarVal(); 8679eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall if (isa<llvm::IntegerType>(value->getType())) { 8689eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall intValue = value; 8699eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall } else { 8709eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall llvm::IntegerType *inputIntTy = 8719eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall llvm::IntegerType::get(getLLVMContext(), atomics.getValueSizeInBits()); 8729eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall if (isa<llvm::PointerType>(value->getType())) { 8739eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall intValue = Builder.CreatePtrToInt(value, inputIntTy); 8749eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall } else { 8759eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall intValue = Builder.CreateBitCast(value, inputIntTy); 8769eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall } 8779eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall } 8789eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 8799eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall // Otherwise, we need to go through memory. 8809eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall } else { 8819eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall // Put the r-value in memory. 8829eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall llvm::Value *addr = atomics.materializeRValue(rvalue); 8839eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 8849eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall // Cast the temporary to the atomic int type and pull a value out. 8859eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall addr = atomics.emitCastToAtomicIntPointer(addr); 8869eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall intValue = Builder.CreateAlignedLoad(addr, 8879eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall atomics.getAtomicAlignment().getQuantity()); 8889eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall } 8899eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 8909eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall // Do the atomic store. 8919eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall llvm::Value *addr = atomics.emitCastToAtomicIntPointer(dest.getAddress()); 8929eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall llvm::StoreInst *store = Builder.CreateStore(intValue, addr); 8939eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 8949eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall // Initializations don't need to be atomic. 8959eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall if (!isInit) store->setAtomic(llvm::SequentiallyConsistent); 8969eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 8979eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall // Other decoration. 8989eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall store->setAlignment(dest.getAlignment().getQuantity()); 8999eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall if (dest.isVolatileQualified()) 9009eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall store->setVolatile(true); 9019eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall if (dest.getTBAAInfo()) 9029eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall CGM.DecorateInstruction(store, dest.getTBAAInfo()); 9039eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall} 9049eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 9059eda3abe7e183b05834947391c0cdc291f4ee0d8John McCallvoid CodeGenFunction::EmitAtomicInit(Expr *init, LValue dest) { 9069eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall AtomicInfo atomics(*this, dest); 9079eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 9089eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall switch (atomics.getEvaluationKind()) { 9099eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall case TEK_Scalar: { 9109eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall llvm::Value *value = EmitScalarExpr(init); 9119eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall atomics.emitCopyIntoMemory(RValue::get(value), dest); 9129eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall return; 9139eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall } 9149eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 9159eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall case TEK_Complex: { 9169eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall ComplexPairTy value = EmitComplexExpr(init); 9179eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall atomics.emitCopyIntoMemory(RValue::getComplex(value), dest); 9189eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall return; 9199eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall } 9209eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 9219eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall case TEK_Aggregate: { 9229eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall // Memset the buffer first if there's any possibility of 9239eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall // uninitialized internal bits. 9249eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall atomics.emitMemSetZeroIfNecessary(dest); 9259eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 9269eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall // HACK: whether the initializer actually has an atomic type 9279eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall // doesn't really seem reliable right now. 9289eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall if (!init->getType()->isAtomicType()) { 9299eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall dest = atomics.projectValue(dest); 9309eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall } 9319eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 9329eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall // Evaluate the expression directly into the destination. 9339eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall AggValueSlot slot = AggValueSlot::forLValue(dest, 9349eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall AggValueSlot::IsNotDestructed, 9359eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall AggValueSlot::DoesNotNeedGCBarriers, 9369eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall AggValueSlot::IsNotAliased); 9379eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall EmitAggExpr(init, slot); 9389eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall return; 9399eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall } 9409eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall } 9419eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall llvm_unreachable("bad evaluation kind"); 9429eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall} 943