1fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall//===--- CGAtomic.cpp - Emit LLVM IR for atomic operations ----------------===// 2fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall// 3fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall// The LLVM Compiler Infrastructure 4fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall// 5fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall// This file is distributed under the University of Illinois Open Source 6fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall// License. See LICENSE.TXT for details. 7fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall// 8fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall//===----------------------------------------------------------------------===// 9fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall// 10fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall// This file contains the code for emitting atomic operations. 11fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall// 12fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall//===----------------------------------------------------------------------===// 13fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall 14fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall#include "CodeGenFunction.h" 15fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall#include "CGCall.h" 160e2c34f92f00628d48968dfea096d36381f494cbStephen Hines#include "CGRecordLayout.h" 17fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall#include "CodeGenModule.h" 18fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall#include "clang/AST/ASTContext.h" 198b54999a831bb195c08541ca995ef0505c96193fMark Lacey#include "clang/CodeGen/CGFunctionInfo.h" 20e469249726840b8baa2df008ca891e213cc5c661Ed Schouten#include "llvm/ADT/StringExtras.h" 21fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall#include "llvm/IR/DataLayout.h" 22fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall#include "llvm/IR/Intrinsics.h" 239eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall#include "llvm/IR/Operator.h" 24fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall 25fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCallusing namespace clang; 26fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCallusing namespace CodeGen; 27fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall 289eda3abe7e183b05834947391c0cdc291f4ee0d8John McCallnamespace { 299eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall class AtomicInfo { 309eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall CodeGenFunction &CGF; 319eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall QualType AtomicTy; 329eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall QualType ValueTy; 339eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall uint64_t AtomicSizeInBits; 349eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall uint64_t ValueSizeInBits; 359eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall CharUnits AtomicAlign; 369eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall CharUnits ValueAlign; 379eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall CharUnits LValueAlign; 389eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall TypeEvaluationKind EvaluationKind; 399eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall bool UseLibcall; 400e2c34f92f00628d48968dfea096d36381f494cbStephen Hines LValue LVal; 410e2c34f92f00628d48968dfea096d36381f494cbStephen Hines CGBitFieldInfo BFI; 429eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall public: 430e2c34f92f00628d48968dfea096d36381f494cbStephen Hines AtomicInfo(CodeGenFunction &CGF, LValue &lvalue) 440e2c34f92f00628d48968dfea096d36381f494cbStephen Hines : CGF(CGF), AtomicSizeInBits(0), ValueSizeInBits(0), 450e2c34f92f00628d48968dfea096d36381f494cbStephen Hines EvaluationKind(TEK_Scalar), UseLibcall(true) { 460e2c34f92f00628d48968dfea096d36381f494cbStephen Hines assert(!lvalue.isGlobalReg()); 479eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall ASTContext &C = CGF.getContext(); 480e2c34f92f00628d48968dfea096d36381f494cbStephen Hines if (lvalue.isSimple()) { 490e2c34f92f00628d48968dfea096d36381f494cbStephen Hines AtomicTy = lvalue.getType(); 500e2c34f92f00628d48968dfea096d36381f494cbStephen Hines if (auto *ATy = AtomicTy->getAs<AtomicType>()) 510e2c34f92f00628d48968dfea096d36381f494cbStephen Hines ValueTy = ATy->getValueType(); 520e2c34f92f00628d48968dfea096d36381f494cbStephen Hines else 530e2c34f92f00628d48968dfea096d36381f494cbStephen Hines ValueTy = AtomicTy; 540e2c34f92f00628d48968dfea096d36381f494cbStephen Hines EvaluationKind = CGF.getEvaluationKind(ValueTy); 550e2c34f92f00628d48968dfea096d36381f494cbStephen Hines 560e2c34f92f00628d48968dfea096d36381f494cbStephen Hines uint64_t ValueAlignInBits; 570e2c34f92f00628d48968dfea096d36381f494cbStephen Hines uint64_t AtomicAlignInBits; 580e2c34f92f00628d48968dfea096d36381f494cbStephen Hines TypeInfo ValueTI = C.getTypeInfo(ValueTy); 590e2c34f92f00628d48968dfea096d36381f494cbStephen Hines ValueSizeInBits = ValueTI.Width; 600e2c34f92f00628d48968dfea096d36381f494cbStephen Hines ValueAlignInBits = ValueTI.Align; 610e2c34f92f00628d48968dfea096d36381f494cbStephen Hines 620e2c34f92f00628d48968dfea096d36381f494cbStephen Hines TypeInfo AtomicTI = C.getTypeInfo(AtomicTy); 630e2c34f92f00628d48968dfea096d36381f494cbStephen Hines AtomicSizeInBits = AtomicTI.Width; 640e2c34f92f00628d48968dfea096d36381f494cbStephen Hines AtomicAlignInBits = AtomicTI.Align; 650e2c34f92f00628d48968dfea096d36381f494cbStephen Hines 660e2c34f92f00628d48968dfea096d36381f494cbStephen Hines assert(ValueSizeInBits <= AtomicSizeInBits); 670e2c34f92f00628d48968dfea096d36381f494cbStephen Hines assert(ValueAlignInBits <= AtomicAlignInBits); 680e2c34f92f00628d48968dfea096d36381f494cbStephen Hines 690e2c34f92f00628d48968dfea096d36381f494cbStephen Hines AtomicAlign = C.toCharUnitsFromBits(AtomicAlignInBits); 700e2c34f92f00628d48968dfea096d36381f494cbStephen Hines ValueAlign = C.toCharUnitsFromBits(ValueAlignInBits); 710e2c34f92f00628d48968dfea096d36381f494cbStephen Hines if (lvalue.getAlignment().isZero()) 720e2c34f92f00628d48968dfea096d36381f494cbStephen Hines lvalue.setAlignment(AtomicAlign); 730e2c34f92f00628d48968dfea096d36381f494cbStephen Hines 740e2c34f92f00628d48968dfea096d36381f494cbStephen Hines LVal = lvalue; 750e2c34f92f00628d48968dfea096d36381f494cbStephen Hines } else if (lvalue.isBitField()) { 763ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar ValueTy = lvalue.getType(); 773ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar ValueSizeInBits = C.getTypeSize(ValueTy); 780e2c34f92f00628d48968dfea096d36381f494cbStephen Hines auto &OrigBFI = lvalue.getBitFieldInfo(); 790e2c34f92f00628d48968dfea096d36381f494cbStephen Hines auto Offset = OrigBFI.Offset % C.toBits(lvalue.getAlignment()); 800e2c34f92f00628d48968dfea096d36381f494cbStephen Hines AtomicSizeInBits = C.toBits( 810e2c34f92f00628d48968dfea096d36381f494cbStephen Hines C.toCharUnitsFromBits(Offset + OrigBFI.Size + C.getCharWidth() - 1) 824967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar .alignTo(lvalue.getAlignment())); 8387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar auto VoidPtrAddr = CGF.EmitCastToVoidPtr(lvalue.getBitFieldPointer()); 840e2c34f92f00628d48968dfea096d36381f494cbStephen Hines auto OffsetInChars = 850e2c34f92f00628d48968dfea096d36381f494cbStephen Hines (C.toCharUnitsFromBits(OrigBFI.Offset) / lvalue.getAlignment()) * 860e2c34f92f00628d48968dfea096d36381f494cbStephen Hines lvalue.getAlignment(); 870e2c34f92f00628d48968dfea096d36381f494cbStephen Hines VoidPtrAddr = CGF.Builder.CreateConstGEP1_64( 880e2c34f92f00628d48968dfea096d36381f494cbStephen Hines VoidPtrAddr, OffsetInChars.getQuantity()); 890e2c34f92f00628d48968dfea096d36381f494cbStephen Hines auto Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( 900e2c34f92f00628d48968dfea096d36381f494cbStephen Hines VoidPtrAddr, 910e2c34f92f00628d48968dfea096d36381f494cbStephen Hines CGF.Builder.getIntNTy(AtomicSizeInBits)->getPointerTo(), 920e2c34f92f00628d48968dfea096d36381f494cbStephen Hines "atomic_bitfield_base"); 930e2c34f92f00628d48968dfea096d36381f494cbStephen Hines BFI = OrigBFI; 940e2c34f92f00628d48968dfea096d36381f494cbStephen Hines BFI.Offset = Offset; 950e2c34f92f00628d48968dfea096d36381f494cbStephen Hines BFI.StorageSize = AtomicSizeInBits; 9687d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar BFI.StorageOffset += OffsetInChars; 9787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar LVal = LValue::MakeBitfield(Address(Addr, lvalue.getAlignment()), 9887d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar BFI, lvalue.getType(), 9987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar lvalue.getAlignmentSource()); 1003ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar LVal.setTBAAInfo(lvalue.getTBAAInfo()); 1013ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar AtomicTy = C.getIntTypeForBitwidth(AtomicSizeInBits, OrigBFI.IsSigned); 1023ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar if (AtomicTy.isNull()) { 1033ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar llvm::APInt Size( 1043ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar /*numBits=*/32, 1053ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar C.toCharUnitsFromBits(AtomicSizeInBits).getQuantity()); 1063ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar AtomicTy = C.getConstantArrayType(C.CharTy, Size, ArrayType::Normal, 1073ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar /*IndexTypeQuals=*/0); 1083ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar } 1093ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar AtomicAlign = ValueAlign = lvalue.getAlignment(); 1100e2c34f92f00628d48968dfea096d36381f494cbStephen Hines } else if (lvalue.isVectorElt()) { 1113ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar ValueTy = lvalue.getType()->getAs<VectorType>()->getElementType(); 1123ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar ValueSizeInBits = C.getTypeSize(ValueTy); 1133ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar AtomicTy = lvalue.getType(); 1143ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar AtomicSizeInBits = C.getTypeSize(AtomicTy); 1153ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar AtomicAlign = ValueAlign = lvalue.getAlignment(); 1160e2c34f92f00628d48968dfea096d36381f494cbStephen Hines LVal = lvalue; 1170e2c34f92f00628d48968dfea096d36381f494cbStephen Hines } else { 1180e2c34f92f00628d48968dfea096d36381f494cbStephen Hines assert(lvalue.isExtVectorElt()); 1193ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar ValueTy = lvalue.getType(); 1203ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar ValueSizeInBits = C.getTypeSize(ValueTy); 1213ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar AtomicTy = ValueTy = CGF.getContext().getExtVectorType( 12287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar lvalue.getType(), lvalue.getExtVectorAddress() 12387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar .getElementType()->getVectorNumElements()); 1243ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar AtomicSizeInBits = C.getTypeSize(AtomicTy); 1253ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar AtomicAlign = ValueAlign = lvalue.getAlignment(); 1260e2c34f92f00628d48968dfea096d36381f494cbStephen Hines LVal = lvalue; 1270e2c34f92f00628d48968dfea096d36381f494cbStephen Hines } 1280e2c34f92f00628d48968dfea096d36381f494cbStephen Hines UseLibcall = !C.getTargetInfo().hasBuiltinAtomic( 1290e2c34f92f00628d48968dfea096d36381f494cbStephen Hines AtomicSizeInBits, C.toBits(lvalue.getAlignment())); 1309eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall } 1319eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 1329eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall QualType getAtomicType() const { return AtomicTy; } 1339eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall QualType getValueType() const { return ValueTy; } 1349eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall CharUnits getAtomicAlignment() const { return AtomicAlign; } 1359eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall CharUnits getValueAlignment() const { return ValueAlign; } 1369eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall uint64_t getAtomicSizeInBits() const { return AtomicSizeInBits; } 1370e2c34f92f00628d48968dfea096d36381f494cbStephen Hines uint64_t getValueSizeInBits() const { return ValueSizeInBits; } 1389eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall TypeEvaluationKind getEvaluationKind() const { return EvaluationKind; } 1399eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall bool shouldUseLibcall() const { return UseLibcall; } 1400e2c34f92f00628d48968dfea096d36381f494cbStephen Hines const LValue &getAtomicLValue() const { return LVal; } 14187d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar llvm::Value *getAtomicPointer() const { 1423ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar if (LVal.isSimple()) 14387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return LVal.getPointer(); 1443ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar else if (LVal.isBitField()) 14587d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return LVal.getBitFieldPointer(); 1463ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar else if (LVal.isVectorElt()) 14787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return LVal.getVectorPointer(); 1483ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar assert(LVal.isExtVectorElt()); 14987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return LVal.getExtVectorPointer(); 15087d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar } 15187d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Address getAtomicAddress() const { 15287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return Address(getAtomicPointer(), getAtomicAlignment()); 15387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar } 15487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar 15587d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Address getAtomicAddressAsAtomicIntPointer() const { 15687d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return emitCastToAtomicIntPointer(getAtomicAddress()); 1573ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar } 1589eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 1599eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall /// Is the atomic size larger than the underlying value type? 1609eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall /// 1619eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall /// Note that the absence of padding does not mean that atomic 1629eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall /// objects are completely interchangeable with non-atomic 1639eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall /// objects: we might have promoted the alignment of a type 1649eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall /// without making it bigger. 1659eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall bool hasPadding() const { 1669eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall return (ValueSizeInBits != AtomicSizeInBits); 1679eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall } 1689eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 1690e2c34f92f00628d48968dfea096d36381f494cbStephen Hines bool emitMemSetZeroIfNecessary() const; 1709eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 1719eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall llvm::Value *getAtomicSizeValue() const { 1729eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall CharUnits size = CGF.getContext().toCharUnitsFromBits(AtomicSizeInBits); 1739eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall return CGF.CGM.getSize(size); 1749eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall } 1759eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 17687d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar /// Cast the given pointer to an integer pointer suitable for atomic 17787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar /// operations if the source. 17887d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Address emitCastToAtomicIntPointer(Address Addr) const; 17987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar 18087d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar /// If Addr is compatible with the iN that will be used for an atomic 18187d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar /// operation, bitcast it. Otherwise, create a temporary that is suitable 18287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar /// and copy the value across. 18387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Address convertToAtomicIntPointer(Address Addr) const; 1849eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 1859eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall /// Turn an atomic-layout object into an r-value. 18687d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar RValue convertAtomicTempToRValue(Address addr, AggValueSlot resultSlot, 18787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar SourceLocation loc, bool AsValue) const; 1889eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 1890e2c34f92f00628d48968dfea096d36381f494cbStephen Hines /// \brief Converts a rvalue to integer value. 1900e2c34f92f00628d48968dfea096d36381f494cbStephen Hines llvm::Value *convertRValueToInt(RValue RVal) const; 1910e2c34f92f00628d48968dfea096d36381f494cbStephen Hines 1923ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar RValue ConvertIntToValueOrAtomic(llvm::Value *IntVal, 1933ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar AggValueSlot ResultSlot, 1943ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar SourceLocation Loc, bool AsValue) const; 1950e2c34f92f00628d48968dfea096d36381f494cbStephen Hines 1969eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall /// Copy an atomic r-value into atomic-layout memory. 1970e2c34f92f00628d48968dfea096d36381f494cbStephen Hines void emitCopyIntoMemory(RValue rvalue) const; 1989eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 1999eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall /// Project an l-value down to the value field. 2000e2c34f92f00628d48968dfea096d36381f494cbStephen Hines LValue projectValue() const { 2010e2c34f92f00628d48968dfea096d36381f494cbStephen Hines assert(LVal.isSimple()); 20287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Address addr = getAtomicAddress(); 2039eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall if (hasPadding()) 20487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar addr = CGF.Builder.CreateStructGEP(addr, 0, CharUnits()); 2059eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 20687d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return LValue::MakeAddr(addr, getValueType(), CGF.getContext(), 20787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar LVal.getAlignmentSource(), LVal.getTBAAInfo()); 2089eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall } 2099eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 2103ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar /// \brief Emits atomic load. 2113ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar /// \returns Loaded value. 2123ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar RValue EmitAtomicLoad(AggValueSlot ResultSlot, SourceLocation Loc, 2133ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar bool AsValue, llvm::AtomicOrdering AO, 2143ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar bool IsVolatile); 2153ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar 2163ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar /// \brief Emits atomic compare-and-exchange sequence. 2173ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar /// \param Expected Expected value. 2183ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar /// \param Desired Desired value. 2193ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar /// \param Success Atomic ordering for success operation. 2203ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar /// \param Failure Atomic ordering for failed operation. 2213ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar /// \param IsWeak true if atomic operation is weak, false otherwise. 2223ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar /// \returns Pair of values: previous value from storage (value type) and 2233ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar /// boolean flag (i1 type) with true if success and false otherwise. 2244967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar std::pair<RValue, llvm::Value *> 2254967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar EmitAtomicCompareExchange(RValue Expected, RValue Desired, 2264967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar llvm::AtomicOrdering Success = 2274967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar llvm::AtomicOrdering::SequentiallyConsistent, 2284967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar llvm::AtomicOrdering Failure = 2294967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar llvm::AtomicOrdering::SequentiallyConsistent, 2304967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar bool IsWeak = false); 2313ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar 232b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar /// \brief Emits atomic update. 233b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar /// \param AO Atomic ordering. 234b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar /// \param UpdateOp Update operation for the current lvalue. 235b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar void EmitAtomicUpdate(llvm::AtomicOrdering AO, 236b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar const llvm::function_ref<RValue(RValue)> &UpdateOp, 237b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar bool IsVolatile); 238b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar /// \brief Emits atomic update. 239b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar /// \param AO Atomic ordering. 240b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar void EmitAtomicUpdate(llvm::AtomicOrdering AO, RValue UpdateRVal, 241b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar bool IsVolatile); 242b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar 2439eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall /// Materialize an atomic r-value in atomic-layout memory. 24487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Address materializeRValue(RValue rvalue) const; 2459eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 24687d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar /// \brief Creates temp alloca for intermediate operations on atomic value. 24787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Address CreateTempAlloca() const; 2489eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall private: 2499eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall bool requiresMemSetZero(llvm::Type *type) const; 2503ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar 2513ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar 2523ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar /// \brief Emits atomic load as a libcall. 2533ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar void EmitAtomicLoadLibcall(llvm::Value *AddForLoaded, 2543ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar llvm::AtomicOrdering AO, bool IsVolatile); 2553ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar /// \brief Emits atomic load as LLVM instruction. 2563ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar llvm::Value *EmitAtomicLoadOp(llvm::AtomicOrdering AO, bool IsVolatile); 2573ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar /// \brief Emits atomic compare-and-exchange op as a libcall. 258b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar llvm::Value *EmitAtomicCompareExchangeLibcall( 259b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar llvm::Value *ExpectedAddr, llvm::Value *DesiredAddr, 2604967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar llvm::AtomicOrdering Success = 2614967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar llvm::AtomicOrdering::SequentiallyConsistent, 2624967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar llvm::AtomicOrdering Failure = 2634967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar llvm::AtomicOrdering::SequentiallyConsistent); 2643ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar /// \brief Emits atomic compare-and-exchange op as LLVM instruction. 265b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar std::pair<llvm::Value *, llvm::Value *> EmitAtomicCompareExchangeOp( 266b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar llvm::Value *ExpectedVal, llvm::Value *DesiredVal, 2674967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar llvm::AtomicOrdering Success = 2684967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar llvm::AtomicOrdering::SequentiallyConsistent, 2694967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar llvm::AtomicOrdering Failure = 2704967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar llvm::AtomicOrdering::SequentiallyConsistent, 2713ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar bool IsWeak = false); 272b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar /// \brief Emit atomic update as libcalls. 273b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar void 274b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO, 275b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar const llvm::function_ref<RValue(RValue)> &UpdateOp, 276b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar bool IsVolatile); 277b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar /// \brief Emit atomic update as LLVM instructions. 278b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar void EmitAtomicUpdateOp(llvm::AtomicOrdering AO, 279b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar const llvm::function_ref<RValue(RValue)> &UpdateOp, 280b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar bool IsVolatile); 281b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar /// \brief Emit atomic update as libcalls. 282b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar void EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO, RValue UpdateRVal, 283b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar bool IsVolatile); 284b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar /// \brief Emit atomic update as LLVM instructions. 285b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar void EmitAtomicUpdateOp(llvm::AtomicOrdering AO, RValue UpdateRal, 286b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar bool IsVolatile); 2879eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall }; 2889eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall} 2899eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 29087d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga NainarAddress AtomicInfo::CreateTempAlloca() const { 29187d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Address TempAlloca = CGF.CreateMemTemp( 2923ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar (LVal.isBitField() && ValueSizeInBits > AtomicSizeInBits) ? ValueTy 2933ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar : AtomicTy, 29487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar getAtomicAlignment(), 2953ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar "atomic-temp"); 2963ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar // Cast to pointer to value type for bitfields. 2973ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar if (LVal.isBitField()) 2983ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar return CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( 29987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar TempAlloca, getAtomicAddress().getType()); 3003ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar return TempAlloca; 3013ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar} 3023ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar 3039eda3abe7e183b05834947391c0cdc291f4ee0d8John McCallstatic RValue emitAtomicLibcall(CodeGenFunction &CGF, 3049eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall StringRef fnName, 3059eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall QualType resultType, 3069eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall CallArgList &args) { 3079eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall const CGFunctionInfo &fnInfo = 3084967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar CGF.CGM.getTypes().arrangeBuiltinFunctionCall(resultType, args); 3099eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall llvm::FunctionType *fnTy = CGF.CGM.getTypes().GetFunctionType(fnInfo); 3109eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall llvm::Constant *fn = CGF.CGM.CreateRuntimeFunction(fnTy, fnName); 3119eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall return CGF.EmitCall(fnInfo, fn, ReturnValueSlot(), args); 3129eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall} 3139eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 3149eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall/// Does a store of the given IR type modify the full expected width? 3159eda3abe7e183b05834947391c0cdc291f4ee0d8John McCallstatic bool isFullSizeType(CodeGenModule &CGM, llvm::Type *type, 3169eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall uint64_t expectedSize) { 3179eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall return (CGM.getDataLayout().getTypeStoreSize(type) * 8 == expectedSize); 3189eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall} 3199eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 3209eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall/// Does the atomic type require memsetting to zero before initialization? 3219eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall/// 3229eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall/// The IR type is provided as a way of making certain queries faster. 3239eda3abe7e183b05834947391c0cdc291f4ee0d8John McCallbool AtomicInfo::requiresMemSetZero(llvm::Type *type) const { 3249eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall // If the atomic type has size padding, we definitely need a memset. 3259eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall if (hasPadding()) return true; 3269eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 3279eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall // Otherwise, do some simple heuristics to try to avoid it: 3289eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall switch (getEvaluationKind()) { 3299eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall // For scalars and complexes, check whether the store size of the 3309eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall // type uses the full size. 3319eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall case TEK_Scalar: 3329eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall return !isFullSizeType(CGF.CGM, type, AtomicSizeInBits); 3339eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall case TEK_Complex: 3349eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall return !isFullSizeType(CGF.CGM, type->getStructElementType(0), 3359eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall AtomicSizeInBits / 2); 3369eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 337336d9df5e628279425344d754dc68047fa5b00a7Eli Friedman // Padding in structs has an undefined bit pattern. User beware. 3389eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall case TEK_Aggregate: 339336d9df5e628279425344d754dc68047fa5b00a7Eli Friedman return false; 3409eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall } 3419eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall llvm_unreachable("bad evaluation kind"); 3429eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall} 3439eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 3440e2c34f92f00628d48968dfea096d36381f494cbStephen Hinesbool AtomicInfo::emitMemSetZeroIfNecessary() const { 3450e2c34f92f00628d48968dfea096d36381f494cbStephen Hines assert(LVal.isSimple()); 34687d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar llvm::Value *addr = LVal.getPointer(); 3479eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall if (!requiresMemSetZero(addr->getType()->getPointerElementType())) 348336d9df5e628279425344d754dc68047fa5b00a7Eli Friedman return false; 3499eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 3503ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar CGF.Builder.CreateMemSet( 3513ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar addr, llvm::ConstantInt::get(CGF.Int8Ty, 0), 3523ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar CGF.getContext().toCharUnitsFromBits(AtomicSizeInBits).getQuantity(), 3533ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar LVal.getAlignment().getQuantity()); 354336d9df5e628279425344d754dc68047fa5b00a7Eli Friedman return true; 3559eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall} 3569eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 357c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hinesstatic void emitAtomicCmpXchg(CodeGenFunction &CGF, AtomicExpr *E, bool IsWeak, 35887d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Address Dest, Address Ptr, 35987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Address Val1, Address Val2, 36087d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar uint64_t Size, 361651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::AtomicOrdering SuccessOrder, 362651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::AtomicOrdering FailureOrder) { 363651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // Note that cmpxchg doesn't support weak cmpxchg, at least at the moment. 36487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar llvm::Value *Expected = CGF.Builder.CreateLoad(Val1); 36587d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar llvm::Value *Desired = CGF.Builder.CreateLoad(Val2); 366651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 367c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines llvm::AtomicCmpXchgInst *Pair = CGF.Builder.CreateAtomicCmpXchg( 36887d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Ptr.getPointer(), Expected, Desired, SuccessOrder, FailureOrder); 369c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines Pair->setVolatile(E->isVolatile()); 370c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines Pair->setWeak(IsWeak); 371651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 372651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // Cmp holds the result of the compare-exchange operation: true on success, 373651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // false on failure. 374c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines llvm::Value *Old = CGF.Builder.CreateExtractValue(Pair, 0); 375c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines llvm::Value *Cmp = CGF.Builder.CreateExtractValue(Pair, 1); 376651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 377651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // This basic block is used to hold the store instruction if the operation 378651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // failed. 379651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::BasicBlock *StoreExpectedBB = 380651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines CGF.createBasicBlock("cmpxchg.store_expected", CGF.CurFn); 381651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 382651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // This basic block is the exit point of the operation, we should end up 383651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // here regardless of whether or not the operation succeeded. 384651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::BasicBlock *ContinueBB = 385651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines CGF.createBasicBlock("cmpxchg.continue", CGF.CurFn); 386651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 387651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // Update Expected if Expected isn't equal to Old, otherwise branch to the 388651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // exit point. 389651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines CGF.Builder.CreateCondBr(Cmp, ContinueBB, StoreExpectedBB); 390651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 391651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines CGF.Builder.SetInsertPoint(StoreExpectedBB); 392651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // Update the memory at Expected with Old's value. 39387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar CGF.Builder.CreateStore(Old, Val1); 394651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // Finally, branch to the exit point. 395651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines CGF.Builder.CreateBr(ContinueBB); 396651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 397651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines CGF.Builder.SetInsertPoint(ContinueBB); 398651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // Update the memory at Dest with Cmp's value. 399651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines CGF.EmitStoreOfScalar(Cmp, CGF.MakeAddrLValue(Dest, E->getType())); 400651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines} 401651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 402651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines/// Given an ordering required on success, emit all possible cmpxchg 403651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines/// instructions to cope with the provided (but possibly only dynamically known) 404651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines/// FailureOrder. 405651f13cea278ec967336033dd032faef0e9fc2ecStephen Hinesstatic void emitAtomicCmpXchgFailureSet(CodeGenFunction &CGF, AtomicExpr *E, 4064967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar bool IsWeak, Address Dest, Address Ptr, 4074967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Address Val1, Address Val2, 408651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::Value *FailureOrderVal, 40987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar uint64_t Size, 410651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::AtomicOrdering SuccessOrder) { 411651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::AtomicOrdering FailureOrder; 412651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines if (llvm::ConstantInt *FO = dyn_cast<llvm::ConstantInt>(FailureOrderVal)) { 4134967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar auto FOS = FO->getSExtValue(); 4144967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar if (!llvm::isValidAtomicOrderingCABI(FOS)) 4154967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar FailureOrder = llvm::AtomicOrdering::Monotonic; 4164967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar else 4174967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar switch ((llvm::AtomicOrderingCABI)FOS) { 4184967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case llvm::AtomicOrderingCABI::relaxed: 4194967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case llvm::AtomicOrderingCABI::release: 4204967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case llvm::AtomicOrderingCABI::acq_rel: 4214967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar FailureOrder = llvm::AtomicOrdering::Monotonic; 4224967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar break; 4234967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case llvm::AtomicOrderingCABI::consume: 4244967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case llvm::AtomicOrderingCABI::acquire: 4254967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar FailureOrder = llvm::AtomicOrdering::Acquire; 4264967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar break; 4274967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case llvm::AtomicOrderingCABI::seq_cst: 4284967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar FailureOrder = llvm::AtomicOrdering::SequentiallyConsistent; 4294967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar break; 4304967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar } 4314967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar if (isStrongerThan(FailureOrder, SuccessOrder)) { 4324967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar // Don't assert on undefined behavior "failure argument shall be no 4334967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar // stronger than the success argument". 434651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines FailureOrder = 4354967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(SuccessOrder); 436651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 4374967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, Size, SuccessOrder, 4384967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar FailureOrder); 439651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines return; 440651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 441651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 442651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // Create all the relevant BB's 4436bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines llvm::BasicBlock *MonotonicBB = nullptr, *AcquireBB = nullptr, 4446bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines *SeqCstBB = nullptr; 445651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines MonotonicBB = CGF.createBasicBlock("monotonic_fail", CGF.CurFn); 4464967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar if (SuccessOrder != llvm::AtomicOrdering::Monotonic && 4474967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar SuccessOrder != llvm::AtomicOrdering::Release) 448651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines AcquireBB = CGF.createBasicBlock("acquire_fail", CGF.CurFn); 4494967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar if (SuccessOrder == llvm::AtomicOrdering::SequentiallyConsistent) 450651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines SeqCstBB = CGF.createBasicBlock("seqcst_fail", CGF.CurFn); 451651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 452651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::BasicBlock *ContBB = CGF.createBasicBlock("atomic.continue", CGF.CurFn); 453651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 454651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines llvm::SwitchInst *SI = CGF.Builder.CreateSwitch(FailureOrderVal, MonotonicBB); 455651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 456651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // Emit all the different atomics 457651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 458651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // MonotonicBB is arbitrarily chosen as the default case; in practice, this 459651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // doesn't matter unless someone is crazy enough to use something that 460651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines // doesn't fold to a constant for the ordering. 461651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines CGF.Builder.SetInsertPoint(MonotonicBB); 462c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, 4634967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Size, SuccessOrder, llvm::AtomicOrdering::Monotonic); 464651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines CGF.Builder.CreateBr(ContBB); 465651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 466651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines if (AcquireBB) { 467651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines CGF.Builder.SetInsertPoint(AcquireBB); 468c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, 4694967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Size, SuccessOrder, llvm::AtomicOrdering::Acquire); 470651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines CGF.Builder.CreateBr(ContBB); 4714967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar SI->addCase(CGF.Builder.getInt32((int)llvm::AtomicOrderingCABI::consume), 472651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines AcquireBB); 4734967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar SI->addCase(CGF.Builder.getInt32((int)llvm::AtomicOrderingCABI::acquire), 474651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines AcquireBB); 475651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 476651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines if (SeqCstBB) { 477651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines CGF.Builder.SetInsertPoint(SeqCstBB); 4784967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, Size, SuccessOrder, 4794967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar llvm::AtomicOrdering::SequentiallyConsistent); 480651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines CGF.Builder.CreateBr(ContBB); 4814967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar SI->addCase(CGF.Builder.getInt32((int)llvm::AtomicOrderingCABI::seq_cst), 482651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines SeqCstBB); 483651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines } 484651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 485651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines CGF.Builder.SetInsertPoint(ContBB); 486651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines} 487651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines 48887d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainarstatic void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *E, Address Dest, 48987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Address Ptr, Address Val1, Address Val2, 490c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines llvm::Value *IsWeak, llvm::Value *FailureOrder, 49187d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar uint64_t Size, llvm::AtomicOrdering Order) { 492fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall llvm::AtomicRMWInst::BinOp Op = llvm::AtomicRMWInst::Add; 493fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall llvm::Instruction::BinaryOps PostOp = (llvm::Instruction::BinaryOps)0; 494fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall 495fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall switch (E->getOp()) { 496fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__c11_atomic_init: 497fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall llvm_unreachable("Already handled!"); 498fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall 499fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__c11_atomic_compare_exchange_strong: 500c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines emitAtomicCmpXchgFailureSet(CGF, E, false, Dest, Ptr, Val1, Val2, 50187d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar FailureOrder, Size, Order); 502c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines return; 503fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__c11_atomic_compare_exchange_weak: 504c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines emitAtomicCmpXchgFailureSet(CGF, E, true, Dest, Ptr, Val1, Val2, 50587d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar FailureOrder, Size, Order); 506c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines return; 507fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__atomic_compare_exchange: 508c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines case AtomicExpr::AO__atomic_compare_exchange_n: { 509c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines if (llvm::ConstantInt *IsWeakC = dyn_cast<llvm::ConstantInt>(IsWeak)) { 510c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines emitAtomicCmpXchgFailureSet(CGF, E, IsWeakC->getZExtValue(), Dest, Ptr, 51187d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Val1, Val2, FailureOrder, Size, Order); 512c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines } else { 513c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines // Create all the relevant BB's 514c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines llvm::BasicBlock *StrongBB = 515c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines CGF.createBasicBlock("cmpxchg.strong", CGF.CurFn); 516c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines llvm::BasicBlock *WeakBB = CGF.createBasicBlock("cmxchg.weak", CGF.CurFn); 517c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines llvm::BasicBlock *ContBB = 518c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines CGF.createBasicBlock("cmpxchg.continue", CGF.CurFn); 519c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines 520c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines llvm::SwitchInst *SI = CGF.Builder.CreateSwitch(IsWeak, WeakBB); 521c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines SI->addCase(CGF.Builder.getInt1(false), StrongBB); 522c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines 523c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines CGF.Builder.SetInsertPoint(StrongBB); 524c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines emitAtomicCmpXchgFailureSet(CGF, E, false, Dest, Ptr, Val1, Val2, 52587d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar FailureOrder, Size, Order); 526c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines CGF.Builder.CreateBr(ContBB); 527c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines 528c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines CGF.Builder.SetInsertPoint(WeakBB); 529c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines emitAtomicCmpXchgFailureSet(CGF, E, true, Dest, Ptr, Val1, Val2, 53087d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar FailureOrder, Size, Order); 531c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines CGF.Builder.CreateBr(ContBB); 532c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines 533c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines CGF.Builder.SetInsertPoint(ContBB); 534c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines } 535fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall return; 536c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines } 537fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__c11_atomic_load: 538fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__atomic_load_n: 539fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__atomic_load: { 540fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall llvm::LoadInst *Load = CGF.Builder.CreateLoad(Ptr); 541fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall Load->setAtomic(Order); 542fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall Load->setVolatile(E->isVolatile()); 54387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar CGF.Builder.CreateStore(Load, Dest); 544fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall return; 545fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall } 546fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall 547fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__c11_atomic_store: 548fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__atomic_store: 549fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__atomic_store_n: { 55087d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar llvm::Value *LoadVal1 = CGF.Builder.CreateLoad(Val1); 551fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall llvm::StoreInst *Store = CGF.Builder.CreateStore(LoadVal1, Ptr); 552fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall Store->setAtomic(Order); 553fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall Store->setVolatile(E->isVolatile()); 554fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall return; 555fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall } 556fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall 557fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__c11_atomic_exchange: 558fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__atomic_exchange_n: 559fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__atomic_exchange: 560fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall Op = llvm::AtomicRMWInst::Xchg; 561fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall break; 562fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall 563fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__atomic_add_fetch: 564fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall PostOp = llvm::Instruction::Add; 565fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall // Fall through. 566fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__c11_atomic_fetch_add: 567fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__atomic_fetch_add: 568fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall Op = llvm::AtomicRMWInst::Add; 569fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall break; 570fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall 571fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__atomic_sub_fetch: 572fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall PostOp = llvm::Instruction::Sub; 573fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall // Fall through. 574fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__c11_atomic_fetch_sub: 575fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__atomic_fetch_sub: 576fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall Op = llvm::AtomicRMWInst::Sub; 577fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall break; 578fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall 579fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__atomic_and_fetch: 580fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall PostOp = llvm::Instruction::And; 581fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall // Fall through. 582fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__c11_atomic_fetch_and: 583fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__atomic_fetch_and: 584fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall Op = llvm::AtomicRMWInst::And; 585fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall break; 586fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall 587fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__atomic_or_fetch: 588fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall PostOp = llvm::Instruction::Or; 589fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall // Fall through. 590fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__c11_atomic_fetch_or: 591fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__atomic_fetch_or: 592fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall Op = llvm::AtomicRMWInst::Or; 593fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall break; 594fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall 595fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__atomic_xor_fetch: 596fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall PostOp = llvm::Instruction::Xor; 597fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall // Fall through. 598fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__c11_atomic_fetch_xor: 599fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__atomic_fetch_xor: 600fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall Op = llvm::AtomicRMWInst::Xor; 601fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall break; 602fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall 603fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__atomic_nand_fetch: 60487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar PostOp = llvm::Instruction::And; // the NOT is special cased below 60587d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar // Fall through. 606fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__atomic_fetch_nand: 607fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall Op = llvm::AtomicRMWInst::Nand; 608fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall break; 609fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall } 610fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall 61187d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar llvm::Value *LoadVal1 = CGF.Builder.CreateLoad(Val1); 612fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall llvm::AtomicRMWInst *RMWI = 61387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar CGF.Builder.CreateAtomicRMW(Op, Ptr.getPointer(), LoadVal1, Order); 614fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall RMWI->setVolatile(E->isVolatile()); 615fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall 616fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall // For __atomic_*_fetch operations, perform the operation again to 617fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall // determine the value which was written. 618fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall llvm::Value *Result = RMWI; 619fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall if (PostOp) 620fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall Result = CGF.Builder.CreateBinOp(PostOp, RMWI, LoadVal1); 621fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall if (E->getOp() == AtomicExpr::AO__atomic_nand_fetch) 622fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall Result = CGF.Builder.CreateNot(Result); 62387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar CGF.Builder.CreateStore(Result, Dest); 624fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall} 625fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall 626fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall// This function emits any expression (scalar, complex, or aggregate) 627fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall// into a temporary alloca. 62887d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainarstatic Address 629fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCallEmitValToTemp(CodeGenFunction &CGF, Expr *E) { 63087d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Address DeclPtr = CGF.CreateMemTemp(E->getType(), ".atomictmp"); 631fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall CGF.EmitAnyExprToMem(E, DeclPtr, E->getType().getQualifiers(), 632fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall /*Init*/ true); 633fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall return DeclPtr; 634fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall} 635fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall 636e469249726840b8baa2df008ca891e213cc5c661Ed Schoutenstatic void 637e469249726840b8baa2df008ca891e213cc5c661Ed SchoutenAddDirectArgument(CodeGenFunction &CGF, CallArgList &Args, 6384ee7dc2369c1f0257a73b2e83a7d38fdebdd9176Nick Lewycky bool UseOptimizedLibcall, llvm::Value *Val, QualType ValTy, 639176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines SourceLocation Loc, CharUnits SizeInChars) { 640e469249726840b8baa2df008ca891e213cc5c661Ed Schouten if (UseOptimizedLibcall) { 641e469249726840b8baa2df008ca891e213cc5c661Ed Schouten // Load value and pass it to the function directly. 64287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar CharUnits Align = CGF.getContext().getTypeAlignInChars(ValTy); 643176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines int64_t SizeInBits = CGF.getContext().toBits(SizeInChars); 644176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines ValTy = 645176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines CGF.getContext().getIntTypeForBitwidth(SizeInBits, /*Signed=*/false); 646176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines llvm::Type *IPtrTy = llvm::IntegerType::get(CGF.getLLVMContext(), 647176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines SizeInBits)->getPointerTo(); 64887d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Address Ptr = Address(CGF.Builder.CreateBitCast(Val, IPtrTy), Align); 64987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Val = CGF.EmitLoadOfScalar(Ptr, false, 65087d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar CGF.getContext().getPointerType(ValTy), 651176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines Loc); 652176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines // Coerce the value into an appropriately sized integer type. 653e469249726840b8baa2df008ca891e213cc5c661Ed Schouten Args.add(RValue::get(Val), ValTy); 654e469249726840b8baa2df008ca891e213cc5c661Ed Schouten } else { 655e469249726840b8baa2df008ca891e213cc5c661Ed Schouten // Non-optimized functions always take a reference. 656e469249726840b8baa2df008ca891e213cc5c661Ed Schouten Args.add(RValue::get(CGF.EmitCastToVoidPtr(Val)), 657e469249726840b8baa2df008ca891e213cc5c661Ed Schouten CGF.getContext().VoidPtrTy); 658e469249726840b8baa2df008ca891e213cc5c661Ed Schouten } 659e469249726840b8baa2df008ca891e213cc5c661Ed Schouten} 660e469249726840b8baa2df008ca891e213cc5c661Ed Schouten 66187d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga NainarRValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) { 662fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall QualType AtomicTy = E->getPtr()->getType()->getPointeeType(); 663fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall QualType MemTy = AtomicTy; 664fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall if (const AtomicType *AT = AtomicTy->getAs<AtomicType>()) 665fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall MemTy = AT->getValueType(); 66687d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar CharUnits sizeChars, alignChars; 66787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar std::tie(sizeChars, alignChars) = getContext().getTypeInfoInChars(AtomicTy); 668fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall uint64_t Size = sizeChars.getQuantity(); 66987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar unsigned MaxInlineWidthInBits = getTarget().getMaxAtomicInlineWidth(); 67087d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar bool UseLibcall = (sizeChars != alignChars || 671fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall getContext().toBits(sizeChars) > MaxInlineWidthInBits); 672fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall 67387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar llvm::Value *IsWeak = nullptr, *OrderFail = nullptr; 67487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar 67587d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Address Val1 = Address::invalid(); 67687d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Address Val2 = Address::invalid(); 67787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Address Dest = Address::invalid(); 67887d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Address Ptr(EmitScalarExpr(E->getPtr()), alignChars); 679fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall 680fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall if (E->getOp() == AtomicExpr::AO__c11_atomic_init) { 68187d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar LValue lvalue = MakeAddrLValue(Ptr, AtomicTy); 6829eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall EmitAtomicInit(E->getVal1(), lvalue); 6836bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return RValue::get(nullptr); 684fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall } 685fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall 6866bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines llvm::Value *Order = EmitScalarExpr(E->getOrder()); 687fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall 688fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall switch (E->getOp()) { 689fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__c11_atomic_init: 69087d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar llvm_unreachable("Already handled above with EmitAtomicInit!"); 691fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall 692fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__c11_atomic_load: 693fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__atomic_load_n: 694fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall break; 695fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall 696fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__atomic_load: 69787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Dest = EmitPointerWithAlignment(E->getVal1()); 698fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall break; 699fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall 700fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__atomic_store: 70187d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Val1 = EmitPointerWithAlignment(E->getVal1()); 702fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall break; 703fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall 704fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__atomic_exchange: 70587d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Val1 = EmitPointerWithAlignment(E->getVal1()); 70687d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Dest = EmitPointerWithAlignment(E->getVal2()); 707fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall break; 708fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall 709fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__c11_atomic_compare_exchange_strong: 710fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__c11_atomic_compare_exchange_weak: 711fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__atomic_compare_exchange_n: 712fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__atomic_compare_exchange: 71387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Val1 = EmitPointerWithAlignment(E->getVal1()); 714fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall if (E->getOp() == AtomicExpr::AO__atomic_compare_exchange) 71587d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Val2 = EmitPointerWithAlignment(E->getVal2()); 716fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall else 717fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall Val2 = EmitValToTemp(*this, E->getVal2()); 718fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall OrderFail = EmitScalarExpr(E->getOrderFail()); 719fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall if (E->getNumSubExprs() == 6) 720c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines IsWeak = EmitScalarExpr(E->getWeak()); 721fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall break; 722fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall 723fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__c11_atomic_fetch_add: 724fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__c11_atomic_fetch_sub: 725fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall if (MemTy->isPointerType()) { 726fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall // For pointer arithmetic, we're required to do a bit of math: 727fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall // adding 1 to an int* is not the same as adding 1 to a uintptr_t. 728fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall // ... but only for the C11 builtins. The GNU builtins expect the 729fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall // user to multiply by sizeof(T). 730fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall QualType Val1Ty = E->getVal1()->getType(); 731fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall llvm::Value *Val1Scalar = EmitScalarExpr(E->getVal1()); 732fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall CharUnits PointeeIncAmt = 733fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall getContext().getTypeSizeInChars(MemTy->getPointeeType()); 734fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall Val1Scalar = Builder.CreateMul(Val1Scalar, CGM.getSize(PointeeIncAmt)); 73587d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar auto Temp = CreateMemTemp(Val1Ty, ".atomictmp"); 73687d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Val1 = Temp; 73787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar EmitStoreOfScalar(Val1Scalar, MakeAddrLValue(Temp, Val1Ty)); 738fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall break; 739fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall } 740fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall // Fall through. 741fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__atomic_fetch_add: 742fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__atomic_fetch_sub: 743fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__atomic_add_fetch: 744fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__atomic_sub_fetch: 745fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__c11_atomic_store: 746fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__c11_atomic_exchange: 747fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__atomic_store_n: 748fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__atomic_exchange_n: 749fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__c11_atomic_fetch_and: 750fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__c11_atomic_fetch_or: 751fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__c11_atomic_fetch_xor: 752fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__atomic_fetch_and: 753fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__atomic_fetch_or: 754fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__atomic_fetch_xor: 755fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__atomic_fetch_nand: 756fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__atomic_and_fetch: 757fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__atomic_or_fetch: 758fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__atomic_xor_fetch: 759fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__atomic_nand_fetch: 760fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall Val1 = EmitValToTemp(*this, E->getVal1()); 761fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall break; 762fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall } 763fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall 7640e2c34f92f00628d48968dfea096d36381f494cbStephen Hines QualType RValTy = E->getType().getUnqualifiedType(); 7650e2c34f92f00628d48968dfea096d36381f494cbStephen Hines 76687d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar // The inlined atomics only function on iN types, where N is a power of 2. We 76787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar // need to make sure (via temporaries if necessary) that all incoming values 76887d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar // are compatible. 76987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar LValue AtomicVal = MakeAddrLValue(Ptr, AtomicTy); 77087d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar AtomicInfo Atomics(*this, AtomicVal); 77187d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar 77287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Ptr = Atomics.emitCastToAtomicIntPointer(Ptr); 77387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar if (Val1.isValid()) Val1 = Atomics.convertToAtomicIntPointer(Val1); 77487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar if (Val2.isValid()) Val2 = Atomics.convertToAtomicIntPointer(Val2); 77587d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar if (Dest.isValid()) 77687d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Dest = Atomics.emitCastToAtomicIntPointer(Dest); 77787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar else if (E->isCmpXChg()) 77887d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Dest = CreateMemTemp(RValTy, "cmpxchg.bool"); 77987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar else if (!RValTy->isVoidType()) 78087d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Dest = Atomics.emitCastToAtomicIntPointer(Atomics.CreateTempAlloca()); 781fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall 782fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall // Use a library call. See: http://gcc.gnu.org/wiki/Atomic/GCCMM/LIbrary . 783fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall if (UseLibcall) { 784e469249726840b8baa2df008ca891e213cc5c661Ed Schouten bool UseOptimizedLibcall = false; 785e469249726840b8baa2df008ca891e213cc5c661Ed Schouten switch (E->getOp()) { 78687d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case AtomicExpr::AO__c11_atomic_init: 78787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar llvm_unreachable("Already handled above with EmitAtomicInit!"); 78887d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar 789e469249726840b8baa2df008ca891e213cc5c661Ed Schouten case AtomicExpr::AO__c11_atomic_fetch_add: 790e469249726840b8baa2df008ca891e213cc5c661Ed Schouten case AtomicExpr::AO__atomic_fetch_add: 791e469249726840b8baa2df008ca891e213cc5c661Ed Schouten case AtomicExpr::AO__c11_atomic_fetch_and: 792e469249726840b8baa2df008ca891e213cc5c661Ed Schouten case AtomicExpr::AO__atomic_fetch_and: 793e469249726840b8baa2df008ca891e213cc5c661Ed Schouten case AtomicExpr::AO__c11_atomic_fetch_or: 794e469249726840b8baa2df008ca891e213cc5c661Ed Schouten case AtomicExpr::AO__atomic_fetch_or: 79587d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case AtomicExpr::AO__atomic_fetch_nand: 796e469249726840b8baa2df008ca891e213cc5c661Ed Schouten case AtomicExpr::AO__c11_atomic_fetch_sub: 797e469249726840b8baa2df008ca891e213cc5c661Ed Schouten case AtomicExpr::AO__atomic_fetch_sub: 798e469249726840b8baa2df008ca891e213cc5c661Ed Schouten case AtomicExpr::AO__c11_atomic_fetch_xor: 799e469249726840b8baa2df008ca891e213cc5c661Ed Schouten case AtomicExpr::AO__atomic_fetch_xor: 80087d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case AtomicExpr::AO__atomic_add_fetch: 80187d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case AtomicExpr::AO__atomic_and_fetch: 80287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case AtomicExpr::AO__atomic_nand_fetch: 80387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case AtomicExpr::AO__atomic_or_fetch: 80487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case AtomicExpr::AO__atomic_sub_fetch: 80587d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case AtomicExpr::AO__atomic_xor_fetch: 806e469249726840b8baa2df008ca891e213cc5c661Ed Schouten // For these, only library calls for certain sizes exist. 807e469249726840b8baa2df008ca891e213cc5c661Ed Schouten UseOptimizedLibcall = true; 808e469249726840b8baa2df008ca891e213cc5c661Ed Schouten break; 80987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar 81087d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case AtomicExpr::AO__c11_atomic_load: 81187d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case AtomicExpr::AO__c11_atomic_store: 81287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case AtomicExpr::AO__c11_atomic_exchange: 81387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case AtomicExpr::AO__c11_atomic_compare_exchange_weak: 81487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case AtomicExpr::AO__c11_atomic_compare_exchange_strong: 81587d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case AtomicExpr::AO__atomic_load_n: 81687d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case AtomicExpr::AO__atomic_load: 81787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case AtomicExpr::AO__atomic_store_n: 81887d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case AtomicExpr::AO__atomic_store: 81987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case AtomicExpr::AO__atomic_exchange_n: 82087d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case AtomicExpr::AO__atomic_exchange: 82187d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case AtomicExpr::AO__atomic_compare_exchange_n: 82287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case AtomicExpr::AO__atomic_compare_exchange: 823e469249726840b8baa2df008ca891e213cc5c661Ed Schouten // Only use optimized library calls for sizes for which they exist. 824e469249726840b8baa2df008ca891e213cc5c661Ed Schouten if (Size == 1 || Size == 2 || Size == 4 || Size == 8) 825e469249726840b8baa2df008ca891e213cc5c661Ed Schouten UseOptimizedLibcall = true; 826e469249726840b8baa2df008ca891e213cc5c661Ed Schouten break; 827e469249726840b8baa2df008ca891e213cc5c661Ed Schouten } 828fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall 829fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall CallArgList Args; 830e469249726840b8baa2df008ca891e213cc5c661Ed Schouten if (!UseOptimizedLibcall) { 831e469249726840b8baa2df008ca891e213cc5c661Ed Schouten // For non-optimized library calls, the size is the first parameter 832e469249726840b8baa2df008ca891e213cc5c661Ed Schouten Args.add(RValue::get(llvm::ConstantInt::get(SizeTy, Size)), 833e469249726840b8baa2df008ca891e213cc5c661Ed Schouten getContext().getSizeType()); 834e469249726840b8baa2df008ca891e213cc5c661Ed Schouten } 835e469249726840b8baa2df008ca891e213cc5c661Ed Schouten // Atomic address is the first or second parameter 83687d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Args.add(RValue::get(EmitCastToVoidPtr(Ptr.getPointer())), 83787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar getContext().VoidPtrTy); 838fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall 839e469249726840b8baa2df008ca891e213cc5c661Ed Schouten std::string LibCallName; 840651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines QualType LoweredMemTy = 841651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines MemTy->isPointerType() ? getContext().getIntPtrType() : MemTy; 842e469249726840b8baa2df008ca891e213cc5c661Ed Schouten QualType RetTy; 843e469249726840b8baa2df008ca891e213cc5c661Ed Schouten bool HaveRetTy = false; 84487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar llvm::Instruction::BinaryOps PostOp = (llvm::Instruction::BinaryOps)0; 845fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall switch (E->getOp()) { 84687d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case AtomicExpr::AO__c11_atomic_init: 84787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar llvm_unreachable("Already handled!"); 84887d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar 849fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall // There is only one libcall for compare an exchange, because there is no 850fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall // optimisation benefit possible from a libcall version of a weak compare 851fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall // and exchange. 852e469249726840b8baa2df008ca891e213cc5c661Ed Schouten // bool __atomic_compare_exchange(size_t size, void *mem, void *expected, 853fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall // void *desired, int success, int failure) 854e469249726840b8baa2df008ca891e213cc5c661Ed Schouten // bool __atomic_compare_exchange_N(T *mem, T *expected, T desired, 855e469249726840b8baa2df008ca891e213cc5c661Ed Schouten // int success, int failure) 856fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__c11_atomic_compare_exchange_weak: 857fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__c11_atomic_compare_exchange_strong: 858fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__atomic_compare_exchange: 859fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__atomic_compare_exchange_n: 860fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall LibCallName = "__atomic_compare_exchange"; 861fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall RetTy = getContext().BoolTy; 862e469249726840b8baa2df008ca891e213cc5c661Ed Schouten HaveRetTy = true; 86387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Args.add(RValue::get(EmitCastToVoidPtr(Val1.getPointer())), 86487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar getContext().VoidPtrTy); 86587d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar AddDirectArgument(*this, Args, UseOptimizedLibcall, Val2.getPointer(), 86687d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar MemTy, E->getExprLoc(), sizeChars); 8675d4a7559df106959dd721744c8971547f0f09097Nick Lewycky Args.add(RValue::get(Order), getContext().IntTy); 868fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall Order = OrderFail; 869fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall break; 870fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall // void __atomic_exchange(size_t size, void *mem, void *val, void *return, 871fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall // int order) 872e469249726840b8baa2df008ca891e213cc5c661Ed Schouten // T __atomic_exchange_N(T *mem, T val, int order) 873fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__c11_atomic_exchange: 874fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__atomic_exchange_n: 875fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__atomic_exchange: 876fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall LibCallName = "__atomic_exchange"; 87787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(), 87887d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar MemTy, E->getExprLoc(), sizeChars); 879fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall break; 880fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall // void __atomic_store(size_t size, void *mem, void *val, int order) 881e469249726840b8baa2df008ca891e213cc5c661Ed Schouten // void __atomic_store_N(T *mem, T val, int order) 882fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__c11_atomic_store: 883fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__atomic_store: 884fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__atomic_store_n: 885fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall LibCallName = "__atomic_store"; 886e469249726840b8baa2df008ca891e213cc5c661Ed Schouten RetTy = getContext().VoidTy; 887e469249726840b8baa2df008ca891e213cc5c661Ed Schouten HaveRetTy = true; 88887d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(), 88987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar MemTy, E->getExprLoc(), sizeChars); 890fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall break; 891fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall // void __atomic_load(size_t size, void *mem, void *return, int order) 892e469249726840b8baa2df008ca891e213cc5c661Ed Schouten // T __atomic_load_N(T *mem, int order) 893fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__c11_atomic_load: 894fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__atomic_load: 895fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall case AtomicExpr::AO__atomic_load_n: 896fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall LibCallName = "__atomic_load"; 897e469249726840b8baa2df008ca891e213cc5c661Ed Schouten break; 89887d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar // T __atomic_add_fetch_N(T *mem, T val, int order) 899e469249726840b8baa2df008ca891e213cc5c661Ed Schouten // T __atomic_fetch_add_N(T *mem, T val, int order) 90087d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case AtomicExpr::AO__atomic_add_fetch: 90187d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar PostOp = llvm::Instruction::Add; 90287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar // Fall through. 903e469249726840b8baa2df008ca891e213cc5c661Ed Schouten case AtomicExpr::AO__c11_atomic_fetch_add: 904e469249726840b8baa2df008ca891e213cc5c661Ed Schouten case AtomicExpr::AO__atomic_fetch_add: 905e469249726840b8baa2df008ca891e213cc5c661Ed Schouten LibCallName = "__atomic_fetch_add"; 90687d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(), 90787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar LoweredMemTy, E->getExprLoc(), sizeChars); 908e469249726840b8baa2df008ca891e213cc5c661Ed Schouten break; 90987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar // T __atomic_and_fetch_N(T *mem, T val, int order) 910e469249726840b8baa2df008ca891e213cc5c661Ed Schouten // T __atomic_fetch_and_N(T *mem, T val, int order) 91187d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case AtomicExpr::AO__atomic_and_fetch: 91287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar PostOp = llvm::Instruction::And; 91387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar // Fall through. 914e469249726840b8baa2df008ca891e213cc5c661Ed Schouten case AtomicExpr::AO__c11_atomic_fetch_and: 915e469249726840b8baa2df008ca891e213cc5c661Ed Schouten case AtomicExpr::AO__atomic_fetch_and: 916e469249726840b8baa2df008ca891e213cc5c661Ed Schouten LibCallName = "__atomic_fetch_and"; 91787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(), 91887d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar MemTy, E->getExprLoc(), sizeChars); 919e469249726840b8baa2df008ca891e213cc5c661Ed Schouten break; 92087d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar // T __atomic_or_fetch_N(T *mem, T val, int order) 921e469249726840b8baa2df008ca891e213cc5c661Ed Schouten // T __atomic_fetch_or_N(T *mem, T val, int order) 92287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case AtomicExpr::AO__atomic_or_fetch: 92387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar PostOp = llvm::Instruction::Or; 92487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar // Fall through. 925e469249726840b8baa2df008ca891e213cc5c661Ed Schouten case AtomicExpr::AO__c11_atomic_fetch_or: 926e469249726840b8baa2df008ca891e213cc5c661Ed Schouten case AtomicExpr::AO__atomic_fetch_or: 927e469249726840b8baa2df008ca891e213cc5c661Ed Schouten LibCallName = "__atomic_fetch_or"; 92887d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(), 92987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar MemTy, E->getExprLoc(), sizeChars); 930e469249726840b8baa2df008ca891e213cc5c661Ed Schouten break; 93187d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar // T __atomic_sub_fetch_N(T *mem, T val, int order) 932e469249726840b8baa2df008ca891e213cc5c661Ed Schouten // T __atomic_fetch_sub_N(T *mem, T val, int order) 93387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case AtomicExpr::AO__atomic_sub_fetch: 93487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar PostOp = llvm::Instruction::Sub; 93587d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar // Fall through. 936e469249726840b8baa2df008ca891e213cc5c661Ed Schouten case AtomicExpr::AO__c11_atomic_fetch_sub: 937e469249726840b8baa2df008ca891e213cc5c661Ed Schouten case AtomicExpr::AO__atomic_fetch_sub: 938e469249726840b8baa2df008ca891e213cc5c661Ed Schouten LibCallName = "__atomic_fetch_sub"; 93987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(), 94087d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar LoweredMemTy, E->getExprLoc(), sizeChars); 941e469249726840b8baa2df008ca891e213cc5c661Ed Schouten break; 94287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar // T __atomic_xor_fetch_N(T *mem, T val, int order) 943e469249726840b8baa2df008ca891e213cc5c661Ed Schouten // T __atomic_fetch_xor_N(T *mem, T val, int order) 94487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case AtomicExpr::AO__atomic_xor_fetch: 94587d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar PostOp = llvm::Instruction::Xor; 94687d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar // Fall through. 947e469249726840b8baa2df008ca891e213cc5c661Ed Schouten case AtomicExpr::AO__c11_atomic_fetch_xor: 948e469249726840b8baa2df008ca891e213cc5c661Ed Schouten case AtomicExpr::AO__atomic_fetch_xor: 949e469249726840b8baa2df008ca891e213cc5c661Ed Schouten LibCallName = "__atomic_fetch_xor"; 95087d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(), 95187d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar MemTy, E->getExprLoc(), sizeChars); 95287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar break; 95387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar // T __atomic_nand_fetch_N(T *mem, T val, int order) 95487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar // T __atomic_fetch_nand_N(T *mem, T val, int order) 95587d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case AtomicExpr::AO__atomic_nand_fetch: 95687d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar PostOp = llvm::Instruction::And; // the NOT is special cased below 95787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar // Fall through. 95887d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar case AtomicExpr::AO__atomic_fetch_nand: 95987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar LibCallName = "__atomic_fetch_nand"; 96087d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(), 96187d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar MemTy, E->getExprLoc(), sizeChars); 962fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall break; 963fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall } 964e469249726840b8baa2df008ca891e213cc5c661Ed Schouten 965e469249726840b8baa2df008ca891e213cc5c661Ed Schouten // Optimized functions have the size in their name. 966e469249726840b8baa2df008ca891e213cc5c661Ed Schouten if (UseOptimizedLibcall) 967e469249726840b8baa2df008ca891e213cc5c661Ed Schouten LibCallName += "_" + llvm::utostr(Size); 968e469249726840b8baa2df008ca891e213cc5c661Ed Schouten // By default, assume we return a value of the atomic type. 969e469249726840b8baa2df008ca891e213cc5c661Ed Schouten if (!HaveRetTy) { 970e469249726840b8baa2df008ca891e213cc5c661Ed Schouten if (UseOptimizedLibcall) { 971e469249726840b8baa2df008ca891e213cc5c661Ed Schouten // Value is returned directly. 972176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines // The function returns an appropriately sized integer type. 973176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines RetTy = getContext().getIntTypeForBitwidth( 974176edba5311f6eff0cad2631449885ddf4fbc9eaStephen Hines getContext().toBits(sizeChars), /*Signed=*/false); 975e469249726840b8baa2df008ca891e213cc5c661Ed Schouten } else { 976e469249726840b8baa2df008ca891e213cc5c661Ed Schouten // Value is returned through parameter before the order. 977e469249726840b8baa2df008ca891e213cc5c661Ed Schouten RetTy = getContext().VoidTy; 97887d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Args.add(RValue::get(EmitCastToVoidPtr(Dest.getPointer())), 97987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar getContext().VoidPtrTy); 980e469249726840b8baa2df008ca891e213cc5c661Ed Schouten } 981e469249726840b8baa2df008ca891e213cc5c661Ed Schouten } 982fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall // order is always the last parameter 983fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall Args.add(RValue::get(Order), 984fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall getContext().IntTy); 985fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall 98687d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar // PostOp is only needed for the atomic_*_fetch operations, and 98787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar // thus is only needed for and implemented in the 98887d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar // UseOptimizedLibcall codepath. 98987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar assert(UseOptimizedLibcall || !PostOp); 99087d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar 9910e2c34f92f00628d48968dfea096d36381f494cbStephen Hines RValue Res = emitAtomicLibcall(*this, LibCallName, RetTy, Args); 9920e2c34f92f00628d48968dfea096d36381f494cbStephen Hines // The value is returned directly from the libcall. 99387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar if (E->isCmpXChg()) 9940e2c34f92f00628d48968dfea096d36381f494cbStephen Hines return Res; 99587d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar 99687d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar // The value is returned directly for optimized libcalls but the expr 99787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar // provided an out-param. 99887d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar if (UseOptimizedLibcall && Res.getScalarVal()) { 9990e2c34f92f00628d48968dfea096d36381f494cbStephen Hines llvm::Value *ResVal = Res.getScalarVal(); 100087d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar if (PostOp) { 100187d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar llvm::Value *LoadVal1 = Args[1].RV.getScalarVal(); 100287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar ResVal = Builder.CreateBinOp(PostOp, ResVal, LoadVal1); 100387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar } 100487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar if (E->getOp() == AtomicExpr::AO__atomic_nand_fetch) 100587d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar ResVal = Builder.CreateNot(ResVal); 100687d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar 100787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Builder.CreateStore( 10080e2c34f92f00628d48968dfea096d36381f494cbStephen Hines ResVal, 100987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Builder.CreateBitCast(Dest, ResVal->getType()->getPointerTo())); 10100e2c34f92f00628d48968dfea096d36381f494cbStephen Hines } 101187d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar 101287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar if (RValTy->isVoidType()) 101387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return RValue::get(nullptr); 101487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar 101587d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return convertTempToRValue( 101687d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Builder.CreateBitCast(Dest, ConvertTypeForMem(RValTy)->getPointerTo()), 101787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar RValTy, E->getExprLoc()); 1018fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall } 1019fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall 1020fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall bool IsStore = E->getOp() == AtomicExpr::AO__c11_atomic_store || 1021fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall E->getOp() == AtomicExpr::AO__atomic_store || 1022fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall E->getOp() == AtomicExpr::AO__atomic_store_n; 1023fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall bool IsLoad = E->getOp() == AtomicExpr::AO__c11_atomic_load || 1024fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall E->getOp() == AtomicExpr::AO__atomic_load || 1025fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall E->getOp() == AtomicExpr::AO__atomic_load_n; 1026fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall 1027fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall if (isa<llvm::ConstantInt>(Order)) { 10284967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar auto ord = cast<llvm::ConstantInt>(Order)->getZExtValue(); 10294967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar // We should not ever get to a case where the ordering isn't a valid C ABI 10304967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar // value, but it's hard to enforce that in general. 10314967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar if (llvm::isValidAtomicOrderingCABI(ord)) 10324967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar switch ((llvm::AtomicOrderingCABI)ord) { 10334967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case llvm::AtomicOrderingCABI::relaxed: 10344967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size, 10354967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar llvm::AtomicOrdering::Monotonic); 10364967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar break; 10374967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case llvm::AtomicOrderingCABI::consume: 10384967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case llvm::AtomicOrderingCABI::acquire: 10394967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar if (IsStore) 10404967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar break; // Avoid crashing on code with undefined behavior 10414967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size, 10424967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar llvm::AtomicOrdering::Acquire); 10434967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar break; 10444967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case llvm::AtomicOrderingCABI::release: 10454967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar if (IsLoad) 10464967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar break; // Avoid crashing on code with undefined behavior 10474967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size, 10484967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar llvm::AtomicOrdering::Release); 10494967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar break; 10504967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case llvm::AtomicOrderingCABI::acq_rel: 10514967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar if (IsLoad || IsStore) 10524967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar break; // Avoid crashing on code with undefined behavior 10534967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size, 10544967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar llvm::AtomicOrdering::AcquireRelease); 10554967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar break; 10564967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar case llvm::AtomicOrderingCABI::seq_cst: 10574967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size, 10584967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar llvm::AtomicOrdering::SequentiallyConsistent); 10594967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar break; 10604967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar } 10610e2c34f92f00628d48968dfea096d36381f494cbStephen Hines if (RValTy->isVoidType()) 10626bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return RValue::get(nullptr); 106387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar 106487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return convertTempToRValue( 106587d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Builder.CreateBitCast(Dest, ConvertTypeForMem(RValTy)->getPointerTo()), 106687d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar RValTy, E->getExprLoc()); 1067fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall } 1068fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall 1069fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall // Long case, when Order isn't obviously constant. 1070fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall 1071fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall // Create all the relevant BB's 10726bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines llvm::BasicBlock *MonotonicBB = nullptr, *AcquireBB = nullptr, 10736bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines *ReleaseBB = nullptr, *AcqRelBB = nullptr, 10746bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines *SeqCstBB = nullptr; 1075fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall MonotonicBB = createBasicBlock("monotonic", CurFn); 1076fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall if (!IsStore) 1077fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall AcquireBB = createBasicBlock("acquire", CurFn); 1078fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall if (!IsLoad) 1079fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall ReleaseBB = createBasicBlock("release", CurFn); 1080fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall if (!IsLoad && !IsStore) 1081fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall AcqRelBB = createBasicBlock("acqrel", CurFn); 1082fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall SeqCstBB = createBasicBlock("seqcst", CurFn); 1083fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall llvm::BasicBlock *ContBB = createBasicBlock("atomic.continue", CurFn); 1084fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall 1085fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall // Create the switch for the split 1086fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall // MonotonicBB is arbitrarily chosen as the default case; in practice, this 1087fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall // doesn't matter unless someone is crazy enough to use something that 1088fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall // doesn't fold to a constant for the ordering. 1089fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall Order = Builder.CreateIntCast(Order, Builder.getInt32Ty(), false); 1090fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall llvm::SwitchInst *SI = Builder.CreateSwitch(Order, MonotonicBB); 1091fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall 1092fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall // Emit all the different atomics 1093fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall Builder.SetInsertPoint(MonotonicBB); 1094c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, 10954967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Size, llvm::AtomicOrdering::Monotonic); 1096fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall Builder.CreateBr(ContBB); 1097fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall if (!IsStore) { 1098fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall Builder.SetInsertPoint(AcquireBB); 1099c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, 11004967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Size, llvm::AtomicOrdering::Acquire); 1101fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall Builder.CreateBr(ContBB); 11024967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::consume), 1103651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines AcquireBB); 11044967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::acquire), 1105651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines AcquireBB); 1106fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall } 1107fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall if (!IsLoad) { 1108fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall Builder.SetInsertPoint(ReleaseBB); 1109c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, 11104967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Size, llvm::AtomicOrdering::Release); 1111fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall Builder.CreateBr(ContBB); 11124967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::release), 1113651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines ReleaseBB); 1114fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall } 1115fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall if (!IsLoad && !IsStore) { 1116fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall Builder.SetInsertPoint(AcqRelBB); 1117c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, 11184967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Size, llvm::AtomicOrdering::AcquireRelease); 1119fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall Builder.CreateBr(ContBB); 11204967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::acq_rel), 1121651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines AcqRelBB); 1122fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall } 1123fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall Builder.SetInsertPoint(SeqCstBB); 1124c568f1e98938584c0ef0b12ae5018ff7d90a4072Stephen Hines EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, 11254967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Size, llvm::AtomicOrdering::SequentiallyConsistent); 1126fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall Builder.CreateBr(ContBB); 11274967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::seq_cst), 1128651f13cea278ec967336033dd032faef0e9fc2ecStephen Hines SeqCstBB); 1129fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall 1130fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall // Cleanup and return 1131fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall Builder.SetInsertPoint(ContBB); 11320e2c34f92f00628d48968dfea096d36381f494cbStephen Hines if (RValTy->isVoidType()) 11336bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89Stephen Hines return RValue::get(nullptr); 113487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar 113587d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar assert(Atomics.getValueSizeInBits() <= Atomics.getAtomicSizeInBits()); 113687d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return convertTempToRValue( 113787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Builder.CreateBitCast(Dest, ConvertTypeForMem(RValTy)->getPointerTo()), 113887d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar RValTy, E->getExprLoc()); 1139fafaaef243322b1e598a72d7dbfaf2ca0c139174John McCall} 11409eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 114187d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga NainarAddress AtomicInfo::emitCastToAtomicIntPointer(Address addr) const { 11429eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall unsigned addrspace = 114387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar cast<llvm::PointerType>(addr.getPointer()->getType())->getAddressSpace(); 11449eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall llvm::IntegerType *ty = 11459eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall llvm::IntegerType::get(CGF.getLLVMContext(), AtomicSizeInBits); 11469eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall return CGF.Builder.CreateBitCast(addr, ty->getPointerTo(addrspace)); 11479eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall} 11489eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 114987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga NainarAddress AtomicInfo::convertToAtomicIntPointer(Address Addr) const { 115087d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar llvm::Type *Ty = Addr.getElementType(); 115187d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar uint64_t SourceSizeInBits = CGF.CGM.getDataLayout().getTypeSizeInBits(Ty); 115287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar if (SourceSizeInBits != AtomicSizeInBits) { 115387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Address Tmp = CreateTempAlloca(); 115487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar CGF.Builder.CreateMemCpy(Tmp, Addr, 115587d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar std::min(AtomicSizeInBits, SourceSizeInBits) / 8); 115687d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Addr = Tmp; 115787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar } 115887d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar 115987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return emitCastToAtomicIntPointer(Addr); 116087d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar} 116187d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar 116287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga NainarRValue AtomicInfo::convertAtomicTempToRValue(Address addr, 116387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar AggValueSlot resultSlot, 116487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar SourceLocation loc, 116587d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar bool asValue) const { 11660e2c34f92f00628d48968dfea096d36381f494cbStephen Hines if (LVal.isSimple()) { 11670e2c34f92f00628d48968dfea096d36381f494cbStephen Hines if (EvaluationKind == TEK_Aggregate) 11680e2c34f92f00628d48968dfea096d36381f494cbStephen Hines return resultSlot.asRValue(); 11690e2c34f92f00628d48968dfea096d36381f494cbStephen Hines 11700e2c34f92f00628d48968dfea096d36381f494cbStephen Hines // Drill into the padding structure if we have one. 11710e2c34f92f00628d48968dfea096d36381f494cbStephen Hines if (hasPadding()) 117287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar addr = CGF.Builder.CreateStructGEP(addr, 0, CharUnits()); 11730e2c34f92f00628d48968dfea096d36381f494cbStephen Hines 11740e2c34f92f00628d48968dfea096d36381f494cbStephen Hines // Otherwise, just convert the temporary to an r-value using the 11750e2c34f92f00628d48968dfea096d36381f494cbStephen Hines // normal conversion routine. 11760e2c34f92f00628d48968dfea096d36381f494cbStephen Hines return CGF.convertTempToRValue(addr, getValueType(), loc); 117758878f85ab89b13e9eea4af3ccf055e42c557bc8Pirama Arumuga Nainar } 117887d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar if (!asValue) 11793ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar // Get RValue from temp memory as atomic for non-simple lvalues 118087d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return RValue::get(CGF.Builder.CreateLoad(addr)); 118158878f85ab89b13e9eea4af3ccf055e42c557bc8Pirama Arumuga Nainar if (LVal.isBitField()) 118287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return CGF.EmitLoadOfBitfieldLValue( 118387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar LValue::MakeBitfield(addr, LVal.getBitFieldInfo(), LVal.getType(), 118487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar LVal.getAlignmentSource())); 118558878f85ab89b13e9eea4af3ccf055e42c557bc8Pirama Arumuga Nainar if (LVal.isVectorElt()) 118687d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return CGF.EmitLoadOfLValue( 118787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar LValue::MakeVectorElt(addr, LVal.getVectorIdx(), LVal.getType(), 118887d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar LVal.getAlignmentSource()), loc); 11890e2c34f92f00628d48968dfea096d36381f494cbStephen Hines assert(LVal.isExtVectorElt()); 11900e2c34f92f00628d48968dfea096d36381f494cbStephen Hines return CGF.EmitLoadOfExtVectorElementLValue(LValue::MakeExtVectorElt( 119187d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar addr, LVal.getExtVectorElts(), LVal.getType(), 119287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar LVal.getAlignmentSource())); 11930e2c34f92f00628d48968dfea096d36381f494cbStephen Hines} 11949eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 11953ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga NainarRValue AtomicInfo::ConvertIntToValueOrAtomic(llvm::Value *IntVal, 11963ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar AggValueSlot ResultSlot, 11973ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar SourceLocation Loc, 11983ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar bool AsValue) const { 11990e2c34f92f00628d48968dfea096d36381f494cbStephen Hines // Try not to in some easy cases. 12000e2c34f92f00628d48968dfea096d36381f494cbStephen Hines assert(IntVal->getType()->isIntegerTy() && "Expected integer value"); 12013ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar if (getEvaluationKind() == TEK_Scalar && 12023ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar (((!LVal.isBitField() || 12033ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar LVal.getBitFieldInfo().Size == ValueSizeInBits) && 12043ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar !hasPadding()) || 12053ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar !AsValue)) { 12063ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar auto *ValTy = AsValue 12073ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar ? CGF.ConvertTypeForMem(ValueTy) 120887d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar : getAtomicAddress().getType()->getPointerElementType(); 12090e2c34f92f00628d48968dfea096d36381f494cbStephen Hines if (ValTy->isIntegerTy()) { 12100e2c34f92f00628d48968dfea096d36381f494cbStephen Hines assert(IntVal->getType() == ValTy && "Different integer types."); 12110e2c34f92f00628d48968dfea096d36381f494cbStephen Hines return RValue::get(CGF.EmitFromMemory(IntVal, ValueTy)); 12120e2c34f92f00628d48968dfea096d36381f494cbStephen Hines } else if (ValTy->isPointerTy()) 12130e2c34f92f00628d48968dfea096d36381f494cbStephen Hines return RValue::get(CGF.Builder.CreateIntToPtr(IntVal, ValTy)); 12140e2c34f92f00628d48968dfea096d36381f494cbStephen Hines else if (llvm::CastInst::isBitCastable(IntVal->getType(), ValTy)) 12150e2c34f92f00628d48968dfea096d36381f494cbStephen Hines return RValue::get(CGF.Builder.CreateBitCast(IntVal, ValTy)); 12160e2c34f92f00628d48968dfea096d36381f494cbStephen Hines } 12170e2c34f92f00628d48968dfea096d36381f494cbStephen Hines 12180e2c34f92f00628d48968dfea096d36381f494cbStephen Hines // Create a temporary. This needs to be big enough to hold the 12190e2c34f92f00628d48968dfea096d36381f494cbStephen Hines // atomic integer. 122087d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Address Temp = Address::invalid(); 12210e2c34f92f00628d48968dfea096d36381f494cbStephen Hines bool TempIsVolatile = false; 12223ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar if (AsValue && getEvaluationKind() == TEK_Aggregate) { 12230e2c34f92f00628d48968dfea096d36381f494cbStephen Hines assert(!ResultSlot.isIgnored()); 122487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Temp = ResultSlot.getAddress(); 12250e2c34f92f00628d48968dfea096d36381f494cbStephen Hines TempIsVolatile = ResultSlot.isVolatile(); 12260e2c34f92f00628d48968dfea096d36381f494cbStephen Hines } else { 12273ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar Temp = CreateTempAlloca(); 12280e2c34f92f00628d48968dfea096d36381f494cbStephen Hines } 12299eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 12300e2c34f92f00628d48968dfea096d36381f494cbStephen Hines // Slam the integer into the temporary. 123187d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Address CastTemp = emitCastToAtomicIntPointer(Temp); 123287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar CGF.Builder.CreateStore(IntVal, CastTemp) 12330e2c34f92f00628d48968dfea096d36381f494cbStephen Hines ->setVolatile(TempIsVolatile); 12340e2c34f92f00628d48968dfea096d36381f494cbStephen Hines 123587d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return convertAtomicTempToRValue(Temp, ResultSlot, Loc, AsValue); 12363ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar} 12373ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar 12383ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainarvoid AtomicInfo::EmitAtomicLoadLibcall(llvm::Value *AddForLoaded, 12393ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar llvm::AtomicOrdering AO, bool) { 12403ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar // void __atomic_load(size_t size, void *mem, void *return, int order); 12413ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar CallArgList Args; 12423ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar Args.add(RValue::get(getAtomicSizeValue()), CGF.getContext().getSizeType()); 124387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Args.add(RValue::get(CGF.EmitCastToVoidPtr(getAtomicPointer())), 12443ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar CGF.getContext().VoidPtrTy); 12453ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar Args.add(RValue::get(CGF.EmitCastToVoidPtr(AddForLoaded)), 12463ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar CGF.getContext().VoidPtrTy); 12474967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Args.add( 12484967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar RValue::get(llvm::ConstantInt::get(CGF.IntTy, (int)llvm::toCABI(AO))), 12494967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar CGF.getContext().IntTy); 12503ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar emitAtomicLibcall(CGF, "__atomic_load", CGF.getContext().VoidTy, Args); 12513ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar} 12523ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar 12533ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainarllvm::Value *AtomicInfo::EmitAtomicLoadOp(llvm::AtomicOrdering AO, 12543ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar bool IsVolatile) { 12553ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar // Okay, we're doing this natively. 125687d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Address Addr = getAtomicAddressAsAtomicIntPointer(); 12573ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar llvm::LoadInst *Load = CGF.Builder.CreateLoad(Addr, "atomic-load"); 12583ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar Load->setAtomic(AO); 12593ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar 12603ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar // Other decoration. 12613ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar if (IsVolatile) 12623ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar Load->setVolatile(true); 12633ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar if (LVal.getTBAAInfo()) 126487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar CGF.CGM.DecorateInstructionWithTBAA(Load, LVal.getTBAAInfo()); 12653ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar return Load; 12660e2c34f92f00628d48968dfea096d36381f494cbStephen Hines} 12670e2c34f92f00628d48968dfea096d36381f494cbStephen Hines 12680e2c34f92f00628d48968dfea096d36381f494cbStephen Hines/// An LValue is a candidate for having its loads and stores be made atomic if 12690e2c34f92f00628d48968dfea096d36381f494cbStephen Hines/// we are operating under /volatile:ms *and* the LValue itself is volatile and 12700e2c34f92f00628d48968dfea096d36381f494cbStephen Hines/// performing such an operation can be performed without a libcall. 12710e2c34f92f00628d48968dfea096d36381f494cbStephen Hinesbool CodeGenFunction::LValueIsSuitableForInlineAtomic(LValue LV) { 127287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar if (!CGM.getCodeGenOpts().MSVolatile) return false; 12730e2c34f92f00628d48968dfea096d36381f494cbStephen Hines AtomicInfo AI(*this, LV); 12740e2c34f92f00628d48968dfea096d36381f494cbStephen Hines bool IsVolatile = LV.isVolatile() || hasVolatileMember(LV.getType()); 12750e2c34f92f00628d48968dfea096d36381f494cbStephen Hines // An atomic is inline if we don't need to use a libcall. 12760e2c34f92f00628d48968dfea096d36381f494cbStephen Hines bool AtomicIsInline = !AI.shouldUseLibcall(); 12774967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar // MSVC doesn't seem to do this for types wider than a pointer. 12784967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar if (getContext().getTypeSize(LV.getType()) > 12794967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar getContext().getTypeSize(getContext().getIntPtrType())) 12804967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar return false; 128187d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return IsVolatile && AtomicIsInline; 12820e2c34f92f00628d48968dfea096d36381f494cbStephen Hines} 12830e2c34f92f00628d48968dfea096d36381f494cbStephen Hines 12840e2c34f92f00628d48968dfea096d36381f494cbStephen HinesRValue CodeGenFunction::EmitAtomicLoad(LValue LV, SourceLocation SL, 12850e2c34f92f00628d48968dfea096d36381f494cbStephen Hines AggValueSlot Slot) { 12860e2c34f92f00628d48968dfea096d36381f494cbStephen Hines llvm::AtomicOrdering AO; 12870e2c34f92f00628d48968dfea096d36381f494cbStephen Hines bool IsVolatile = LV.isVolatileQualified(); 12880e2c34f92f00628d48968dfea096d36381f494cbStephen Hines if (LV.getType()->isAtomicType()) { 12894967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar AO = llvm::AtomicOrdering::SequentiallyConsistent; 12900e2c34f92f00628d48968dfea096d36381f494cbStephen Hines } else { 12914967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar AO = llvm::AtomicOrdering::Acquire; 12920e2c34f92f00628d48968dfea096d36381f494cbStephen Hines IsVolatile = true; 12930e2c34f92f00628d48968dfea096d36381f494cbStephen Hines } 12940e2c34f92f00628d48968dfea096d36381f494cbStephen Hines return EmitAtomicLoad(LV, SL, AO, IsVolatile, Slot); 12959eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall} 12969eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 12973ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga NainarRValue AtomicInfo::EmitAtomicLoad(AggValueSlot ResultSlot, SourceLocation Loc, 12983ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar bool AsValue, llvm::AtomicOrdering AO, 12993ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar bool IsVolatile) { 13009eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall // Check whether we should use a library call. 13013ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar if (shouldUseLibcall()) { 130287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Address TempAddr = Address::invalid(); 13033ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar if (LVal.isSimple() && !ResultSlot.isIgnored()) { 13043ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar assert(getEvaluationKind() == TEK_Aggregate); 130587d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar TempAddr = ResultSlot.getAddress(); 13060e2c34f92f00628d48968dfea096d36381f494cbStephen Hines } else 13073ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar TempAddr = CreateTempAlloca(); 13089eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 130987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar EmitAtomicLoadLibcall(TempAddr.getPointer(), AO, IsVolatile); 13103ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar 13113ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar // Okay, turn that back into the original value or whole atomic (for 13123ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar // non-simple lvalues) type. 131387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return convertAtomicTempToRValue(TempAddr, ResultSlot, Loc, AsValue); 13149eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall } 13159eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 13169eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall // Okay, we're doing this natively. 13173ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar auto *Load = EmitAtomicLoadOp(AO, IsVolatile); 13189eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 13199eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall // If we're ignoring an aggregate return, don't do anything. 13203ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar if (getEvaluationKind() == TEK_Aggregate && ResultSlot.isIgnored()) 132187d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return RValue::getAggregate(Address::invalid(), false); 13229eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 13233ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar // Okay, turn that back into the original value or atomic (for non-simple 13243ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar // lvalues) type. 13253ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar return ConvertIntToValueOrAtomic(Load, ResultSlot, Loc, AsValue); 13269eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall} 13279eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 13283ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar/// Emit a load from an l-value of atomic type. Note that the r-value 13293ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar/// we produce is an r-value of the atomic *value* type. 13303ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga NainarRValue CodeGenFunction::EmitAtomicLoad(LValue src, SourceLocation loc, 13313ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar llvm::AtomicOrdering AO, bool IsVolatile, 13323ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar AggValueSlot resultSlot) { 13333ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar AtomicInfo Atomics(*this, src); 13343ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar return Atomics.EmitAtomicLoad(resultSlot, loc, /*AsValue=*/true, AO, 13353ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar IsVolatile); 13363ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar} 13379eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 13389eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall/// Copy an r-value into memory as part of storing to an atomic type. 13399eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall/// This needs to create a bit-pattern suitable for atomic operations. 13400e2c34f92f00628d48968dfea096d36381f494cbStephen Hinesvoid AtomicInfo::emitCopyIntoMemory(RValue rvalue) const { 13410e2c34f92f00628d48968dfea096d36381f494cbStephen Hines assert(LVal.isSimple()); 13429eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall // If we have an r-value, the rvalue should be of the atomic type, 13439eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall // which means that the caller is responsible for having zeroed 13449eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall // any padding. Just do an aggregate copy of that type. 13459eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall if (rvalue.isAggregate()) { 13463ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar CGF.EmitAggregateCopy(getAtomicAddress(), 134787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar rvalue.getAggregateAddress(), 13489eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall getAtomicType(), 13499eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall (rvalue.isVolatileQualified() 135087d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar || LVal.isVolatileQualified())); 13519eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall return; 13529eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall } 13539eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 13549eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall // Okay, otherwise we're copying stuff. 13559eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 13569eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall // Zero out the buffer if necessary. 13570e2c34f92f00628d48968dfea096d36381f494cbStephen Hines emitMemSetZeroIfNecessary(); 13589eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 13599eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall // Drill past the padding if present. 13600e2c34f92f00628d48968dfea096d36381f494cbStephen Hines LValue TempLVal = projectValue(); 13619eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 13629eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall // Okay, store the rvalue in. 13639eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall if (rvalue.isScalar()) { 13640e2c34f92f00628d48968dfea096d36381f494cbStephen Hines CGF.EmitStoreOfScalar(rvalue.getScalarVal(), TempLVal, /*init*/ true); 13659eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall } else { 13660e2c34f92f00628d48968dfea096d36381f494cbStephen Hines CGF.EmitStoreOfComplex(rvalue.getComplexVal(), TempLVal, /*init*/ true); 13679eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall } 13689eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall} 13699eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 13709eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 13719eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall/// Materialize an r-value into memory for the purposes of storing it 13729eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall/// to an atomic type. 137387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga NainarAddress AtomicInfo::materializeRValue(RValue rvalue) const { 13749eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall // Aggregate r-values are already in memory, and EmitAtomicStore 13759eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall // requires them to be values of the atomic type. 13769eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall if (rvalue.isAggregate()) 137787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return rvalue.getAggregateAddress(); 13789eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 13799eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall // Otherwise, make a temporary and materialize into it. 138087d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar LValue TempLV = CGF.MakeAddrLValue(CreateTempAlloca(), getAtomicType()); 13813ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar AtomicInfo Atomics(CGF, TempLV); 13820e2c34f92f00628d48968dfea096d36381f494cbStephen Hines Atomics.emitCopyIntoMemory(rvalue); 13833ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar return TempLV.getAddress(); 13849eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall} 13859eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 13860e2c34f92f00628d48968dfea096d36381f494cbStephen Hinesllvm::Value *AtomicInfo::convertRValueToInt(RValue RVal) const { 13870e2c34f92f00628d48968dfea096d36381f494cbStephen Hines // If we've got a scalar value of the right size, try to avoid going 13880e2c34f92f00628d48968dfea096d36381f494cbStephen Hines // through memory. 13893ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar if (RVal.isScalar() && (!hasPadding() || !LVal.isSimple())) { 13900e2c34f92f00628d48968dfea096d36381f494cbStephen Hines llvm::Value *Value = RVal.getScalarVal(); 13910e2c34f92f00628d48968dfea096d36381f494cbStephen Hines if (isa<llvm::IntegerType>(Value->getType())) 139258878f85ab89b13e9eea4af3ccf055e42c557bc8Pirama Arumuga Nainar return CGF.EmitToMemory(Value, ValueTy); 13930e2c34f92f00628d48968dfea096d36381f494cbStephen Hines else { 13943ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar llvm::IntegerType *InputIntTy = llvm::IntegerType::get( 13953ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar CGF.getLLVMContext(), 13963ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar LVal.isSimple() ? getValueSizeInBits() : getAtomicSizeInBits()); 13970e2c34f92f00628d48968dfea096d36381f494cbStephen Hines if (isa<llvm::PointerType>(Value->getType())) 13980e2c34f92f00628d48968dfea096d36381f494cbStephen Hines return CGF.Builder.CreatePtrToInt(Value, InputIntTy); 13990e2c34f92f00628d48968dfea096d36381f494cbStephen Hines else if (llvm::BitCastInst::isBitCastable(Value->getType(), InputIntTy)) 14000e2c34f92f00628d48968dfea096d36381f494cbStephen Hines return CGF.Builder.CreateBitCast(Value, InputIntTy); 14010e2c34f92f00628d48968dfea096d36381f494cbStephen Hines } 14020e2c34f92f00628d48968dfea096d36381f494cbStephen Hines } 14030e2c34f92f00628d48968dfea096d36381f494cbStephen Hines // Otherwise, we need to go through memory. 14040e2c34f92f00628d48968dfea096d36381f494cbStephen Hines // Put the r-value in memory. 140587d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Address Addr = materializeRValue(RVal); 14060e2c34f92f00628d48968dfea096d36381f494cbStephen Hines 14070e2c34f92f00628d48968dfea096d36381f494cbStephen Hines // Cast the temporary to the atomic int type and pull a value out. 14080e2c34f92f00628d48968dfea096d36381f494cbStephen Hines Addr = emitCastToAtomicIntPointer(Addr); 140987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar return CGF.Builder.CreateLoad(Addr); 14100e2c34f92f00628d48968dfea096d36381f494cbStephen Hines} 14110e2c34f92f00628d48968dfea096d36381f494cbStephen Hines 1412b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainarstd::pair<llvm::Value *, llvm::Value *> AtomicInfo::EmitAtomicCompareExchangeOp( 1413b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar llvm::Value *ExpectedVal, llvm::Value *DesiredVal, 1414b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar llvm::AtomicOrdering Success, llvm::AtomicOrdering Failure, bool IsWeak) { 14153ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar // Do the atomic store. 141687d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Address Addr = getAtomicAddressAsAtomicIntPointer(); 141787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar auto *Inst = CGF.Builder.CreateAtomicCmpXchg(Addr.getPointer(), 141887d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar ExpectedVal, DesiredVal, 141958878f85ab89b13e9eea4af3ccf055e42c557bc8Pirama Arumuga Nainar Success, Failure); 14203ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar // Other decoration. 14213ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar Inst->setVolatile(LVal.isVolatileQualified()); 14223ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar Inst->setWeak(IsWeak); 14233ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar 14243ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar // Okay, turn that back into the original value type. 14253ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar auto *PreviousVal = CGF.Builder.CreateExtractValue(Inst, /*Idxs=*/0); 14263ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar auto *SuccessFailureVal = CGF.Builder.CreateExtractValue(Inst, /*Idxs=*/1); 1427b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar return std::make_pair(PreviousVal, SuccessFailureVal); 14283ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar} 14293ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar 1430b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainarllvm::Value * 1431b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga NainarAtomicInfo::EmitAtomicCompareExchangeLibcall(llvm::Value *ExpectedAddr, 1432b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar llvm::Value *DesiredAddr, 14333ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar llvm::AtomicOrdering Success, 14343ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar llvm::AtomicOrdering Failure) { 14353ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar // bool __atomic_compare_exchange(size_t size, void *obj, void *expected, 14363ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar // void *desired, int success, int failure); 14373ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar CallArgList Args; 14383ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar Args.add(RValue::get(getAtomicSizeValue()), CGF.getContext().getSizeType()); 143987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Args.add(RValue::get(CGF.EmitCastToVoidPtr(getAtomicPointer())), 14403ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar CGF.getContext().VoidPtrTy); 14413ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar Args.add(RValue::get(CGF.EmitCastToVoidPtr(ExpectedAddr)), 14423ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar CGF.getContext().VoidPtrTy); 14433ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar Args.add(RValue::get(CGF.EmitCastToVoidPtr(DesiredAddr)), 14443ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar CGF.getContext().VoidPtrTy); 14454967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Args.add(RValue::get( 14464967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar llvm::ConstantInt::get(CGF.IntTy, (int)llvm::toCABI(Success))), 14473ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar CGF.getContext().IntTy); 14484967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar Args.add(RValue::get( 14494967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar llvm::ConstantInt::get(CGF.IntTy, (int)llvm::toCABI(Failure))), 14503ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar CGF.getContext().IntTy); 14513ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar auto SuccessFailureRVal = emitAtomicLibcall(CGF, "__atomic_compare_exchange", 14523ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar CGF.getContext().BoolTy, Args); 145358878f85ab89b13e9eea4af3ccf055e42c557bc8Pirama Arumuga Nainar 1454b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar return SuccessFailureRVal.getScalarVal(); 14553ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar} 14563ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar 145758878f85ab89b13e9eea4af3ccf055e42c557bc8Pirama Arumuga Nainarstd::pair<RValue, llvm::Value *> AtomicInfo::EmitAtomicCompareExchange( 14583ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar RValue Expected, RValue Desired, llvm::AtomicOrdering Success, 14593ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar llvm::AtomicOrdering Failure, bool IsWeak) { 14604967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar if (isStrongerThan(Failure, Success)) 14614967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar // Don't assert on undefined behavior "failure argument shall be no stronger 14624967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar // than the success argument". 14633ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(Success); 14643ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar 14653ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar // Check whether we should use a library call. 14663ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar if (shouldUseLibcall()) { 14673ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar // Produce a source address. 146887d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Address ExpectedAddr = materializeRValue(Expected); 146987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Address DesiredAddr = materializeRValue(Desired); 147087d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar auto *Res = EmitAtomicCompareExchangeLibcall(ExpectedAddr.getPointer(), 147187d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar DesiredAddr.getPointer(), 1472b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar Success, Failure); 1473b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar return std::make_pair( 147487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar convertAtomicTempToRValue(ExpectedAddr, AggValueSlot::ignored(), 147587d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar SourceLocation(), /*AsValue=*/false), 1476b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar Res); 14773ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar } 14783ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar 14793ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar // If we've got a scalar value of the right size, try to avoid going 14803ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar // through memory. 1481b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar auto *ExpectedVal = convertRValueToInt(Expected); 1482b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar auto *DesiredVal = convertRValueToInt(Desired); 1483b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar auto Res = EmitAtomicCompareExchangeOp(ExpectedVal, DesiredVal, Success, 1484b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar Failure, IsWeak); 1485b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar return std::make_pair( 1486b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar ConvertIntToValueOrAtomic(Res.first, AggValueSlot::ignored(), 1487b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar SourceLocation(), /*AsValue=*/false), 1488b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar Res.second); 1489b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar} 1490b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar 1491b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainarstatic void 1492b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga NainarEmitAtomicUpdateValue(CodeGenFunction &CGF, AtomicInfo &Atomics, RValue OldRVal, 1493b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar const llvm::function_ref<RValue(RValue)> &UpdateOp, 149487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Address DesiredAddr) { 1495b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar RValue UpRVal; 1496b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar LValue AtomicLVal = Atomics.getAtomicLValue(); 1497b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar LValue DesiredLVal; 1498b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar if (AtomicLVal.isSimple()) { 1499b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar UpRVal = OldRVal; 150087d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar DesiredLVal = CGF.MakeAddrLValue(DesiredAddr, AtomicLVal.getType()); 1501b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar } else { 1502b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar // Build new lvalue for temp address 150387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Address Ptr = Atomics.materializeRValue(OldRVal); 150487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar LValue UpdateLVal; 1505b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar if (AtomicLVal.isBitField()) { 1506b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar UpdateLVal = 1507b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar LValue::MakeBitfield(Ptr, AtomicLVal.getBitFieldInfo(), 150887d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar AtomicLVal.getType(), 150987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar AtomicLVal.getAlignmentSource()); 1510b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar DesiredLVal = 1511b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar LValue::MakeBitfield(DesiredAddr, AtomicLVal.getBitFieldInfo(), 151287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar AtomicLVal.getType(), 151387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar AtomicLVal.getAlignmentSource()); 1514b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar } else if (AtomicLVal.isVectorElt()) { 1515b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar UpdateLVal = LValue::MakeVectorElt(Ptr, AtomicLVal.getVectorIdx(), 1516b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar AtomicLVal.getType(), 151787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar AtomicLVal.getAlignmentSource()); 1518b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar DesiredLVal = LValue::MakeVectorElt( 1519b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar DesiredAddr, AtomicLVal.getVectorIdx(), AtomicLVal.getType(), 152087d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar AtomicLVal.getAlignmentSource()); 1521b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar } else { 1522b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar assert(AtomicLVal.isExtVectorElt()); 1523b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar UpdateLVal = LValue::MakeExtVectorElt(Ptr, AtomicLVal.getExtVectorElts(), 1524b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar AtomicLVal.getType(), 152587d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar AtomicLVal.getAlignmentSource()); 1526b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar DesiredLVal = LValue::MakeExtVectorElt( 1527b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar DesiredAddr, AtomicLVal.getExtVectorElts(), AtomicLVal.getType(), 152887d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar AtomicLVal.getAlignmentSource()); 1529b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar } 1530b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar UpdateLVal.setTBAAInfo(AtomicLVal.getTBAAInfo()); 1531b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar DesiredLVal.setTBAAInfo(AtomicLVal.getTBAAInfo()); 1532b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar UpRVal = CGF.EmitLoadOfLValue(UpdateLVal, SourceLocation()); 1533b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar } 1534b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar // Store new value in the corresponding memory area 1535b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar RValue NewRVal = UpdateOp(UpRVal); 1536b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar if (NewRVal.isScalar()) { 1537b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar CGF.EmitStoreThroughLValue(NewRVal, DesiredLVal); 1538b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar } else { 1539b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar assert(NewRVal.isComplex()); 1540b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar CGF.EmitStoreOfComplex(NewRVal.getComplexVal(), DesiredLVal, 1541b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar /*isInit=*/false); 1542b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar } 1543b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar} 1544b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar 1545b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainarvoid AtomicInfo::EmitAtomicUpdateLibcall( 1546b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar llvm::AtomicOrdering AO, const llvm::function_ref<RValue(RValue)> &UpdateOp, 1547b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar bool IsVolatile) { 1548b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO); 1549b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar 155087d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Address ExpectedAddr = CreateTempAlloca(); 1551b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar 155287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar EmitAtomicLoadLibcall(ExpectedAddr.getPointer(), AO, IsVolatile); 1553b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar auto *ContBB = CGF.createBasicBlock("atomic_cont"); 1554b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar auto *ExitBB = CGF.createBasicBlock("atomic_exit"); 1555b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar CGF.EmitBlock(ContBB); 155687d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Address DesiredAddr = CreateTempAlloca(); 1557b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) || 155887d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar requiresMemSetZero(getAtomicAddress().getElementType())) { 155987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar auto *OldVal = CGF.Builder.CreateLoad(ExpectedAddr); 156087d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar CGF.Builder.CreateStore(OldVal, DesiredAddr); 1561b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar } 156287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar auto OldRVal = convertAtomicTempToRValue(ExpectedAddr, 156387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar AggValueSlot::ignored(), 156487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar SourceLocation(), /*AsValue=*/false); 1565b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar EmitAtomicUpdateValue(CGF, *this, OldRVal, UpdateOp, DesiredAddr); 1566b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar auto *Res = 156787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar EmitAtomicCompareExchangeLibcall(ExpectedAddr.getPointer(), 156887d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar DesiredAddr.getPointer(), 156987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar AO, Failure); 1570b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar CGF.Builder.CreateCondBr(Res, ExitBB, ContBB); 1571b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar CGF.EmitBlock(ExitBB, /*IsFinished=*/true); 1572b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar} 1573b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar 1574b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainarvoid AtomicInfo::EmitAtomicUpdateOp( 1575b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar llvm::AtomicOrdering AO, const llvm::function_ref<RValue(RValue)> &UpdateOp, 1576b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar bool IsVolatile) { 1577b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO); 1578b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar 1579b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar // Do the atomic load. 1580b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar auto *OldVal = EmitAtomicLoadOp(AO, IsVolatile); 1581b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar // For non-simple lvalues perform compare-and-swap procedure. 1582b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar auto *ContBB = CGF.createBasicBlock("atomic_cont"); 1583b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar auto *ExitBB = CGF.createBasicBlock("atomic_exit"); 1584b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar auto *CurBB = CGF.Builder.GetInsertBlock(); 1585b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar CGF.EmitBlock(ContBB); 1586b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar llvm::PHINode *PHI = CGF.Builder.CreatePHI(OldVal->getType(), 1587b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar /*NumReservedValues=*/2); 1588b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar PHI->addIncoming(OldVal, CurBB); 158987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Address NewAtomicAddr = CreateTempAlloca(); 159087d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Address NewAtomicIntAddr = emitCastToAtomicIntPointer(NewAtomicAddr); 1591b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) || 159287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar requiresMemSetZero(getAtomicAddress().getElementType())) { 159387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar CGF.Builder.CreateStore(PHI, NewAtomicIntAddr); 1594b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar } 1595b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar auto OldRVal = ConvertIntToValueOrAtomic(PHI, AggValueSlot::ignored(), 1596b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar SourceLocation(), /*AsValue=*/false); 1597b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar EmitAtomicUpdateValue(CGF, *this, OldRVal, UpdateOp, NewAtomicAddr); 159887d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar auto *DesiredVal = CGF.Builder.CreateLoad(NewAtomicIntAddr); 1599b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar // Try to write new value using cmpxchg operation 1600b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar auto Res = EmitAtomicCompareExchangeOp(PHI, DesiredVal, AO, Failure); 1601b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar PHI->addIncoming(Res.first, CGF.Builder.GetInsertBlock()); 1602b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar CGF.Builder.CreateCondBr(Res.second, ExitBB, ContBB); 1603b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar CGF.EmitBlock(ExitBB, /*IsFinished=*/true); 1604b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar} 1605b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar 1606b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainarstatic void EmitAtomicUpdateValue(CodeGenFunction &CGF, AtomicInfo &Atomics, 160787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar RValue UpdateRVal, Address DesiredAddr) { 1608b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar LValue AtomicLVal = Atomics.getAtomicLValue(); 1609b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar LValue DesiredLVal; 1610b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar // Build new lvalue for temp address 1611b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar if (AtomicLVal.isBitField()) { 1612b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar DesiredLVal = 1613b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar LValue::MakeBitfield(DesiredAddr, AtomicLVal.getBitFieldInfo(), 161487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar AtomicLVal.getType(), 161587d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar AtomicLVal.getAlignmentSource()); 1616b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar } else if (AtomicLVal.isVectorElt()) { 1617b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar DesiredLVal = 1618b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar LValue::MakeVectorElt(DesiredAddr, AtomicLVal.getVectorIdx(), 161987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar AtomicLVal.getType(), 162087d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar AtomicLVal.getAlignmentSource()); 1621b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar } else { 1622b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar assert(AtomicLVal.isExtVectorElt()); 1623b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar DesiredLVal = LValue::MakeExtVectorElt( 1624b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar DesiredAddr, AtomicLVal.getExtVectorElts(), AtomicLVal.getType(), 162587d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar AtomicLVal.getAlignmentSource()); 1626b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar } 1627b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar DesiredLVal.setTBAAInfo(AtomicLVal.getTBAAInfo()); 1628b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar // Store new value in the corresponding memory area 1629b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar assert(UpdateRVal.isScalar()); 1630b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar CGF.EmitStoreThroughLValue(UpdateRVal, DesiredLVal); 1631b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar} 1632b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar 1633b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainarvoid AtomicInfo::EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO, 1634b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar RValue UpdateRVal, bool IsVolatile) { 1635b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO); 1636b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar 163787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Address ExpectedAddr = CreateTempAlloca(); 1638b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar 163987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar EmitAtomicLoadLibcall(ExpectedAddr.getPointer(), AO, IsVolatile); 1640b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar auto *ContBB = CGF.createBasicBlock("atomic_cont"); 1641b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar auto *ExitBB = CGF.createBasicBlock("atomic_exit"); 1642b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar CGF.EmitBlock(ContBB); 164387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Address DesiredAddr = CreateTempAlloca(); 1644b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) || 164587d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar requiresMemSetZero(getAtomicAddress().getElementType())) { 164687d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar auto *OldVal = CGF.Builder.CreateLoad(ExpectedAddr); 164787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar CGF.Builder.CreateStore(OldVal, DesiredAddr); 1648b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar } 1649b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar EmitAtomicUpdateValue(CGF, *this, UpdateRVal, DesiredAddr); 1650b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar auto *Res = 165187d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar EmitAtomicCompareExchangeLibcall(ExpectedAddr.getPointer(), 165287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar DesiredAddr.getPointer(), 165387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar AO, Failure); 1654b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar CGF.Builder.CreateCondBr(Res, ExitBB, ContBB); 1655b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar CGF.EmitBlock(ExitBB, /*IsFinished=*/true); 1656b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar} 1657b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar 1658b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainarvoid AtomicInfo::EmitAtomicUpdateOp(llvm::AtomicOrdering AO, RValue UpdateRVal, 1659b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar bool IsVolatile) { 1660b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO); 1661b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar 1662b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar // Do the atomic load. 1663b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar auto *OldVal = EmitAtomicLoadOp(AO, IsVolatile); 1664b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar // For non-simple lvalues perform compare-and-swap procedure. 1665b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar auto *ContBB = CGF.createBasicBlock("atomic_cont"); 1666b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar auto *ExitBB = CGF.createBasicBlock("atomic_exit"); 1667b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar auto *CurBB = CGF.Builder.GetInsertBlock(); 1668b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar CGF.EmitBlock(ContBB); 1669b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar llvm::PHINode *PHI = CGF.Builder.CreatePHI(OldVal->getType(), 1670b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar /*NumReservedValues=*/2); 1671b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar PHI->addIncoming(OldVal, CurBB); 167287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Address NewAtomicAddr = CreateTempAlloca(); 167387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Address NewAtomicIntAddr = emitCastToAtomicIntPointer(NewAtomicAddr); 1674b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) || 167587d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar requiresMemSetZero(getAtomicAddress().getElementType())) { 167687d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar CGF.Builder.CreateStore(PHI, NewAtomicIntAddr); 1677b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar } 1678b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar EmitAtomicUpdateValue(CGF, *this, UpdateRVal, NewAtomicAddr); 167987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar auto *DesiredVal = CGF.Builder.CreateLoad(NewAtomicIntAddr); 1680b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar // Try to write new value using cmpxchg operation 1681b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar auto Res = EmitAtomicCompareExchangeOp(PHI, DesiredVal, AO, Failure); 1682b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar PHI->addIncoming(Res.first, CGF.Builder.GetInsertBlock()); 1683b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar CGF.Builder.CreateCondBr(Res.second, ExitBB, ContBB); 1684b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar CGF.EmitBlock(ExitBB, /*IsFinished=*/true); 1685b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar} 1686b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar 1687b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainarvoid AtomicInfo::EmitAtomicUpdate( 1688b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar llvm::AtomicOrdering AO, const llvm::function_ref<RValue(RValue)> &UpdateOp, 1689b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar bool IsVolatile) { 1690b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar if (shouldUseLibcall()) { 1691b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar EmitAtomicUpdateLibcall(AO, UpdateOp, IsVolatile); 1692b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar } else { 1693b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar EmitAtomicUpdateOp(AO, UpdateOp, IsVolatile); 1694b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar } 1695b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar} 1696b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar 1697b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainarvoid AtomicInfo::EmitAtomicUpdate(llvm::AtomicOrdering AO, RValue UpdateRVal, 1698b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar bool IsVolatile) { 1699b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar if (shouldUseLibcall()) { 1700b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar EmitAtomicUpdateLibcall(AO, UpdateRVal, IsVolatile); 1701b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar } else { 1702b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar EmitAtomicUpdateOp(AO, UpdateRVal, IsVolatile); 1703b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar } 17043ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar} 17053ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar 17060e2c34f92f00628d48968dfea096d36381f494cbStephen Hinesvoid CodeGenFunction::EmitAtomicStore(RValue rvalue, LValue lvalue, 17070e2c34f92f00628d48968dfea096d36381f494cbStephen Hines bool isInit) { 17080e2c34f92f00628d48968dfea096d36381f494cbStephen Hines bool IsVolatile = lvalue.isVolatileQualified(); 17090e2c34f92f00628d48968dfea096d36381f494cbStephen Hines llvm::AtomicOrdering AO; 17100e2c34f92f00628d48968dfea096d36381f494cbStephen Hines if (lvalue.getType()->isAtomicType()) { 17114967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar AO = llvm::AtomicOrdering::SequentiallyConsistent; 17120e2c34f92f00628d48968dfea096d36381f494cbStephen Hines } else { 17134967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar AO = llvm::AtomicOrdering::Release; 17140e2c34f92f00628d48968dfea096d36381f494cbStephen Hines IsVolatile = true; 17150e2c34f92f00628d48968dfea096d36381f494cbStephen Hines } 17160e2c34f92f00628d48968dfea096d36381f494cbStephen Hines return EmitAtomicStore(rvalue, lvalue, AO, IsVolatile, isInit); 17170e2c34f92f00628d48968dfea096d36381f494cbStephen Hines} 17180e2c34f92f00628d48968dfea096d36381f494cbStephen Hines 17199eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall/// Emit a store to an l-value of atomic type. 17209eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall/// 17219eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall/// Note that the r-value is expected to be an r-value *of the atomic 17229eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall/// type*; this means that for aggregate r-values, it should include 17239eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall/// storage for any padding that was necessary. 17240e2c34f92f00628d48968dfea096d36381f494cbStephen Hinesvoid CodeGenFunction::EmitAtomicStore(RValue rvalue, LValue dest, 17250e2c34f92f00628d48968dfea096d36381f494cbStephen Hines llvm::AtomicOrdering AO, bool IsVolatile, 17260e2c34f92f00628d48968dfea096d36381f494cbStephen Hines bool isInit) { 17279eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall // If this is an aggregate r-value, it should agree in type except 17289eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall // maybe for address-space qualification. 17299eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall assert(!rvalue.isAggregate() || 173087d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar rvalue.getAggregateAddress().getElementType() 173187d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar == dest.getAddress().getElementType()); 17329eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 17339eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall AtomicInfo atomics(*this, dest); 17343ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar LValue LVal = atomics.getAtomicLValue(); 17359eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 17369eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall // If this is an initialization, just put the value there normally. 17373ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar if (LVal.isSimple()) { 17383ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar if (isInit) { 17393ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar atomics.emitCopyIntoMemory(rvalue); 17403ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar return; 17413ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar } 17429eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 17433ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar // Check whether we should use a library call. 17443ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar if (atomics.shouldUseLibcall()) { 17453ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar // Produce a source address. 174687d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Address srcAddr = atomics.materializeRValue(rvalue); 17479eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 17483ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar // void __atomic_store(size_t size, void *mem, void *val, int order) 17493ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar CallArgList args; 17503ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar args.add(RValue::get(atomics.getAtomicSizeValue()), 17513ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar getContext().getSizeType()); 175287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar args.add(RValue::get(EmitCastToVoidPtr(atomics.getAtomicPointer())), 175387d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar getContext().VoidPtrTy); 175487d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar args.add(RValue::get(EmitCastToVoidPtr(srcAddr.getPointer())), 17553ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar getContext().VoidPtrTy); 17564967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar args.add( 17574967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar RValue::get(llvm::ConstantInt::get(IntTy, (int)llvm::toCABI(AO))), 17584967a710c84587c654b56c828382219c3937dacbPirama Arumuga Nainar getContext().IntTy); 17593ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar emitAtomicLibcall(*this, "__atomic_store", getContext().VoidTy, args); 17603ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar return; 17613ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar } 17623ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar 17633ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar // Okay, we're doing this natively. 17643ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar llvm::Value *intValue = atomics.convertRValueToInt(rvalue); 17653ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar 17663ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar // Do the atomic store. 176787d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Address addr = 17683ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar atomics.emitCastToAtomicIntPointer(atomics.getAtomicAddress()); 17693ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar intValue = Builder.CreateIntCast( 177087d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar intValue, addr.getElementType(), /*isSigned=*/false); 17713ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar llvm::StoreInst *store = Builder.CreateStore(intValue, addr); 17723ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar 17733ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar // Initializations don't need to be atomic. 17743ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar if (!isInit) 17753ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar store->setAtomic(AO); 17763ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar 17773ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar // Other decoration. 17783ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar if (IsVolatile) 17793ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar store->setVolatile(true); 17803ea9e33ea25e0c2b12db56418ba3f994eb662c04Pirama Arumuga Nainar if (dest.getTBAAInfo()) 178187d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar CGM.DecorateInstructionWithTBAA(store, dest.getTBAAInfo()); 17829eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall return; 17839eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall } 17849eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 1785b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar // Emit simple atomic update operation. 1786b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar atomics.EmitAtomicUpdate(AO, rvalue, IsVolatile); 17879eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall} 17889eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 17890e2c34f92f00628d48968dfea096d36381f494cbStephen Hines/// Emit a compare-and-exchange op for atomic type. 17900e2c34f92f00628d48968dfea096d36381f494cbStephen Hines/// 179158878f85ab89b13e9eea4af3ccf055e42c557bc8Pirama Arumuga Nainarstd::pair<RValue, llvm::Value *> CodeGenFunction::EmitAtomicCompareExchange( 17920e2c34f92f00628d48968dfea096d36381f494cbStephen Hines LValue Obj, RValue Expected, RValue Desired, SourceLocation Loc, 17930e2c34f92f00628d48968dfea096d36381f494cbStephen Hines llvm::AtomicOrdering Success, llvm::AtomicOrdering Failure, bool IsWeak, 17940e2c34f92f00628d48968dfea096d36381f494cbStephen Hines AggValueSlot Slot) { 17950e2c34f92f00628d48968dfea096d36381f494cbStephen Hines // If this is an aggregate r-value, it should agree in type except 17960e2c34f92f00628d48968dfea096d36381f494cbStephen Hines // maybe for address-space qualification. 17970e2c34f92f00628d48968dfea096d36381f494cbStephen Hines assert(!Expected.isAggregate() || 179887d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Expected.getAggregateAddress().getElementType() == 179987d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Obj.getAddress().getElementType()); 18000e2c34f92f00628d48968dfea096d36381f494cbStephen Hines assert(!Desired.isAggregate() || 180187d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Desired.getAggregateAddress().getElementType() == 180287d948ecccffea9e9e37d0d053b246e2d6d6c47bPirama Arumuga Nainar Obj.getAddress().getElementType()); 18030e2c34f92f00628d48968dfea096d36381f494cbStephen Hines AtomicInfo Atomics(*this, Obj); 18040e2c34f92f00628d48968dfea096d36381f494cbStephen Hines 180558878f85ab89b13e9eea4af3ccf055e42c557bc8Pirama Arumuga Nainar return Atomics.EmitAtomicCompareExchange(Expected, Desired, Success, Failure, 180658878f85ab89b13e9eea4af3ccf055e42c557bc8Pirama Arumuga Nainar IsWeak); 180758878f85ab89b13e9eea4af3ccf055e42c557bc8Pirama Arumuga Nainar} 180858878f85ab89b13e9eea4af3ccf055e42c557bc8Pirama Arumuga Nainar 180958878f85ab89b13e9eea4af3ccf055e42c557bc8Pirama Arumuga Nainarvoid CodeGenFunction::EmitAtomicUpdate( 181058878f85ab89b13e9eea4af3ccf055e42c557bc8Pirama Arumuga Nainar LValue LVal, llvm::AtomicOrdering AO, 1811b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar const llvm::function_ref<RValue(RValue)> &UpdateOp, bool IsVolatile) { 181258878f85ab89b13e9eea4af3ccf055e42c557bc8Pirama Arumuga Nainar AtomicInfo Atomics(*this, LVal); 1813b6d6993e6e6d3daf4d9876794254d20a134e37c2Pirama Arumuga Nainar Atomics.EmitAtomicUpdate(AO, UpdateOp, IsVolatile); 18140e2c34f92f00628d48968dfea096d36381f494cbStephen Hines} 18150e2c34f92f00628d48968dfea096d36381f494cbStephen Hines 18169eda3abe7e183b05834947391c0cdc291f4ee0d8John McCallvoid CodeGenFunction::EmitAtomicInit(Expr *init, LValue dest) { 18179eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall AtomicInfo atomics(*this, dest); 18189eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 18199eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall switch (atomics.getEvaluationKind()) { 18209eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall case TEK_Scalar: { 18219eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall llvm::Value *value = EmitScalarExpr(init); 18220e2c34f92f00628d48968dfea096d36381f494cbStephen Hines atomics.emitCopyIntoMemory(RValue::get(value)); 18239eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall return; 18249eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall } 18259eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 18269eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall case TEK_Complex: { 18279eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall ComplexPairTy value = EmitComplexExpr(init); 18280e2c34f92f00628d48968dfea096d36381f494cbStephen Hines atomics.emitCopyIntoMemory(RValue::getComplex(value)); 18299eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall return; 18309eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall } 18319eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 18329eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall case TEK_Aggregate: { 1833336d9df5e628279425344d754dc68047fa5b00a7Eli Friedman // Fix up the destination if the initializer isn't an expression 1834336d9df5e628279425344d754dc68047fa5b00a7Eli Friedman // of atomic type. 1835336d9df5e628279425344d754dc68047fa5b00a7Eli Friedman bool Zeroed = false; 18369eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall if (!init->getType()->isAtomicType()) { 18370e2c34f92f00628d48968dfea096d36381f494cbStephen Hines Zeroed = atomics.emitMemSetZeroIfNecessary(); 18380e2c34f92f00628d48968dfea096d36381f494cbStephen Hines dest = atomics.projectValue(); 18399eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall } 18409eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall 18419eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall // Evaluate the expression directly into the destination. 18429eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall AggValueSlot slot = AggValueSlot::forLValue(dest, 18439eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall AggValueSlot::IsNotDestructed, 18449eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall AggValueSlot::DoesNotNeedGCBarriers, 1845336d9df5e628279425344d754dc68047fa5b00a7Eli Friedman AggValueSlot::IsNotAliased, 1846336d9df5e628279425344d754dc68047fa5b00a7Eli Friedman Zeroed ? AggValueSlot::IsZeroed : 1847336d9df5e628279425344d754dc68047fa5b00a7Eli Friedman AggValueSlot::IsNotZeroed); 1848336d9df5e628279425344d754dc68047fa5b00a7Eli Friedman 18499eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall EmitAggExpr(init, slot); 18509eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall return; 18519eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall } 18529eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall } 18539eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall llvm_unreachable("bad evaluation kind"); 18549eda3abe7e183b05834947391c0cdc291f4ee0d8John McCall} 1855