1//===--- CGAtomic.cpp - Emit LLVM IR for atomic operations ----------------===//
2//
3//                     The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10// This file contains the code for emitting atomic operations.
11//
12//===----------------------------------------------------------------------===//
13
14#include "CodeGenFunction.h"
15#include "CGCall.h"
16#include "CGRecordLayout.h"
17#include "CodeGenModule.h"
18#include "clang/AST/ASTContext.h"
19#include "clang/CodeGen/CGFunctionInfo.h"
20#include "llvm/ADT/StringExtras.h"
21#include "llvm/IR/DataLayout.h"
22#include "llvm/IR/Intrinsics.h"
23#include "llvm/IR/Operator.h"
24
25using namespace clang;
26using namespace CodeGen;
27
28namespace {
29  class AtomicInfo {
30    CodeGenFunction &CGF;
31    QualType AtomicTy;
32    QualType ValueTy;
33    uint64_t AtomicSizeInBits;
34    uint64_t ValueSizeInBits;
35    CharUnits AtomicAlign;
36    CharUnits ValueAlign;
37    CharUnits LValueAlign;
38    TypeEvaluationKind EvaluationKind;
39    bool UseLibcall;
40    LValue LVal;
41    CGBitFieldInfo BFI;
42  public:
43    AtomicInfo(CodeGenFunction &CGF, LValue &lvalue)
44        : CGF(CGF), AtomicSizeInBits(0), ValueSizeInBits(0),
45          EvaluationKind(TEK_Scalar), UseLibcall(true) {
46      assert(!lvalue.isGlobalReg());
47      ASTContext &C = CGF.getContext();
48      if (lvalue.isSimple()) {
49        AtomicTy = lvalue.getType();
50        if (auto *ATy = AtomicTy->getAs<AtomicType>())
51          ValueTy = ATy->getValueType();
52        else
53          ValueTy = AtomicTy;
54        EvaluationKind = CGF.getEvaluationKind(ValueTy);
55
56        uint64_t ValueAlignInBits;
57        uint64_t AtomicAlignInBits;
58        TypeInfo ValueTI = C.getTypeInfo(ValueTy);
59        ValueSizeInBits = ValueTI.Width;
60        ValueAlignInBits = ValueTI.Align;
61
62        TypeInfo AtomicTI = C.getTypeInfo(AtomicTy);
63        AtomicSizeInBits = AtomicTI.Width;
64        AtomicAlignInBits = AtomicTI.Align;
65
66        assert(ValueSizeInBits <= AtomicSizeInBits);
67        assert(ValueAlignInBits <= AtomicAlignInBits);
68
69        AtomicAlign = C.toCharUnitsFromBits(AtomicAlignInBits);
70        ValueAlign = C.toCharUnitsFromBits(ValueAlignInBits);
71        if (lvalue.getAlignment().isZero())
72          lvalue.setAlignment(AtomicAlign);
73
74        LVal = lvalue;
75      } else if (lvalue.isBitField()) {
76        ValueTy = lvalue.getType();
77        ValueSizeInBits = C.getTypeSize(ValueTy);
78        auto &OrigBFI = lvalue.getBitFieldInfo();
79        auto Offset = OrigBFI.Offset % C.toBits(lvalue.getAlignment());
80        AtomicSizeInBits = C.toBits(
81            C.toCharUnitsFromBits(Offset + OrigBFI.Size + C.getCharWidth() - 1)
82                .alignTo(lvalue.getAlignment()));
83        auto VoidPtrAddr = CGF.EmitCastToVoidPtr(lvalue.getBitFieldPointer());
84        auto OffsetInChars =
85            (C.toCharUnitsFromBits(OrigBFI.Offset) / lvalue.getAlignment()) *
86            lvalue.getAlignment();
87        VoidPtrAddr = CGF.Builder.CreateConstGEP1_64(
88            VoidPtrAddr, OffsetInChars.getQuantity());
89        auto Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
90            VoidPtrAddr,
91            CGF.Builder.getIntNTy(AtomicSizeInBits)->getPointerTo(),
92            "atomic_bitfield_base");
93        BFI = OrigBFI;
94        BFI.Offset = Offset;
95        BFI.StorageSize = AtomicSizeInBits;
96        BFI.StorageOffset += OffsetInChars;
97        LVal = LValue::MakeBitfield(Address(Addr, lvalue.getAlignment()),
98                                    BFI, lvalue.getType(),
99                                    lvalue.getAlignmentSource());
100        LVal.setTBAAInfo(lvalue.getTBAAInfo());
101        AtomicTy = C.getIntTypeForBitwidth(AtomicSizeInBits, OrigBFI.IsSigned);
102        if (AtomicTy.isNull()) {
103          llvm::APInt Size(
104              /*numBits=*/32,
105              C.toCharUnitsFromBits(AtomicSizeInBits).getQuantity());
106          AtomicTy = C.getConstantArrayType(C.CharTy, Size, ArrayType::Normal,
107                                            /*IndexTypeQuals=*/0);
108        }
109        AtomicAlign = ValueAlign = lvalue.getAlignment();
110      } else if (lvalue.isVectorElt()) {
111        ValueTy = lvalue.getType()->getAs<VectorType>()->getElementType();
112        ValueSizeInBits = C.getTypeSize(ValueTy);
113        AtomicTy = lvalue.getType();
114        AtomicSizeInBits = C.getTypeSize(AtomicTy);
115        AtomicAlign = ValueAlign = lvalue.getAlignment();
116        LVal = lvalue;
117      } else {
118        assert(lvalue.isExtVectorElt());
119        ValueTy = lvalue.getType();
120        ValueSizeInBits = C.getTypeSize(ValueTy);
121        AtomicTy = ValueTy = CGF.getContext().getExtVectorType(
122            lvalue.getType(), lvalue.getExtVectorAddress()
123                                  .getElementType()->getVectorNumElements());
124        AtomicSizeInBits = C.getTypeSize(AtomicTy);
125        AtomicAlign = ValueAlign = lvalue.getAlignment();
126        LVal = lvalue;
127      }
128      UseLibcall = !C.getTargetInfo().hasBuiltinAtomic(
129          AtomicSizeInBits, C.toBits(lvalue.getAlignment()));
130    }
131
132    QualType getAtomicType() const { return AtomicTy; }
133    QualType getValueType() const { return ValueTy; }
134    CharUnits getAtomicAlignment() const { return AtomicAlign; }
135    CharUnits getValueAlignment() const { return ValueAlign; }
136    uint64_t getAtomicSizeInBits() const { return AtomicSizeInBits; }
137    uint64_t getValueSizeInBits() const { return ValueSizeInBits; }
138    TypeEvaluationKind getEvaluationKind() const { return EvaluationKind; }
139    bool shouldUseLibcall() const { return UseLibcall; }
140    const LValue &getAtomicLValue() const { return LVal; }
141    llvm::Value *getAtomicPointer() const {
142      if (LVal.isSimple())
143        return LVal.getPointer();
144      else if (LVal.isBitField())
145        return LVal.getBitFieldPointer();
146      else if (LVal.isVectorElt())
147        return LVal.getVectorPointer();
148      assert(LVal.isExtVectorElt());
149      return LVal.getExtVectorPointer();
150    }
151    Address getAtomicAddress() const {
152      return Address(getAtomicPointer(), getAtomicAlignment());
153    }
154
155    Address getAtomicAddressAsAtomicIntPointer() const {
156      return emitCastToAtomicIntPointer(getAtomicAddress());
157    }
158
159    /// Is the atomic size larger than the underlying value type?
160    ///
161    /// Note that the absence of padding does not mean that atomic
162    /// objects are completely interchangeable with non-atomic
163    /// objects: we might have promoted the alignment of a type
164    /// without making it bigger.
165    bool hasPadding() const {
166      return (ValueSizeInBits != AtomicSizeInBits);
167    }
168
169    bool emitMemSetZeroIfNecessary() const;
170
171    llvm::Value *getAtomicSizeValue() const {
172      CharUnits size = CGF.getContext().toCharUnitsFromBits(AtomicSizeInBits);
173      return CGF.CGM.getSize(size);
174    }
175
176    /// Cast the given pointer to an integer pointer suitable for atomic
177    /// operations if the source.
178    Address emitCastToAtomicIntPointer(Address Addr) const;
179
180    /// If Addr is compatible with the iN that will be used for an atomic
181    /// operation, bitcast it. Otherwise, create a temporary that is suitable
182    /// and copy the value across.
183    Address convertToAtomicIntPointer(Address Addr) const;
184
185    /// Turn an atomic-layout object into an r-value.
186    RValue convertAtomicTempToRValue(Address addr, AggValueSlot resultSlot,
187                                     SourceLocation loc, bool AsValue) const;
188
189    /// \brief Converts a rvalue to integer value.
190    llvm::Value *convertRValueToInt(RValue RVal) const;
191
192    RValue ConvertIntToValueOrAtomic(llvm::Value *IntVal,
193                                     AggValueSlot ResultSlot,
194                                     SourceLocation Loc, bool AsValue) const;
195
196    /// Copy an atomic r-value into atomic-layout memory.
197    void emitCopyIntoMemory(RValue rvalue) const;
198
199    /// Project an l-value down to the value field.
200    LValue projectValue() const {
201      assert(LVal.isSimple());
202      Address addr = getAtomicAddress();
203      if (hasPadding())
204        addr = CGF.Builder.CreateStructGEP(addr, 0, CharUnits());
205
206      return LValue::MakeAddr(addr, getValueType(), CGF.getContext(),
207                              LVal.getAlignmentSource(), LVal.getTBAAInfo());
208    }
209
210    /// \brief Emits atomic load.
211    /// \returns Loaded value.
212    RValue EmitAtomicLoad(AggValueSlot ResultSlot, SourceLocation Loc,
213                          bool AsValue, llvm::AtomicOrdering AO,
214                          bool IsVolatile);
215
216    /// \brief Emits atomic compare-and-exchange sequence.
217    /// \param Expected Expected value.
218    /// \param Desired Desired value.
219    /// \param Success Atomic ordering for success operation.
220    /// \param Failure Atomic ordering for failed operation.
221    /// \param IsWeak true if atomic operation is weak, false otherwise.
222    /// \returns Pair of values: previous value from storage (value type) and
223    /// boolean flag (i1 type) with true if success and false otherwise.
224    std::pair<RValue, llvm::Value *>
225    EmitAtomicCompareExchange(RValue Expected, RValue Desired,
226                              llvm::AtomicOrdering Success =
227                                  llvm::AtomicOrdering::SequentiallyConsistent,
228                              llvm::AtomicOrdering Failure =
229                                  llvm::AtomicOrdering::SequentiallyConsistent,
230                              bool IsWeak = false);
231
232    /// \brief Emits atomic update.
233    /// \param AO Atomic ordering.
234    /// \param UpdateOp Update operation for the current lvalue.
235    void EmitAtomicUpdate(llvm::AtomicOrdering AO,
236                          const llvm::function_ref<RValue(RValue)> &UpdateOp,
237                          bool IsVolatile);
238    /// \brief Emits atomic update.
239    /// \param AO Atomic ordering.
240    void EmitAtomicUpdate(llvm::AtomicOrdering AO, RValue UpdateRVal,
241                          bool IsVolatile);
242
243    /// Materialize an atomic r-value in atomic-layout memory.
244    Address materializeRValue(RValue rvalue) const;
245
246    /// \brief Creates temp alloca for intermediate operations on atomic value.
247    Address CreateTempAlloca() const;
248  private:
249    bool requiresMemSetZero(llvm::Type *type) const;
250
251
252    /// \brief Emits atomic load as a libcall.
253    void EmitAtomicLoadLibcall(llvm::Value *AddForLoaded,
254                               llvm::AtomicOrdering AO, bool IsVolatile);
255    /// \brief Emits atomic load as LLVM instruction.
256    llvm::Value *EmitAtomicLoadOp(llvm::AtomicOrdering AO, bool IsVolatile);
257    /// \brief Emits atomic compare-and-exchange op as a libcall.
258    llvm::Value *EmitAtomicCompareExchangeLibcall(
259        llvm::Value *ExpectedAddr, llvm::Value *DesiredAddr,
260        llvm::AtomicOrdering Success =
261            llvm::AtomicOrdering::SequentiallyConsistent,
262        llvm::AtomicOrdering Failure =
263            llvm::AtomicOrdering::SequentiallyConsistent);
264    /// \brief Emits atomic compare-and-exchange op as LLVM instruction.
265    std::pair<llvm::Value *, llvm::Value *> EmitAtomicCompareExchangeOp(
266        llvm::Value *ExpectedVal, llvm::Value *DesiredVal,
267        llvm::AtomicOrdering Success =
268            llvm::AtomicOrdering::SequentiallyConsistent,
269        llvm::AtomicOrdering Failure =
270            llvm::AtomicOrdering::SequentiallyConsistent,
271        bool IsWeak = false);
272    /// \brief Emit atomic update as libcalls.
273    void
274    EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO,
275                            const llvm::function_ref<RValue(RValue)> &UpdateOp,
276                            bool IsVolatile);
277    /// \brief Emit atomic update as LLVM instructions.
278    void EmitAtomicUpdateOp(llvm::AtomicOrdering AO,
279                            const llvm::function_ref<RValue(RValue)> &UpdateOp,
280                            bool IsVolatile);
281    /// \brief Emit atomic update as libcalls.
282    void EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO, RValue UpdateRVal,
283                                 bool IsVolatile);
284    /// \brief Emit atomic update as LLVM instructions.
285    void EmitAtomicUpdateOp(llvm::AtomicOrdering AO, RValue UpdateRal,
286                            bool IsVolatile);
287  };
288}
289
290Address AtomicInfo::CreateTempAlloca() const {
291  Address TempAlloca = CGF.CreateMemTemp(
292      (LVal.isBitField() && ValueSizeInBits > AtomicSizeInBits) ? ValueTy
293                                                                : AtomicTy,
294      getAtomicAlignment(),
295      "atomic-temp");
296  // Cast to pointer to value type for bitfields.
297  if (LVal.isBitField())
298    return CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
299        TempAlloca, getAtomicAddress().getType());
300  return TempAlloca;
301}
302
303static RValue emitAtomicLibcall(CodeGenFunction &CGF,
304                                StringRef fnName,
305                                QualType resultType,
306                                CallArgList &args) {
307  const CGFunctionInfo &fnInfo =
308    CGF.CGM.getTypes().arrangeBuiltinFunctionCall(resultType, args);
309  llvm::FunctionType *fnTy = CGF.CGM.getTypes().GetFunctionType(fnInfo);
310  llvm::Constant *fn = CGF.CGM.CreateRuntimeFunction(fnTy, fnName);
311  return CGF.EmitCall(fnInfo, fn, ReturnValueSlot(), args);
312}
313
314/// Does a store of the given IR type modify the full expected width?
315static bool isFullSizeType(CodeGenModule &CGM, llvm::Type *type,
316                           uint64_t expectedSize) {
317  return (CGM.getDataLayout().getTypeStoreSize(type) * 8 == expectedSize);
318}
319
320/// Does the atomic type require memsetting to zero before initialization?
321///
322/// The IR type is provided as a way of making certain queries faster.
323bool AtomicInfo::requiresMemSetZero(llvm::Type *type) const {
324  // If the atomic type has size padding, we definitely need a memset.
325  if (hasPadding()) return true;
326
327  // Otherwise, do some simple heuristics to try to avoid it:
328  switch (getEvaluationKind()) {
329  // For scalars and complexes, check whether the store size of the
330  // type uses the full size.
331  case TEK_Scalar:
332    return !isFullSizeType(CGF.CGM, type, AtomicSizeInBits);
333  case TEK_Complex:
334    return !isFullSizeType(CGF.CGM, type->getStructElementType(0),
335                           AtomicSizeInBits / 2);
336
337  // Padding in structs has an undefined bit pattern.  User beware.
338  case TEK_Aggregate:
339    return false;
340  }
341  llvm_unreachable("bad evaluation kind");
342}
343
344bool AtomicInfo::emitMemSetZeroIfNecessary() const {
345  assert(LVal.isSimple());
346  llvm::Value *addr = LVal.getPointer();
347  if (!requiresMemSetZero(addr->getType()->getPointerElementType()))
348    return false;
349
350  CGF.Builder.CreateMemSet(
351      addr, llvm::ConstantInt::get(CGF.Int8Ty, 0),
352      CGF.getContext().toCharUnitsFromBits(AtomicSizeInBits).getQuantity(),
353      LVal.getAlignment().getQuantity());
354  return true;
355}
356
357static void emitAtomicCmpXchg(CodeGenFunction &CGF, AtomicExpr *E, bool IsWeak,
358                              Address Dest, Address Ptr,
359                              Address Val1, Address Val2,
360                              uint64_t Size,
361                              llvm::AtomicOrdering SuccessOrder,
362                              llvm::AtomicOrdering FailureOrder) {
363  // Note that cmpxchg doesn't support weak cmpxchg, at least at the moment.
364  llvm::Value *Expected = CGF.Builder.CreateLoad(Val1);
365  llvm::Value *Desired = CGF.Builder.CreateLoad(Val2);
366
367  llvm::AtomicCmpXchgInst *Pair = CGF.Builder.CreateAtomicCmpXchg(
368      Ptr.getPointer(), Expected, Desired, SuccessOrder, FailureOrder);
369  Pair->setVolatile(E->isVolatile());
370  Pair->setWeak(IsWeak);
371
372  // Cmp holds the result of the compare-exchange operation: true on success,
373  // false on failure.
374  llvm::Value *Old = CGF.Builder.CreateExtractValue(Pair, 0);
375  llvm::Value *Cmp = CGF.Builder.CreateExtractValue(Pair, 1);
376
377  // This basic block is used to hold the store instruction if the operation
378  // failed.
379  llvm::BasicBlock *StoreExpectedBB =
380      CGF.createBasicBlock("cmpxchg.store_expected", CGF.CurFn);
381
382  // This basic block is the exit point of the operation, we should end up
383  // here regardless of whether or not the operation succeeded.
384  llvm::BasicBlock *ContinueBB =
385      CGF.createBasicBlock("cmpxchg.continue", CGF.CurFn);
386
387  // Update Expected if Expected isn't equal to Old, otherwise branch to the
388  // exit point.
389  CGF.Builder.CreateCondBr(Cmp, ContinueBB, StoreExpectedBB);
390
391  CGF.Builder.SetInsertPoint(StoreExpectedBB);
392  // Update the memory at Expected with Old's value.
393  CGF.Builder.CreateStore(Old, Val1);
394  // Finally, branch to the exit point.
395  CGF.Builder.CreateBr(ContinueBB);
396
397  CGF.Builder.SetInsertPoint(ContinueBB);
398  // Update the memory at Dest with Cmp's value.
399  CGF.EmitStoreOfScalar(Cmp, CGF.MakeAddrLValue(Dest, E->getType()));
400}
401
402/// Given an ordering required on success, emit all possible cmpxchg
403/// instructions to cope with the provided (but possibly only dynamically known)
404/// FailureOrder.
405static void emitAtomicCmpXchgFailureSet(CodeGenFunction &CGF, AtomicExpr *E,
406                                        bool IsWeak, Address Dest, Address Ptr,
407                                        Address Val1, Address Val2,
408                                        llvm::Value *FailureOrderVal,
409                                        uint64_t Size,
410                                        llvm::AtomicOrdering SuccessOrder) {
411  llvm::AtomicOrdering FailureOrder;
412  if (llvm::ConstantInt *FO = dyn_cast<llvm::ConstantInt>(FailureOrderVal)) {
413    auto FOS = FO->getSExtValue();
414    if (!llvm::isValidAtomicOrderingCABI(FOS))
415      FailureOrder = llvm::AtomicOrdering::Monotonic;
416    else
417      switch ((llvm::AtomicOrderingCABI)FOS) {
418      case llvm::AtomicOrderingCABI::relaxed:
419      case llvm::AtomicOrderingCABI::release:
420      case llvm::AtomicOrderingCABI::acq_rel:
421        FailureOrder = llvm::AtomicOrdering::Monotonic;
422        break;
423      case llvm::AtomicOrderingCABI::consume:
424      case llvm::AtomicOrderingCABI::acquire:
425        FailureOrder = llvm::AtomicOrdering::Acquire;
426        break;
427      case llvm::AtomicOrderingCABI::seq_cst:
428        FailureOrder = llvm::AtomicOrdering::SequentiallyConsistent;
429        break;
430      }
431    if (isStrongerThan(FailureOrder, SuccessOrder)) {
432      // Don't assert on undefined behavior "failure argument shall be no
433      // stronger than the success argument".
434      FailureOrder =
435          llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(SuccessOrder);
436    }
437    emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, Size, SuccessOrder,
438                      FailureOrder);
439    return;
440  }
441
442  // Create all the relevant BB's
443  llvm::BasicBlock *MonotonicBB = nullptr, *AcquireBB = nullptr,
444                   *SeqCstBB = nullptr;
445  MonotonicBB = CGF.createBasicBlock("monotonic_fail", CGF.CurFn);
446  if (SuccessOrder != llvm::AtomicOrdering::Monotonic &&
447      SuccessOrder != llvm::AtomicOrdering::Release)
448    AcquireBB = CGF.createBasicBlock("acquire_fail", CGF.CurFn);
449  if (SuccessOrder == llvm::AtomicOrdering::SequentiallyConsistent)
450    SeqCstBB = CGF.createBasicBlock("seqcst_fail", CGF.CurFn);
451
452  llvm::BasicBlock *ContBB = CGF.createBasicBlock("atomic.continue", CGF.CurFn);
453
454  llvm::SwitchInst *SI = CGF.Builder.CreateSwitch(FailureOrderVal, MonotonicBB);
455
456  // Emit all the different atomics
457
458  // MonotonicBB is arbitrarily chosen as the default case; in practice, this
459  // doesn't matter unless someone is crazy enough to use something that
460  // doesn't fold to a constant for the ordering.
461  CGF.Builder.SetInsertPoint(MonotonicBB);
462  emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2,
463                    Size, SuccessOrder, llvm::AtomicOrdering::Monotonic);
464  CGF.Builder.CreateBr(ContBB);
465
466  if (AcquireBB) {
467    CGF.Builder.SetInsertPoint(AcquireBB);
468    emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2,
469                      Size, SuccessOrder, llvm::AtomicOrdering::Acquire);
470    CGF.Builder.CreateBr(ContBB);
471    SI->addCase(CGF.Builder.getInt32((int)llvm::AtomicOrderingCABI::consume),
472                AcquireBB);
473    SI->addCase(CGF.Builder.getInt32((int)llvm::AtomicOrderingCABI::acquire),
474                AcquireBB);
475  }
476  if (SeqCstBB) {
477    CGF.Builder.SetInsertPoint(SeqCstBB);
478    emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, Size, SuccessOrder,
479                      llvm::AtomicOrdering::SequentiallyConsistent);
480    CGF.Builder.CreateBr(ContBB);
481    SI->addCase(CGF.Builder.getInt32((int)llvm::AtomicOrderingCABI::seq_cst),
482                SeqCstBB);
483  }
484
485  CGF.Builder.SetInsertPoint(ContBB);
486}
487
488static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *E, Address Dest,
489                         Address Ptr, Address Val1, Address Val2,
490                         llvm::Value *IsWeak, llvm::Value *FailureOrder,
491                         uint64_t Size, llvm::AtomicOrdering Order) {
492  llvm::AtomicRMWInst::BinOp Op = llvm::AtomicRMWInst::Add;
493  llvm::Instruction::BinaryOps PostOp = (llvm::Instruction::BinaryOps)0;
494
495  switch (E->getOp()) {
496  case AtomicExpr::AO__c11_atomic_init:
497    llvm_unreachable("Already handled!");
498
499  case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
500    emitAtomicCmpXchgFailureSet(CGF, E, false, Dest, Ptr, Val1, Val2,
501                                FailureOrder, Size, Order);
502    return;
503  case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
504    emitAtomicCmpXchgFailureSet(CGF, E, true, Dest, Ptr, Val1, Val2,
505                                FailureOrder, Size, Order);
506    return;
507  case AtomicExpr::AO__atomic_compare_exchange:
508  case AtomicExpr::AO__atomic_compare_exchange_n: {
509    if (llvm::ConstantInt *IsWeakC = dyn_cast<llvm::ConstantInt>(IsWeak)) {
510      emitAtomicCmpXchgFailureSet(CGF, E, IsWeakC->getZExtValue(), Dest, Ptr,
511                                  Val1, Val2, FailureOrder, Size, Order);
512    } else {
513      // Create all the relevant BB's
514      llvm::BasicBlock *StrongBB =
515          CGF.createBasicBlock("cmpxchg.strong", CGF.CurFn);
516      llvm::BasicBlock *WeakBB = CGF.createBasicBlock("cmxchg.weak", CGF.CurFn);
517      llvm::BasicBlock *ContBB =
518          CGF.createBasicBlock("cmpxchg.continue", CGF.CurFn);
519
520      llvm::SwitchInst *SI = CGF.Builder.CreateSwitch(IsWeak, WeakBB);
521      SI->addCase(CGF.Builder.getInt1(false), StrongBB);
522
523      CGF.Builder.SetInsertPoint(StrongBB);
524      emitAtomicCmpXchgFailureSet(CGF, E, false, Dest, Ptr, Val1, Val2,
525                                  FailureOrder, Size, Order);
526      CGF.Builder.CreateBr(ContBB);
527
528      CGF.Builder.SetInsertPoint(WeakBB);
529      emitAtomicCmpXchgFailureSet(CGF, E, true, Dest, Ptr, Val1, Val2,
530                                  FailureOrder, Size, Order);
531      CGF.Builder.CreateBr(ContBB);
532
533      CGF.Builder.SetInsertPoint(ContBB);
534    }
535    return;
536  }
537  case AtomicExpr::AO__c11_atomic_load:
538  case AtomicExpr::AO__atomic_load_n:
539  case AtomicExpr::AO__atomic_load: {
540    llvm::LoadInst *Load = CGF.Builder.CreateLoad(Ptr);
541    Load->setAtomic(Order);
542    Load->setVolatile(E->isVolatile());
543    CGF.Builder.CreateStore(Load, Dest);
544    return;
545  }
546
547  case AtomicExpr::AO__c11_atomic_store:
548  case AtomicExpr::AO__atomic_store:
549  case AtomicExpr::AO__atomic_store_n: {
550    llvm::Value *LoadVal1 = CGF.Builder.CreateLoad(Val1);
551    llvm::StoreInst *Store = CGF.Builder.CreateStore(LoadVal1, Ptr);
552    Store->setAtomic(Order);
553    Store->setVolatile(E->isVolatile());
554    return;
555  }
556
557  case AtomicExpr::AO__c11_atomic_exchange:
558  case AtomicExpr::AO__atomic_exchange_n:
559  case AtomicExpr::AO__atomic_exchange:
560    Op = llvm::AtomicRMWInst::Xchg;
561    break;
562
563  case AtomicExpr::AO__atomic_add_fetch:
564    PostOp = llvm::Instruction::Add;
565    // Fall through.
566  case AtomicExpr::AO__c11_atomic_fetch_add:
567  case AtomicExpr::AO__atomic_fetch_add:
568    Op = llvm::AtomicRMWInst::Add;
569    break;
570
571  case AtomicExpr::AO__atomic_sub_fetch:
572    PostOp = llvm::Instruction::Sub;
573    // Fall through.
574  case AtomicExpr::AO__c11_atomic_fetch_sub:
575  case AtomicExpr::AO__atomic_fetch_sub:
576    Op = llvm::AtomicRMWInst::Sub;
577    break;
578
579  case AtomicExpr::AO__atomic_and_fetch:
580    PostOp = llvm::Instruction::And;
581    // Fall through.
582  case AtomicExpr::AO__c11_atomic_fetch_and:
583  case AtomicExpr::AO__atomic_fetch_and:
584    Op = llvm::AtomicRMWInst::And;
585    break;
586
587  case AtomicExpr::AO__atomic_or_fetch:
588    PostOp = llvm::Instruction::Or;
589    // Fall through.
590  case AtomicExpr::AO__c11_atomic_fetch_or:
591  case AtomicExpr::AO__atomic_fetch_or:
592    Op = llvm::AtomicRMWInst::Or;
593    break;
594
595  case AtomicExpr::AO__atomic_xor_fetch:
596    PostOp = llvm::Instruction::Xor;
597    // Fall through.
598  case AtomicExpr::AO__c11_atomic_fetch_xor:
599  case AtomicExpr::AO__atomic_fetch_xor:
600    Op = llvm::AtomicRMWInst::Xor;
601    break;
602
603  case AtomicExpr::AO__atomic_nand_fetch:
604    PostOp = llvm::Instruction::And; // the NOT is special cased below
605  // Fall through.
606  case AtomicExpr::AO__atomic_fetch_nand:
607    Op = llvm::AtomicRMWInst::Nand;
608    break;
609  }
610
611  llvm::Value *LoadVal1 = CGF.Builder.CreateLoad(Val1);
612  llvm::AtomicRMWInst *RMWI =
613      CGF.Builder.CreateAtomicRMW(Op, Ptr.getPointer(), LoadVal1, Order);
614  RMWI->setVolatile(E->isVolatile());
615
616  // For __atomic_*_fetch operations, perform the operation again to
617  // determine the value which was written.
618  llvm::Value *Result = RMWI;
619  if (PostOp)
620    Result = CGF.Builder.CreateBinOp(PostOp, RMWI, LoadVal1);
621  if (E->getOp() == AtomicExpr::AO__atomic_nand_fetch)
622    Result = CGF.Builder.CreateNot(Result);
623  CGF.Builder.CreateStore(Result, Dest);
624}
625
626// This function emits any expression (scalar, complex, or aggregate)
627// into a temporary alloca.
628static Address
629EmitValToTemp(CodeGenFunction &CGF, Expr *E) {
630  Address DeclPtr = CGF.CreateMemTemp(E->getType(), ".atomictmp");
631  CGF.EmitAnyExprToMem(E, DeclPtr, E->getType().getQualifiers(),
632                       /*Init*/ true);
633  return DeclPtr;
634}
635
636static void
637AddDirectArgument(CodeGenFunction &CGF, CallArgList &Args,
638                  bool UseOptimizedLibcall, llvm::Value *Val, QualType ValTy,
639                  SourceLocation Loc, CharUnits SizeInChars) {
640  if (UseOptimizedLibcall) {
641    // Load value and pass it to the function directly.
642    CharUnits Align = CGF.getContext().getTypeAlignInChars(ValTy);
643    int64_t SizeInBits = CGF.getContext().toBits(SizeInChars);
644    ValTy =
645        CGF.getContext().getIntTypeForBitwidth(SizeInBits, /*Signed=*/false);
646    llvm::Type *IPtrTy = llvm::IntegerType::get(CGF.getLLVMContext(),
647                                                SizeInBits)->getPointerTo();
648    Address Ptr = Address(CGF.Builder.CreateBitCast(Val, IPtrTy), Align);
649    Val = CGF.EmitLoadOfScalar(Ptr, false,
650                               CGF.getContext().getPointerType(ValTy),
651                               Loc);
652    // Coerce the value into an appropriately sized integer type.
653    Args.add(RValue::get(Val), ValTy);
654  } else {
655    // Non-optimized functions always take a reference.
656    Args.add(RValue::get(CGF.EmitCastToVoidPtr(Val)),
657                         CGF.getContext().VoidPtrTy);
658  }
659}
660
661RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) {
662  QualType AtomicTy = E->getPtr()->getType()->getPointeeType();
663  QualType MemTy = AtomicTy;
664  if (const AtomicType *AT = AtomicTy->getAs<AtomicType>())
665    MemTy = AT->getValueType();
666  CharUnits sizeChars, alignChars;
667  std::tie(sizeChars, alignChars) = getContext().getTypeInfoInChars(AtomicTy);
668  uint64_t Size = sizeChars.getQuantity();
669  unsigned MaxInlineWidthInBits = getTarget().getMaxAtomicInlineWidth();
670  bool UseLibcall = (sizeChars != alignChars ||
671                     getContext().toBits(sizeChars) > MaxInlineWidthInBits);
672
673  llvm::Value *IsWeak = nullptr, *OrderFail = nullptr;
674
675  Address Val1 = Address::invalid();
676  Address Val2 = Address::invalid();
677  Address Dest = Address::invalid();
678  Address Ptr(EmitScalarExpr(E->getPtr()), alignChars);
679
680  if (E->getOp() == AtomicExpr::AO__c11_atomic_init) {
681    LValue lvalue = MakeAddrLValue(Ptr, AtomicTy);
682    EmitAtomicInit(E->getVal1(), lvalue);
683    return RValue::get(nullptr);
684  }
685
686  llvm::Value *Order = EmitScalarExpr(E->getOrder());
687
688  switch (E->getOp()) {
689  case AtomicExpr::AO__c11_atomic_init:
690    llvm_unreachable("Already handled above with EmitAtomicInit!");
691
692  case AtomicExpr::AO__c11_atomic_load:
693  case AtomicExpr::AO__atomic_load_n:
694    break;
695
696  case AtomicExpr::AO__atomic_load:
697    Dest = EmitPointerWithAlignment(E->getVal1());
698    break;
699
700  case AtomicExpr::AO__atomic_store:
701    Val1 = EmitPointerWithAlignment(E->getVal1());
702    break;
703
704  case AtomicExpr::AO__atomic_exchange:
705    Val1 = EmitPointerWithAlignment(E->getVal1());
706    Dest = EmitPointerWithAlignment(E->getVal2());
707    break;
708
709  case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
710  case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
711  case AtomicExpr::AO__atomic_compare_exchange_n:
712  case AtomicExpr::AO__atomic_compare_exchange:
713    Val1 = EmitPointerWithAlignment(E->getVal1());
714    if (E->getOp() == AtomicExpr::AO__atomic_compare_exchange)
715      Val2 = EmitPointerWithAlignment(E->getVal2());
716    else
717      Val2 = EmitValToTemp(*this, E->getVal2());
718    OrderFail = EmitScalarExpr(E->getOrderFail());
719    if (E->getNumSubExprs() == 6)
720      IsWeak = EmitScalarExpr(E->getWeak());
721    break;
722
723  case AtomicExpr::AO__c11_atomic_fetch_add:
724  case AtomicExpr::AO__c11_atomic_fetch_sub:
725    if (MemTy->isPointerType()) {
726      // For pointer arithmetic, we're required to do a bit of math:
727      // adding 1 to an int* is not the same as adding 1 to a uintptr_t.
728      // ... but only for the C11 builtins. The GNU builtins expect the
729      // user to multiply by sizeof(T).
730      QualType Val1Ty = E->getVal1()->getType();
731      llvm::Value *Val1Scalar = EmitScalarExpr(E->getVal1());
732      CharUnits PointeeIncAmt =
733          getContext().getTypeSizeInChars(MemTy->getPointeeType());
734      Val1Scalar = Builder.CreateMul(Val1Scalar, CGM.getSize(PointeeIncAmt));
735      auto Temp = CreateMemTemp(Val1Ty, ".atomictmp");
736      Val1 = Temp;
737      EmitStoreOfScalar(Val1Scalar, MakeAddrLValue(Temp, Val1Ty));
738      break;
739    }
740    // Fall through.
741  case AtomicExpr::AO__atomic_fetch_add:
742  case AtomicExpr::AO__atomic_fetch_sub:
743  case AtomicExpr::AO__atomic_add_fetch:
744  case AtomicExpr::AO__atomic_sub_fetch:
745  case AtomicExpr::AO__c11_atomic_store:
746  case AtomicExpr::AO__c11_atomic_exchange:
747  case AtomicExpr::AO__atomic_store_n:
748  case AtomicExpr::AO__atomic_exchange_n:
749  case AtomicExpr::AO__c11_atomic_fetch_and:
750  case AtomicExpr::AO__c11_atomic_fetch_or:
751  case AtomicExpr::AO__c11_atomic_fetch_xor:
752  case AtomicExpr::AO__atomic_fetch_and:
753  case AtomicExpr::AO__atomic_fetch_or:
754  case AtomicExpr::AO__atomic_fetch_xor:
755  case AtomicExpr::AO__atomic_fetch_nand:
756  case AtomicExpr::AO__atomic_and_fetch:
757  case AtomicExpr::AO__atomic_or_fetch:
758  case AtomicExpr::AO__atomic_xor_fetch:
759  case AtomicExpr::AO__atomic_nand_fetch:
760    Val1 = EmitValToTemp(*this, E->getVal1());
761    break;
762  }
763
764  QualType RValTy = E->getType().getUnqualifiedType();
765
766  // The inlined atomics only function on iN types, where N is a power of 2. We
767  // need to make sure (via temporaries if necessary) that all incoming values
768  // are compatible.
769  LValue AtomicVal = MakeAddrLValue(Ptr, AtomicTy);
770  AtomicInfo Atomics(*this, AtomicVal);
771
772  Ptr = Atomics.emitCastToAtomicIntPointer(Ptr);
773  if (Val1.isValid()) Val1 = Atomics.convertToAtomicIntPointer(Val1);
774  if (Val2.isValid()) Val2 = Atomics.convertToAtomicIntPointer(Val2);
775  if (Dest.isValid())
776    Dest = Atomics.emitCastToAtomicIntPointer(Dest);
777  else if (E->isCmpXChg())
778    Dest = CreateMemTemp(RValTy, "cmpxchg.bool");
779  else if (!RValTy->isVoidType())
780    Dest = Atomics.emitCastToAtomicIntPointer(Atomics.CreateTempAlloca());
781
782  // Use a library call.  See: http://gcc.gnu.org/wiki/Atomic/GCCMM/LIbrary .
783  if (UseLibcall) {
784    bool UseOptimizedLibcall = false;
785    switch (E->getOp()) {
786    case AtomicExpr::AO__c11_atomic_init:
787      llvm_unreachable("Already handled above with EmitAtomicInit!");
788
789    case AtomicExpr::AO__c11_atomic_fetch_add:
790    case AtomicExpr::AO__atomic_fetch_add:
791    case AtomicExpr::AO__c11_atomic_fetch_and:
792    case AtomicExpr::AO__atomic_fetch_and:
793    case AtomicExpr::AO__c11_atomic_fetch_or:
794    case AtomicExpr::AO__atomic_fetch_or:
795    case AtomicExpr::AO__atomic_fetch_nand:
796    case AtomicExpr::AO__c11_atomic_fetch_sub:
797    case AtomicExpr::AO__atomic_fetch_sub:
798    case AtomicExpr::AO__c11_atomic_fetch_xor:
799    case AtomicExpr::AO__atomic_fetch_xor:
800    case AtomicExpr::AO__atomic_add_fetch:
801    case AtomicExpr::AO__atomic_and_fetch:
802    case AtomicExpr::AO__atomic_nand_fetch:
803    case AtomicExpr::AO__atomic_or_fetch:
804    case AtomicExpr::AO__atomic_sub_fetch:
805    case AtomicExpr::AO__atomic_xor_fetch:
806      // For these, only library calls for certain sizes exist.
807      UseOptimizedLibcall = true;
808      break;
809
810    case AtomicExpr::AO__c11_atomic_load:
811    case AtomicExpr::AO__c11_atomic_store:
812    case AtomicExpr::AO__c11_atomic_exchange:
813    case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
814    case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
815    case AtomicExpr::AO__atomic_load_n:
816    case AtomicExpr::AO__atomic_load:
817    case AtomicExpr::AO__atomic_store_n:
818    case AtomicExpr::AO__atomic_store:
819    case AtomicExpr::AO__atomic_exchange_n:
820    case AtomicExpr::AO__atomic_exchange:
821    case AtomicExpr::AO__atomic_compare_exchange_n:
822    case AtomicExpr::AO__atomic_compare_exchange:
823      // Only use optimized library calls for sizes for which they exist.
824      if (Size == 1 || Size == 2 || Size == 4 || Size == 8)
825        UseOptimizedLibcall = true;
826      break;
827    }
828
829    CallArgList Args;
830    if (!UseOptimizedLibcall) {
831      // For non-optimized library calls, the size is the first parameter
832      Args.add(RValue::get(llvm::ConstantInt::get(SizeTy, Size)),
833               getContext().getSizeType());
834    }
835    // Atomic address is the first or second parameter
836    Args.add(RValue::get(EmitCastToVoidPtr(Ptr.getPointer())),
837             getContext().VoidPtrTy);
838
839    std::string LibCallName;
840    QualType LoweredMemTy =
841      MemTy->isPointerType() ? getContext().getIntPtrType() : MemTy;
842    QualType RetTy;
843    bool HaveRetTy = false;
844    llvm::Instruction::BinaryOps PostOp = (llvm::Instruction::BinaryOps)0;
845    switch (E->getOp()) {
846    case AtomicExpr::AO__c11_atomic_init:
847      llvm_unreachable("Already handled!");
848
849    // There is only one libcall for compare an exchange, because there is no
850    // optimisation benefit possible from a libcall version of a weak compare
851    // and exchange.
852    // bool __atomic_compare_exchange(size_t size, void *mem, void *expected,
853    //                                void *desired, int success, int failure)
854    // bool __atomic_compare_exchange_N(T *mem, T *expected, T desired,
855    //                                  int success, int failure)
856    case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
857    case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
858    case AtomicExpr::AO__atomic_compare_exchange:
859    case AtomicExpr::AO__atomic_compare_exchange_n:
860      LibCallName = "__atomic_compare_exchange";
861      RetTy = getContext().BoolTy;
862      HaveRetTy = true;
863      Args.add(RValue::get(EmitCastToVoidPtr(Val1.getPointer())),
864               getContext().VoidPtrTy);
865      AddDirectArgument(*this, Args, UseOptimizedLibcall, Val2.getPointer(),
866                        MemTy, E->getExprLoc(), sizeChars);
867      Args.add(RValue::get(Order), getContext().IntTy);
868      Order = OrderFail;
869      break;
870    // void __atomic_exchange(size_t size, void *mem, void *val, void *return,
871    //                        int order)
872    // T __atomic_exchange_N(T *mem, T val, int order)
873    case AtomicExpr::AO__c11_atomic_exchange:
874    case AtomicExpr::AO__atomic_exchange_n:
875    case AtomicExpr::AO__atomic_exchange:
876      LibCallName = "__atomic_exchange";
877      AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
878                        MemTy, E->getExprLoc(), sizeChars);
879      break;
880    // void __atomic_store(size_t size, void *mem, void *val, int order)
881    // void __atomic_store_N(T *mem, T val, int order)
882    case AtomicExpr::AO__c11_atomic_store:
883    case AtomicExpr::AO__atomic_store:
884    case AtomicExpr::AO__atomic_store_n:
885      LibCallName = "__atomic_store";
886      RetTy = getContext().VoidTy;
887      HaveRetTy = true;
888      AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
889                        MemTy, E->getExprLoc(), sizeChars);
890      break;
891    // void __atomic_load(size_t size, void *mem, void *return, int order)
892    // T __atomic_load_N(T *mem, int order)
893    case AtomicExpr::AO__c11_atomic_load:
894    case AtomicExpr::AO__atomic_load:
895    case AtomicExpr::AO__atomic_load_n:
896      LibCallName = "__atomic_load";
897      break;
898    // T __atomic_add_fetch_N(T *mem, T val, int order)
899    // T __atomic_fetch_add_N(T *mem, T val, int order)
900    case AtomicExpr::AO__atomic_add_fetch:
901      PostOp = llvm::Instruction::Add;
902    // Fall through.
903    case AtomicExpr::AO__c11_atomic_fetch_add:
904    case AtomicExpr::AO__atomic_fetch_add:
905      LibCallName = "__atomic_fetch_add";
906      AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
907                        LoweredMemTy, E->getExprLoc(), sizeChars);
908      break;
909    // T __atomic_and_fetch_N(T *mem, T val, int order)
910    // T __atomic_fetch_and_N(T *mem, T val, int order)
911    case AtomicExpr::AO__atomic_and_fetch:
912      PostOp = llvm::Instruction::And;
913    // Fall through.
914    case AtomicExpr::AO__c11_atomic_fetch_and:
915    case AtomicExpr::AO__atomic_fetch_and:
916      LibCallName = "__atomic_fetch_and";
917      AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
918                        MemTy, E->getExprLoc(), sizeChars);
919      break;
920    // T __atomic_or_fetch_N(T *mem, T val, int order)
921    // T __atomic_fetch_or_N(T *mem, T val, int order)
922    case AtomicExpr::AO__atomic_or_fetch:
923      PostOp = llvm::Instruction::Or;
924    // Fall through.
925    case AtomicExpr::AO__c11_atomic_fetch_or:
926    case AtomicExpr::AO__atomic_fetch_or:
927      LibCallName = "__atomic_fetch_or";
928      AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
929                        MemTy, E->getExprLoc(), sizeChars);
930      break;
931    // T __atomic_sub_fetch_N(T *mem, T val, int order)
932    // T __atomic_fetch_sub_N(T *mem, T val, int order)
933    case AtomicExpr::AO__atomic_sub_fetch:
934      PostOp = llvm::Instruction::Sub;
935    // Fall through.
936    case AtomicExpr::AO__c11_atomic_fetch_sub:
937    case AtomicExpr::AO__atomic_fetch_sub:
938      LibCallName = "__atomic_fetch_sub";
939      AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
940                        LoweredMemTy, E->getExprLoc(), sizeChars);
941      break;
942    // T __atomic_xor_fetch_N(T *mem, T val, int order)
943    // T __atomic_fetch_xor_N(T *mem, T val, int order)
944    case AtomicExpr::AO__atomic_xor_fetch:
945      PostOp = llvm::Instruction::Xor;
946    // Fall through.
947    case AtomicExpr::AO__c11_atomic_fetch_xor:
948    case AtomicExpr::AO__atomic_fetch_xor:
949      LibCallName = "__atomic_fetch_xor";
950      AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
951                        MemTy, E->getExprLoc(), sizeChars);
952      break;
953    // T __atomic_nand_fetch_N(T *mem, T val, int order)
954    // T __atomic_fetch_nand_N(T *mem, T val, int order)
955    case AtomicExpr::AO__atomic_nand_fetch:
956      PostOp = llvm::Instruction::And; // the NOT is special cased below
957    // Fall through.
958    case AtomicExpr::AO__atomic_fetch_nand:
959      LibCallName = "__atomic_fetch_nand";
960      AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
961                        MemTy, E->getExprLoc(), sizeChars);
962      break;
963    }
964
965    // Optimized functions have the size in their name.
966    if (UseOptimizedLibcall)
967      LibCallName += "_" + llvm::utostr(Size);
968    // By default, assume we return a value of the atomic type.
969    if (!HaveRetTy) {
970      if (UseOptimizedLibcall) {
971        // Value is returned directly.
972        // The function returns an appropriately sized integer type.
973        RetTy = getContext().getIntTypeForBitwidth(
974            getContext().toBits(sizeChars), /*Signed=*/false);
975      } else {
976        // Value is returned through parameter before the order.
977        RetTy = getContext().VoidTy;
978        Args.add(RValue::get(EmitCastToVoidPtr(Dest.getPointer())),
979                 getContext().VoidPtrTy);
980      }
981    }
982    // order is always the last parameter
983    Args.add(RValue::get(Order),
984             getContext().IntTy);
985
986    // PostOp is only needed for the atomic_*_fetch operations, and
987    // thus is only needed for and implemented in the
988    // UseOptimizedLibcall codepath.
989    assert(UseOptimizedLibcall || !PostOp);
990
991    RValue Res = emitAtomicLibcall(*this, LibCallName, RetTy, Args);
992    // The value is returned directly from the libcall.
993    if (E->isCmpXChg())
994      return Res;
995
996    // The value is returned directly for optimized libcalls but the expr
997    // provided an out-param.
998    if (UseOptimizedLibcall && Res.getScalarVal()) {
999      llvm::Value *ResVal = Res.getScalarVal();
1000      if (PostOp) {
1001        llvm::Value *LoadVal1 = Args[1].RV.getScalarVal();
1002        ResVal = Builder.CreateBinOp(PostOp, ResVal, LoadVal1);
1003      }
1004      if (E->getOp() == AtomicExpr::AO__atomic_nand_fetch)
1005        ResVal = Builder.CreateNot(ResVal);
1006
1007      Builder.CreateStore(
1008          ResVal,
1009          Builder.CreateBitCast(Dest, ResVal->getType()->getPointerTo()));
1010    }
1011
1012    if (RValTy->isVoidType())
1013      return RValue::get(nullptr);
1014
1015    return convertTempToRValue(
1016        Builder.CreateBitCast(Dest, ConvertTypeForMem(RValTy)->getPointerTo()),
1017        RValTy, E->getExprLoc());
1018  }
1019
1020  bool IsStore = E->getOp() == AtomicExpr::AO__c11_atomic_store ||
1021                 E->getOp() == AtomicExpr::AO__atomic_store ||
1022                 E->getOp() == AtomicExpr::AO__atomic_store_n;
1023  bool IsLoad = E->getOp() == AtomicExpr::AO__c11_atomic_load ||
1024                E->getOp() == AtomicExpr::AO__atomic_load ||
1025                E->getOp() == AtomicExpr::AO__atomic_load_n;
1026
1027  if (isa<llvm::ConstantInt>(Order)) {
1028    auto ord = cast<llvm::ConstantInt>(Order)->getZExtValue();
1029    // We should not ever get to a case where the ordering isn't a valid C ABI
1030    // value, but it's hard to enforce that in general.
1031    if (llvm::isValidAtomicOrderingCABI(ord))
1032      switch ((llvm::AtomicOrderingCABI)ord) {
1033      case llvm::AtomicOrderingCABI::relaxed:
1034        EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1035                     llvm::AtomicOrdering::Monotonic);
1036        break;
1037      case llvm::AtomicOrderingCABI::consume:
1038      case llvm::AtomicOrderingCABI::acquire:
1039        if (IsStore)
1040          break; // Avoid crashing on code with undefined behavior
1041        EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1042                     llvm::AtomicOrdering::Acquire);
1043        break;
1044      case llvm::AtomicOrderingCABI::release:
1045        if (IsLoad)
1046          break; // Avoid crashing on code with undefined behavior
1047        EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1048                     llvm::AtomicOrdering::Release);
1049        break;
1050      case llvm::AtomicOrderingCABI::acq_rel:
1051        if (IsLoad || IsStore)
1052          break; // Avoid crashing on code with undefined behavior
1053        EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1054                     llvm::AtomicOrdering::AcquireRelease);
1055        break;
1056      case llvm::AtomicOrderingCABI::seq_cst:
1057        EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1058                     llvm::AtomicOrdering::SequentiallyConsistent);
1059        break;
1060      }
1061    if (RValTy->isVoidType())
1062      return RValue::get(nullptr);
1063
1064    return convertTempToRValue(
1065        Builder.CreateBitCast(Dest, ConvertTypeForMem(RValTy)->getPointerTo()),
1066        RValTy, E->getExprLoc());
1067  }
1068
1069  // Long case, when Order isn't obviously constant.
1070
1071  // Create all the relevant BB's
1072  llvm::BasicBlock *MonotonicBB = nullptr, *AcquireBB = nullptr,
1073                   *ReleaseBB = nullptr, *AcqRelBB = nullptr,
1074                   *SeqCstBB = nullptr;
1075  MonotonicBB = createBasicBlock("monotonic", CurFn);
1076  if (!IsStore)
1077    AcquireBB = createBasicBlock("acquire", CurFn);
1078  if (!IsLoad)
1079    ReleaseBB = createBasicBlock("release", CurFn);
1080  if (!IsLoad && !IsStore)
1081    AcqRelBB = createBasicBlock("acqrel", CurFn);
1082  SeqCstBB = createBasicBlock("seqcst", CurFn);
1083  llvm::BasicBlock *ContBB = createBasicBlock("atomic.continue", CurFn);
1084
1085  // Create the switch for the split
1086  // MonotonicBB is arbitrarily chosen as the default case; in practice, this
1087  // doesn't matter unless someone is crazy enough to use something that
1088  // doesn't fold to a constant for the ordering.
1089  Order = Builder.CreateIntCast(Order, Builder.getInt32Ty(), false);
1090  llvm::SwitchInst *SI = Builder.CreateSwitch(Order, MonotonicBB);
1091
1092  // Emit all the different atomics
1093  Builder.SetInsertPoint(MonotonicBB);
1094  EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail,
1095               Size, llvm::AtomicOrdering::Monotonic);
1096  Builder.CreateBr(ContBB);
1097  if (!IsStore) {
1098    Builder.SetInsertPoint(AcquireBB);
1099    EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail,
1100                 Size, llvm::AtomicOrdering::Acquire);
1101    Builder.CreateBr(ContBB);
1102    SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::consume),
1103                AcquireBB);
1104    SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::acquire),
1105                AcquireBB);
1106  }
1107  if (!IsLoad) {
1108    Builder.SetInsertPoint(ReleaseBB);
1109    EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail,
1110                 Size, llvm::AtomicOrdering::Release);
1111    Builder.CreateBr(ContBB);
1112    SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::release),
1113                ReleaseBB);
1114  }
1115  if (!IsLoad && !IsStore) {
1116    Builder.SetInsertPoint(AcqRelBB);
1117    EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail,
1118                 Size, llvm::AtomicOrdering::AcquireRelease);
1119    Builder.CreateBr(ContBB);
1120    SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::acq_rel),
1121                AcqRelBB);
1122  }
1123  Builder.SetInsertPoint(SeqCstBB);
1124  EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail,
1125               Size, llvm::AtomicOrdering::SequentiallyConsistent);
1126  Builder.CreateBr(ContBB);
1127  SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::seq_cst),
1128              SeqCstBB);
1129
1130  // Cleanup and return
1131  Builder.SetInsertPoint(ContBB);
1132  if (RValTy->isVoidType())
1133    return RValue::get(nullptr);
1134
1135  assert(Atomics.getValueSizeInBits() <= Atomics.getAtomicSizeInBits());
1136  return convertTempToRValue(
1137      Builder.CreateBitCast(Dest, ConvertTypeForMem(RValTy)->getPointerTo()),
1138      RValTy, E->getExprLoc());
1139}
1140
1141Address AtomicInfo::emitCastToAtomicIntPointer(Address addr) const {
1142  unsigned addrspace =
1143    cast<llvm::PointerType>(addr.getPointer()->getType())->getAddressSpace();
1144  llvm::IntegerType *ty =
1145    llvm::IntegerType::get(CGF.getLLVMContext(), AtomicSizeInBits);
1146  return CGF.Builder.CreateBitCast(addr, ty->getPointerTo(addrspace));
1147}
1148
1149Address AtomicInfo::convertToAtomicIntPointer(Address Addr) const {
1150  llvm::Type *Ty = Addr.getElementType();
1151  uint64_t SourceSizeInBits = CGF.CGM.getDataLayout().getTypeSizeInBits(Ty);
1152  if (SourceSizeInBits != AtomicSizeInBits) {
1153    Address Tmp = CreateTempAlloca();
1154    CGF.Builder.CreateMemCpy(Tmp, Addr,
1155                             std::min(AtomicSizeInBits, SourceSizeInBits) / 8);
1156    Addr = Tmp;
1157  }
1158
1159  return emitCastToAtomicIntPointer(Addr);
1160}
1161
1162RValue AtomicInfo::convertAtomicTempToRValue(Address addr,
1163                                             AggValueSlot resultSlot,
1164                                             SourceLocation loc,
1165                                             bool asValue) const {
1166  if (LVal.isSimple()) {
1167    if (EvaluationKind == TEK_Aggregate)
1168      return resultSlot.asRValue();
1169
1170    // Drill into the padding structure if we have one.
1171    if (hasPadding())
1172      addr = CGF.Builder.CreateStructGEP(addr, 0, CharUnits());
1173
1174    // Otherwise, just convert the temporary to an r-value using the
1175    // normal conversion routine.
1176    return CGF.convertTempToRValue(addr, getValueType(), loc);
1177  }
1178  if (!asValue)
1179    // Get RValue from temp memory as atomic for non-simple lvalues
1180    return RValue::get(CGF.Builder.CreateLoad(addr));
1181  if (LVal.isBitField())
1182    return CGF.EmitLoadOfBitfieldLValue(
1183        LValue::MakeBitfield(addr, LVal.getBitFieldInfo(), LVal.getType(),
1184                             LVal.getAlignmentSource()));
1185  if (LVal.isVectorElt())
1186    return CGF.EmitLoadOfLValue(
1187        LValue::MakeVectorElt(addr, LVal.getVectorIdx(), LVal.getType(),
1188                              LVal.getAlignmentSource()), loc);
1189  assert(LVal.isExtVectorElt());
1190  return CGF.EmitLoadOfExtVectorElementLValue(LValue::MakeExtVectorElt(
1191      addr, LVal.getExtVectorElts(), LVal.getType(),
1192      LVal.getAlignmentSource()));
1193}
1194
1195RValue AtomicInfo::ConvertIntToValueOrAtomic(llvm::Value *IntVal,
1196                                             AggValueSlot ResultSlot,
1197                                             SourceLocation Loc,
1198                                             bool AsValue) const {
1199  // Try not to in some easy cases.
1200  assert(IntVal->getType()->isIntegerTy() && "Expected integer value");
1201  if (getEvaluationKind() == TEK_Scalar &&
1202      (((!LVal.isBitField() ||
1203         LVal.getBitFieldInfo().Size == ValueSizeInBits) &&
1204        !hasPadding()) ||
1205       !AsValue)) {
1206    auto *ValTy = AsValue
1207                      ? CGF.ConvertTypeForMem(ValueTy)
1208                      : getAtomicAddress().getType()->getPointerElementType();
1209    if (ValTy->isIntegerTy()) {
1210      assert(IntVal->getType() == ValTy && "Different integer types.");
1211      return RValue::get(CGF.EmitFromMemory(IntVal, ValueTy));
1212    } else if (ValTy->isPointerTy())
1213      return RValue::get(CGF.Builder.CreateIntToPtr(IntVal, ValTy));
1214    else if (llvm::CastInst::isBitCastable(IntVal->getType(), ValTy))
1215      return RValue::get(CGF.Builder.CreateBitCast(IntVal, ValTy));
1216  }
1217
1218  // Create a temporary.  This needs to be big enough to hold the
1219  // atomic integer.
1220  Address Temp = Address::invalid();
1221  bool TempIsVolatile = false;
1222  if (AsValue && getEvaluationKind() == TEK_Aggregate) {
1223    assert(!ResultSlot.isIgnored());
1224    Temp = ResultSlot.getAddress();
1225    TempIsVolatile = ResultSlot.isVolatile();
1226  } else {
1227    Temp = CreateTempAlloca();
1228  }
1229
1230  // Slam the integer into the temporary.
1231  Address CastTemp = emitCastToAtomicIntPointer(Temp);
1232  CGF.Builder.CreateStore(IntVal, CastTemp)
1233      ->setVolatile(TempIsVolatile);
1234
1235  return convertAtomicTempToRValue(Temp, ResultSlot, Loc, AsValue);
1236}
1237
1238void AtomicInfo::EmitAtomicLoadLibcall(llvm::Value *AddForLoaded,
1239                                       llvm::AtomicOrdering AO, bool) {
1240  // void __atomic_load(size_t size, void *mem, void *return, int order);
1241  CallArgList Args;
1242  Args.add(RValue::get(getAtomicSizeValue()), CGF.getContext().getSizeType());
1243  Args.add(RValue::get(CGF.EmitCastToVoidPtr(getAtomicPointer())),
1244           CGF.getContext().VoidPtrTy);
1245  Args.add(RValue::get(CGF.EmitCastToVoidPtr(AddForLoaded)),
1246           CGF.getContext().VoidPtrTy);
1247  Args.add(
1248      RValue::get(llvm::ConstantInt::get(CGF.IntTy, (int)llvm::toCABI(AO))),
1249      CGF.getContext().IntTy);
1250  emitAtomicLibcall(CGF, "__atomic_load", CGF.getContext().VoidTy, Args);
1251}
1252
1253llvm::Value *AtomicInfo::EmitAtomicLoadOp(llvm::AtomicOrdering AO,
1254                                          bool IsVolatile) {
1255  // Okay, we're doing this natively.
1256  Address Addr = getAtomicAddressAsAtomicIntPointer();
1257  llvm::LoadInst *Load = CGF.Builder.CreateLoad(Addr, "atomic-load");
1258  Load->setAtomic(AO);
1259
1260  // Other decoration.
1261  if (IsVolatile)
1262    Load->setVolatile(true);
1263  if (LVal.getTBAAInfo())
1264    CGF.CGM.DecorateInstructionWithTBAA(Load, LVal.getTBAAInfo());
1265  return Load;
1266}
1267
1268/// An LValue is a candidate for having its loads and stores be made atomic if
1269/// we are operating under /volatile:ms *and* the LValue itself is volatile and
1270/// performing such an operation can be performed without a libcall.
1271bool CodeGenFunction::LValueIsSuitableForInlineAtomic(LValue LV) {
1272  if (!CGM.getCodeGenOpts().MSVolatile) return false;
1273  AtomicInfo AI(*this, LV);
1274  bool IsVolatile = LV.isVolatile() || hasVolatileMember(LV.getType());
1275  // An atomic is inline if we don't need to use a libcall.
1276  bool AtomicIsInline = !AI.shouldUseLibcall();
1277  // MSVC doesn't seem to do this for types wider than a pointer.
1278  if (getContext().getTypeSize(LV.getType()) >
1279      getContext().getTypeSize(getContext().getIntPtrType()))
1280    return false;
1281  return IsVolatile && AtomicIsInline;
1282}
1283
1284RValue CodeGenFunction::EmitAtomicLoad(LValue LV, SourceLocation SL,
1285                                       AggValueSlot Slot) {
1286  llvm::AtomicOrdering AO;
1287  bool IsVolatile = LV.isVolatileQualified();
1288  if (LV.getType()->isAtomicType()) {
1289    AO = llvm::AtomicOrdering::SequentiallyConsistent;
1290  } else {
1291    AO = llvm::AtomicOrdering::Acquire;
1292    IsVolatile = true;
1293  }
1294  return EmitAtomicLoad(LV, SL, AO, IsVolatile, Slot);
1295}
1296
1297RValue AtomicInfo::EmitAtomicLoad(AggValueSlot ResultSlot, SourceLocation Loc,
1298                                  bool AsValue, llvm::AtomicOrdering AO,
1299                                  bool IsVolatile) {
1300  // Check whether we should use a library call.
1301  if (shouldUseLibcall()) {
1302    Address TempAddr = Address::invalid();
1303    if (LVal.isSimple() && !ResultSlot.isIgnored()) {
1304      assert(getEvaluationKind() == TEK_Aggregate);
1305      TempAddr = ResultSlot.getAddress();
1306    } else
1307      TempAddr = CreateTempAlloca();
1308
1309    EmitAtomicLoadLibcall(TempAddr.getPointer(), AO, IsVolatile);
1310
1311    // Okay, turn that back into the original value or whole atomic (for
1312    // non-simple lvalues) type.
1313    return convertAtomicTempToRValue(TempAddr, ResultSlot, Loc, AsValue);
1314  }
1315
1316  // Okay, we're doing this natively.
1317  auto *Load = EmitAtomicLoadOp(AO, IsVolatile);
1318
1319  // If we're ignoring an aggregate return, don't do anything.
1320  if (getEvaluationKind() == TEK_Aggregate && ResultSlot.isIgnored())
1321    return RValue::getAggregate(Address::invalid(), false);
1322
1323  // Okay, turn that back into the original value or atomic (for non-simple
1324  // lvalues) type.
1325  return ConvertIntToValueOrAtomic(Load, ResultSlot, Loc, AsValue);
1326}
1327
1328/// Emit a load from an l-value of atomic type.  Note that the r-value
1329/// we produce is an r-value of the atomic *value* type.
1330RValue CodeGenFunction::EmitAtomicLoad(LValue src, SourceLocation loc,
1331                                       llvm::AtomicOrdering AO, bool IsVolatile,
1332                                       AggValueSlot resultSlot) {
1333  AtomicInfo Atomics(*this, src);
1334  return Atomics.EmitAtomicLoad(resultSlot, loc, /*AsValue=*/true, AO,
1335                                IsVolatile);
1336}
1337
1338/// Copy an r-value into memory as part of storing to an atomic type.
1339/// This needs to create a bit-pattern suitable for atomic operations.
1340void AtomicInfo::emitCopyIntoMemory(RValue rvalue) const {
1341  assert(LVal.isSimple());
1342  // If we have an r-value, the rvalue should be of the atomic type,
1343  // which means that the caller is responsible for having zeroed
1344  // any padding.  Just do an aggregate copy of that type.
1345  if (rvalue.isAggregate()) {
1346    CGF.EmitAggregateCopy(getAtomicAddress(),
1347                          rvalue.getAggregateAddress(),
1348                          getAtomicType(),
1349                          (rvalue.isVolatileQualified()
1350                           || LVal.isVolatileQualified()));
1351    return;
1352  }
1353
1354  // Okay, otherwise we're copying stuff.
1355
1356  // Zero out the buffer if necessary.
1357  emitMemSetZeroIfNecessary();
1358
1359  // Drill past the padding if present.
1360  LValue TempLVal = projectValue();
1361
1362  // Okay, store the rvalue in.
1363  if (rvalue.isScalar()) {
1364    CGF.EmitStoreOfScalar(rvalue.getScalarVal(), TempLVal, /*init*/ true);
1365  } else {
1366    CGF.EmitStoreOfComplex(rvalue.getComplexVal(), TempLVal, /*init*/ true);
1367  }
1368}
1369
1370
1371/// Materialize an r-value into memory for the purposes of storing it
1372/// to an atomic type.
1373Address AtomicInfo::materializeRValue(RValue rvalue) const {
1374  // Aggregate r-values are already in memory, and EmitAtomicStore
1375  // requires them to be values of the atomic type.
1376  if (rvalue.isAggregate())
1377    return rvalue.getAggregateAddress();
1378
1379  // Otherwise, make a temporary and materialize into it.
1380  LValue TempLV = CGF.MakeAddrLValue(CreateTempAlloca(), getAtomicType());
1381  AtomicInfo Atomics(CGF, TempLV);
1382  Atomics.emitCopyIntoMemory(rvalue);
1383  return TempLV.getAddress();
1384}
1385
1386llvm::Value *AtomicInfo::convertRValueToInt(RValue RVal) const {
1387  // If we've got a scalar value of the right size, try to avoid going
1388  // through memory.
1389  if (RVal.isScalar() && (!hasPadding() || !LVal.isSimple())) {
1390    llvm::Value *Value = RVal.getScalarVal();
1391    if (isa<llvm::IntegerType>(Value->getType()))
1392      return CGF.EmitToMemory(Value, ValueTy);
1393    else {
1394      llvm::IntegerType *InputIntTy = llvm::IntegerType::get(
1395          CGF.getLLVMContext(),
1396          LVal.isSimple() ? getValueSizeInBits() : getAtomicSizeInBits());
1397      if (isa<llvm::PointerType>(Value->getType()))
1398        return CGF.Builder.CreatePtrToInt(Value, InputIntTy);
1399      else if (llvm::BitCastInst::isBitCastable(Value->getType(), InputIntTy))
1400        return CGF.Builder.CreateBitCast(Value, InputIntTy);
1401    }
1402  }
1403  // Otherwise, we need to go through memory.
1404  // Put the r-value in memory.
1405  Address Addr = materializeRValue(RVal);
1406
1407  // Cast the temporary to the atomic int type and pull a value out.
1408  Addr = emitCastToAtomicIntPointer(Addr);
1409  return CGF.Builder.CreateLoad(Addr);
1410}
1411
1412std::pair<llvm::Value *, llvm::Value *> AtomicInfo::EmitAtomicCompareExchangeOp(
1413    llvm::Value *ExpectedVal, llvm::Value *DesiredVal,
1414    llvm::AtomicOrdering Success, llvm::AtomicOrdering Failure, bool IsWeak) {
1415  // Do the atomic store.
1416  Address Addr = getAtomicAddressAsAtomicIntPointer();
1417  auto *Inst = CGF.Builder.CreateAtomicCmpXchg(Addr.getPointer(),
1418                                               ExpectedVal, DesiredVal,
1419                                               Success, Failure);
1420  // Other decoration.
1421  Inst->setVolatile(LVal.isVolatileQualified());
1422  Inst->setWeak(IsWeak);
1423
1424  // Okay, turn that back into the original value type.
1425  auto *PreviousVal = CGF.Builder.CreateExtractValue(Inst, /*Idxs=*/0);
1426  auto *SuccessFailureVal = CGF.Builder.CreateExtractValue(Inst, /*Idxs=*/1);
1427  return std::make_pair(PreviousVal, SuccessFailureVal);
1428}
1429
1430llvm::Value *
1431AtomicInfo::EmitAtomicCompareExchangeLibcall(llvm::Value *ExpectedAddr,
1432                                             llvm::Value *DesiredAddr,
1433                                             llvm::AtomicOrdering Success,
1434                                             llvm::AtomicOrdering Failure) {
1435  // bool __atomic_compare_exchange(size_t size, void *obj, void *expected,
1436  // void *desired, int success, int failure);
1437  CallArgList Args;
1438  Args.add(RValue::get(getAtomicSizeValue()), CGF.getContext().getSizeType());
1439  Args.add(RValue::get(CGF.EmitCastToVoidPtr(getAtomicPointer())),
1440           CGF.getContext().VoidPtrTy);
1441  Args.add(RValue::get(CGF.EmitCastToVoidPtr(ExpectedAddr)),
1442           CGF.getContext().VoidPtrTy);
1443  Args.add(RValue::get(CGF.EmitCastToVoidPtr(DesiredAddr)),
1444           CGF.getContext().VoidPtrTy);
1445  Args.add(RValue::get(
1446               llvm::ConstantInt::get(CGF.IntTy, (int)llvm::toCABI(Success))),
1447           CGF.getContext().IntTy);
1448  Args.add(RValue::get(
1449               llvm::ConstantInt::get(CGF.IntTy, (int)llvm::toCABI(Failure))),
1450           CGF.getContext().IntTy);
1451  auto SuccessFailureRVal = emitAtomicLibcall(CGF, "__atomic_compare_exchange",
1452                                              CGF.getContext().BoolTy, Args);
1453
1454  return SuccessFailureRVal.getScalarVal();
1455}
1456
1457std::pair<RValue, llvm::Value *> AtomicInfo::EmitAtomicCompareExchange(
1458    RValue Expected, RValue Desired, llvm::AtomicOrdering Success,
1459    llvm::AtomicOrdering Failure, bool IsWeak) {
1460  if (isStrongerThan(Failure, Success))
1461    // Don't assert on undefined behavior "failure argument shall be no stronger
1462    // than the success argument".
1463    Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(Success);
1464
1465  // Check whether we should use a library call.
1466  if (shouldUseLibcall()) {
1467    // Produce a source address.
1468    Address ExpectedAddr = materializeRValue(Expected);
1469    Address DesiredAddr = materializeRValue(Desired);
1470    auto *Res = EmitAtomicCompareExchangeLibcall(ExpectedAddr.getPointer(),
1471                                                 DesiredAddr.getPointer(),
1472                                                 Success, Failure);
1473    return std::make_pair(
1474        convertAtomicTempToRValue(ExpectedAddr, AggValueSlot::ignored(),
1475                                  SourceLocation(), /*AsValue=*/false),
1476        Res);
1477  }
1478
1479  // If we've got a scalar value of the right size, try to avoid going
1480  // through memory.
1481  auto *ExpectedVal = convertRValueToInt(Expected);
1482  auto *DesiredVal = convertRValueToInt(Desired);
1483  auto Res = EmitAtomicCompareExchangeOp(ExpectedVal, DesiredVal, Success,
1484                                         Failure, IsWeak);
1485  return std::make_pair(
1486      ConvertIntToValueOrAtomic(Res.first, AggValueSlot::ignored(),
1487                                SourceLocation(), /*AsValue=*/false),
1488      Res.second);
1489}
1490
1491static void
1492EmitAtomicUpdateValue(CodeGenFunction &CGF, AtomicInfo &Atomics, RValue OldRVal,
1493                      const llvm::function_ref<RValue(RValue)> &UpdateOp,
1494                      Address DesiredAddr) {
1495  RValue UpRVal;
1496  LValue AtomicLVal = Atomics.getAtomicLValue();
1497  LValue DesiredLVal;
1498  if (AtomicLVal.isSimple()) {
1499    UpRVal = OldRVal;
1500    DesiredLVal = CGF.MakeAddrLValue(DesiredAddr, AtomicLVal.getType());
1501  } else {
1502    // Build new lvalue for temp address
1503    Address Ptr = Atomics.materializeRValue(OldRVal);
1504    LValue UpdateLVal;
1505    if (AtomicLVal.isBitField()) {
1506      UpdateLVal =
1507          LValue::MakeBitfield(Ptr, AtomicLVal.getBitFieldInfo(),
1508                               AtomicLVal.getType(),
1509                               AtomicLVal.getAlignmentSource());
1510      DesiredLVal =
1511          LValue::MakeBitfield(DesiredAddr, AtomicLVal.getBitFieldInfo(),
1512                               AtomicLVal.getType(),
1513                               AtomicLVal.getAlignmentSource());
1514    } else if (AtomicLVal.isVectorElt()) {
1515      UpdateLVal = LValue::MakeVectorElt(Ptr, AtomicLVal.getVectorIdx(),
1516                                         AtomicLVal.getType(),
1517                                         AtomicLVal.getAlignmentSource());
1518      DesiredLVal = LValue::MakeVectorElt(
1519          DesiredAddr, AtomicLVal.getVectorIdx(), AtomicLVal.getType(),
1520          AtomicLVal.getAlignmentSource());
1521    } else {
1522      assert(AtomicLVal.isExtVectorElt());
1523      UpdateLVal = LValue::MakeExtVectorElt(Ptr, AtomicLVal.getExtVectorElts(),
1524                                            AtomicLVal.getType(),
1525                                            AtomicLVal.getAlignmentSource());
1526      DesiredLVal = LValue::MakeExtVectorElt(
1527          DesiredAddr, AtomicLVal.getExtVectorElts(), AtomicLVal.getType(),
1528          AtomicLVal.getAlignmentSource());
1529    }
1530    UpdateLVal.setTBAAInfo(AtomicLVal.getTBAAInfo());
1531    DesiredLVal.setTBAAInfo(AtomicLVal.getTBAAInfo());
1532    UpRVal = CGF.EmitLoadOfLValue(UpdateLVal, SourceLocation());
1533  }
1534  // Store new value in the corresponding memory area
1535  RValue NewRVal = UpdateOp(UpRVal);
1536  if (NewRVal.isScalar()) {
1537    CGF.EmitStoreThroughLValue(NewRVal, DesiredLVal);
1538  } else {
1539    assert(NewRVal.isComplex());
1540    CGF.EmitStoreOfComplex(NewRVal.getComplexVal(), DesiredLVal,
1541                           /*isInit=*/false);
1542  }
1543}
1544
1545void AtomicInfo::EmitAtomicUpdateLibcall(
1546    llvm::AtomicOrdering AO, const llvm::function_ref<RValue(RValue)> &UpdateOp,
1547    bool IsVolatile) {
1548  auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO);
1549
1550  Address ExpectedAddr = CreateTempAlloca();
1551
1552  EmitAtomicLoadLibcall(ExpectedAddr.getPointer(), AO, IsVolatile);
1553  auto *ContBB = CGF.createBasicBlock("atomic_cont");
1554  auto *ExitBB = CGF.createBasicBlock("atomic_exit");
1555  CGF.EmitBlock(ContBB);
1556  Address DesiredAddr = CreateTempAlloca();
1557  if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) ||
1558      requiresMemSetZero(getAtomicAddress().getElementType())) {
1559    auto *OldVal = CGF.Builder.CreateLoad(ExpectedAddr);
1560    CGF.Builder.CreateStore(OldVal, DesiredAddr);
1561  }
1562  auto OldRVal = convertAtomicTempToRValue(ExpectedAddr,
1563                                           AggValueSlot::ignored(),
1564                                           SourceLocation(), /*AsValue=*/false);
1565  EmitAtomicUpdateValue(CGF, *this, OldRVal, UpdateOp, DesiredAddr);
1566  auto *Res =
1567      EmitAtomicCompareExchangeLibcall(ExpectedAddr.getPointer(),
1568                                       DesiredAddr.getPointer(),
1569                                       AO, Failure);
1570  CGF.Builder.CreateCondBr(Res, ExitBB, ContBB);
1571  CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
1572}
1573
1574void AtomicInfo::EmitAtomicUpdateOp(
1575    llvm::AtomicOrdering AO, const llvm::function_ref<RValue(RValue)> &UpdateOp,
1576    bool IsVolatile) {
1577  auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO);
1578
1579  // Do the atomic load.
1580  auto *OldVal = EmitAtomicLoadOp(AO, IsVolatile);
1581  // For non-simple lvalues perform compare-and-swap procedure.
1582  auto *ContBB = CGF.createBasicBlock("atomic_cont");
1583  auto *ExitBB = CGF.createBasicBlock("atomic_exit");
1584  auto *CurBB = CGF.Builder.GetInsertBlock();
1585  CGF.EmitBlock(ContBB);
1586  llvm::PHINode *PHI = CGF.Builder.CreatePHI(OldVal->getType(),
1587                                             /*NumReservedValues=*/2);
1588  PHI->addIncoming(OldVal, CurBB);
1589  Address NewAtomicAddr = CreateTempAlloca();
1590  Address NewAtomicIntAddr = emitCastToAtomicIntPointer(NewAtomicAddr);
1591  if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) ||
1592      requiresMemSetZero(getAtomicAddress().getElementType())) {
1593    CGF.Builder.CreateStore(PHI, NewAtomicIntAddr);
1594  }
1595  auto OldRVal = ConvertIntToValueOrAtomic(PHI, AggValueSlot::ignored(),
1596                                           SourceLocation(), /*AsValue=*/false);
1597  EmitAtomicUpdateValue(CGF, *this, OldRVal, UpdateOp, NewAtomicAddr);
1598  auto *DesiredVal = CGF.Builder.CreateLoad(NewAtomicIntAddr);
1599  // Try to write new value using cmpxchg operation
1600  auto Res = EmitAtomicCompareExchangeOp(PHI, DesiredVal, AO, Failure);
1601  PHI->addIncoming(Res.first, CGF.Builder.GetInsertBlock());
1602  CGF.Builder.CreateCondBr(Res.second, ExitBB, ContBB);
1603  CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
1604}
1605
1606static void EmitAtomicUpdateValue(CodeGenFunction &CGF, AtomicInfo &Atomics,
1607                                  RValue UpdateRVal, Address DesiredAddr) {
1608  LValue AtomicLVal = Atomics.getAtomicLValue();
1609  LValue DesiredLVal;
1610  // Build new lvalue for temp address
1611  if (AtomicLVal.isBitField()) {
1612    DesiredLVal =
1613        LValue::MakeBitfield(DesiredAddr, AtomicLVal.getBitFieldInfo(),
1614                             AtomicLVal.getType(),
1615                             AtomicLVal.getAlignmentSource());
1616  } else if (AtomicLVal.isVectorElt()) {
1617    DesiredLVal =
1618        LValue::MakeVectorElt(DesiredAddr, AtomicLVal.getVectorIdx(),
1619                              AtomicLVal.getType(),
1620                              AtomicLVal.getAlignmentSource());
1621  } else {
1622    assert(AtomicLVal.isExtVectorElt());
1623    DesiredLVal = LValue::MakeExtVectorElt(
1624        DesiredAddr, AtomicLVal.getExtVectorElts(), AtomicLVal.getType(),
1625        AtomicLVal.getAlignmentSource());
1626  }
1627  DesiredLVal.setTBAAInfo(AtomicLVal.getTBAAInfo());
1628  // Store new value in the corresponding memory area
1629  assert(UpdateRVal.isScalar());
1630  CGF.EmitStoreThroughLValue(UpdateRVal, DesiredLVal);
1631}
1632
1633void AtomicInfo::EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO,
1634                                         RValue UpdateRVal, bool IsVolatile) {
1635  auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO);
1636
1637  Address ExpectedAddr = CreateTempAlloca();
1638
1639  EmitAtomicLoadLibcall(ExpectedAddr.getPointer(), AO, IsVolatile);
1640  auto *ContBB = CGF.createBasicBlock("atomic_cont");
1641  auto *ExitBB = CGF.createBasicBlock("atomic_exit");
1642  CGF.EmitBlock(ContBB);
1643  Address DesiredAddr = CreateTempAlloca();
1644  if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) ||
1645      requiresMemSetZero(getAtomicAddress().getElementType())) {
1646    auto *OldVal = CGF.Builder.CreateLoad(ExpectedAddr);
1647    CGF.Builder.CreateStore(OldVal, DesiredAddr);
1648  }
1649  EmitAtomicUpdateValue(CGF, *this, UpdateRVal, DesiredAddr);
1650  auto *Res =
1651      EmitAtomicCompareExchangeLibcall(ExpectedAddr.getPointer(),
1652                                       DesiredAddr.getPointer(),
1653                                       AO, Failure);
1654  CGF.Builder.CreateCondBr(Res, ExitBB, ContBB);
1655  CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
1656}
1657
1658void AtomicInfo::EmitAtomicUpdateOp(llvm::AtomicOrdering AO, RValue UpdateRVal,
1659                                    bool IsVolatile) {
1660  auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO);
1661
1662  // Do the atomic load.
1663  auto *OldVal = EmitAtomicLoadOp(AO, IsVolatile);
1664  // For non-simple lvalues perform compare-and-swap procedure.
1665  auto *ContBB = CGF.createBasicBlock("atomic_cont");
1666  auto *ExitBB = CGF.createBasicBlock("atomic_exit");
1667  auto *CurBB = CGF.Builder.GetInsertBlock();
1668  CGF.EmitBlock(ContBB);
1669  llvm::PHINode *PHI = CGF.Builder.CreatePHI(OldVal->getType(),
1670                                             /*NumReservedValues=*/2);
1671  PHI->addIncoming(OldVal, CurBB);
1672  Address NewAtomicAddr = CreateTempAlloca();
1673  Address NewAtomicIntAddr = emitCastToAtomicIntPointer(NewAtomicAddr);
1674  if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) ||
1675      requiresMemSetZero(getAtomicAddress().getElementType())) {
1676    CGF.Builder.CreateStore(PHI, NewAtomicIntAddr);
1677  }
1678  EmitAtomicUpdateValue(CGF, *this, UpdateRVal, NewAtomicAddr);
1679  auto *DesiredVal = CGF.Builder.CreateLoad(NewAtomicIntAddr);
1680  // Try to write new value using cmpxchg operation
1681  auto Res = EmitAtomicCompareExchangeOp(PHI, DesiredVal, AO, Failure);
1682  PHI->addIncoming(Res.first, CGF.Builder.GetInsertBlock());
1683  CGF.Builder.CreateCondBr(Res.second, ExitBB, ContBB);
1684  CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
1685}
1686
1687void AtomicInfo::EmitAtomicUpdate(
1688    llvm::AtomicOrdering AO, const llvm::function_ref<RValue(RValue)> &UpdateOp,
1689    bool IsVolatile) {
1690  if (shouldUseLibcall()) {
1691    EmitAtomicUpdateLibcall(AO, UpdateOp, IsVolatile);
1692  } else {
1693    EmitAtomicUpdateOp(AO, UpdateOp, IsVolatile);
1694  }
1695}
1696
1697void AtomicInfo::EmitAtomicUpdate(llvm::AtomicOrdering AO, RValue UpdateRVal,
1698                                  bool IsVolatile) {
1699  if (shouldUseLibcall()) {
1700    EmitAtomicUpdateLibcall(AO, UpdateRVal, IsVolatile);
1701  } else {
1702    EmitAtomicUpdateOp(AO, UpdateRVal, IsVolatile);
1703  }
1704}
1705
1706void CodeGenFunction::EmitAtomicStore(RValue rvalue, LValue lvalue,
1707                                      bool isInit) {
1708  bool IsVolatile = lvalue.isVolatileQualified();
1709  llvm::AtomicOrdering AO;
1710  if (lvalue.getType()->isAtomicType()) {
1711    AO = llvm::AtomicOrdering::SequentiallyConsistent;
1712  } else {
1713    AO = llvm::AtomicOrdering::Release;
1714    IsVolatile = true;
1715  }
1716  return EmitAtomicStore(rvalue, lvalue, AO, IsVolatile, isInit);
1717}
1718
1719/// Emit a store to an l-value of atomic type.
1720///
1721/// Note that the r-value is expected to be an r-value *of the atomic
1722/// type*; this means that for aggregate r-values, it should include
1723/// storage for any padding that was necessary.
1724void CodeGenFunction::EmitAtomicStore(RValue rvalue, LValue dest,
1725                                      llvm::AtomicOrdering AO, bool IsVolatile,
1726                                      bool isInit) {
1727  // If this is an aggregate r-value, it should agree in type except
1728  // maybe for address-space qualification.
1729  assert(!rvalue.isAggregate() ||
1730         rvalue.getAggregateAddress().getElementType()
1731           == dest.getAddress().getElementType());
1732
1733  AtomicInfo atomics(*this, dest);
1734  LValue LVal = atomics.getAtomicLValue();
1735
1736  // If this is an initialization, just put the value there normally.
1737  if (LVal.isSimple()) {
1738    if (isInit) {
1739      atomics.emitCopyIntoMemory(rvalue);
1740      return;
1741    }
1742
1743    // Check whether we should use a library call.
1744    if (atomics.shouldUseLibcall()) {
1745      // Produce a source address.
1746      Address srcAddr = atomics.materializeRValue(rvalue);
1747
1748      // void __atomic_store(size_t size, void *mem, void *val, int order)
1749      CallArgList args;
1750      args.add(RValue::get(atomics.getAtomicSizeValue()),
1751               getContext().getSizeType());
1752      args.add(RValue::get(EmitCastToVoidPtr(atomics.getAtomicPointer())),
1753               getContext().VoidPtrTy);
1754      args.add(RValue::get(EmitCastToVoidPtr(srcAddr.getPointer())),
1755               getContext().VoidPtrTy);
1756      args.add(
1757          RValue::get(llvm::ConstantInt::get(IntTy, (int)llvm::toCABI(AO))),
1758          getContext().IntTy);
1759      emitAtomicLibcall(*this, "__atomic_store", getContext().VoidTy, args);
1760      return;
1761    }
1762
1763    // Okay, we're doing this natively.
1764    llvm::Value *intValue = atomics.convertRValueToInt(rvalue);
1765
1766    // Do the atomic store.
1767    Address addr =
1768        atomics.emitCastToAtomicIntPointer(atomics.getAtomicAddress());
1769    intValue = Builder.CreateIntCast(
1770        intValue, addr.getElementType(), /*isSigned=*/false);
1771    llvm::StoreInst *store = Builder.CreateStore(intValue, addr);
1772
1773    // Initializations don't need to be atomic.
1774    if (!isInit)
1775      store->setAtomic(AO);
1776
1777    // Other decoration.
1778    if (IsVolatile)
1779      store->setVolatile(true);
1780    if (dest.getTBAAInfo())
1781      CGM.DecorateInstructionWithTBAA(store, dest.getTBAAInfo());
1782    return;
1783  }
1784
1785  // Emit simple atomic update operation.
1786  atomics.EmitAtomicUpdate(AO, rvalue, IsVolatile);
1787}
1788
1789/// Emit a compare-and-exchange op for atomic type.
1790///
1791std::pair<RValue, llvm::Value *> CodeGenFunction::EmitAtomicCompareExchange(
1792    LValue Obj, RValue Expected, RValue Desired, SourceLocation Loc,
1793    llvm::AtomicOrdering Success, llvm::AtomicOrdering Failure, bool IsWeak,
1794    AggValueSlot Slot) {
1795  // If this is an aggregate r-value, it should agree in type except
1796  // maybe for address-space qualification.
1797  assert(!Expected.isAggregate() ||
1798         Expected.getAggregateAddress().getElementType() ==
1799             Obj.getAddress().getElementType());
1800  assert(!Desired.isAggregate() ||
1801         Desired.getAggregateAddress().getElementType() ==
1802             Obj.getAddress().getElementType());
1803  AtomicInfo Atomics(*this, Obj);
1804
1805  return Atomics.EmitAtomicCompareExchange(Expected, Desired, Success, Failure,
1806                                           IsWeak);
1807}
1808
1809void CodeGenFunction::EmitAtomicUpdate(
1810    LValue LVal, llvm::AtomicOrdering AO,
1811    const llvm::function_ref<RValue(RValue)> &UpdateOp, bool IsVolatile) {
1812  AtomicInfo Atomics(*this, LVal);
1813  Atomics.EmitAtomicUpdate(AO, UpdateOp, IsVolatile);
1814}
1815
1816void CodeGenFunction::EmitAtomicInit(Expr *init, LValue dest) {
1817  AtomicInfo atomics(*this, dest);
1818
1819  switch (atomics.getEvaluationKind()) {
1820  case TEK_Scalar: {
1821    llvm::Value *value = EmitScalarExpr(init);
1822    atomics.emitCopyIntoMemory(RValue::get(value));
1823    return;
1824  }
1825
1826  case TEK_Complex: {
1827    ComplexPairTy value = EmitComplexExpr(init);
1828    atomics.emitCopyIntoMemory(RValue::getComplex(value));
1829    return;
1830  }
1831
1832  case TEK_Aggregate: {
1833    // Fix up the destination if the initializer isn't an expression
1834    // of atomic type.
1835    bool Zeroed = false;
1836    if (!init->getType()->isAtomicType()) {
1837      Zeroed = atomics.emitMemSetZeroIfNecessary();
1838      dest = atomics.projectValue();
1839    }
1840
1841    // Evaluate the expression directly into the destination.
1842    AggValueSlot slot = AggValueSlot::forLValue(dest,
1843                                        AggValueSlot::IsNotDestructed,
1844                                        AggValueSlot::DoesNotNeedGCBarriers,
1845                                        AggValueSlot::IsNotAliased,
1846                                        Zeroed ? AggValueSlot::IsZeroed :
1847                                                 AggValueSlot::IsNotZeroed);
1848
1849    EmitAggExpr(init, slot);
1850    return;
1851  }
1852  }
1853  llvm_unreachable("bad evaluation kind");
1854}
1855