CGExpr.cpp revision 28ad063b279378b19cb0704f977429df366a151e 
1//===--- CGExpr.cpp - Emit LLVM Code from Expressions ---------------------===//
2//
3//                     The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10// This contains code to emit Expr nodes as LLVM code.
11//
12//===----------------------------------------------------------------------===//
13
14#include "CodeGenFunction.h"
15#include "CodeGenModule.h"
16#include "CGCall.h"
17#include "CGCXXABI.h"
18#include "CGDebugInfo.h"
19#include "CGRecordLayout.h"
20#include "CGObjCRuntime.h"
21#include "TargetInfo.h"
22#include "clang/AST/ASTContext.h"
23#include "clang/AST/DeclObjC.h"
24#include "clang/Basic/ConvertUTF.h"
25#include "clang/Frontend/CodeGenOptions.h"
26#include "llvm/Intrinsics.h"
27#include "llvm/LLVMContext.h"
28#include "llvm/Support/MDBuilder.h"
29#include "llvm/Target/TargetData.h"
30using namespace clang;
31using namespace CodeGen;
32
33//===--------------------------------------------------------------------===//
34//                        Miscellaneous Helper Methods
35//===--------------------------------------------------------------------===//
36
37llvm::Value *CodeGenFunction::EmitCastToVoidPtr(llvm::Value *value) {
38  unsigned addressSpace =
39    cast<llvm::PointerType>(value->getType())->getAddressSpace();
40
41  llvm::PointerType *destType = Int8PtrTy;
42  if (addressSpace)
43    destType = llvm::Type::getInt8PtrTy(getLLVMContext(), addressSpace);
44
45  if (value->getType() == destType) return value;
46  return Builder.CreateBitCast(value, destType);
47}
48
49/// CreateTempAlloca - This creates a alloca and inserts it into the entry
50/// block.
51llvm::AllocaInst *CodeGenFunction::CreateTempAlloca(llvm::Type *Ty,
52                                                    const Twine &Name) {
53  if (!Builder.isNamePreserving())
54    return new llvm::AllocaInst(Ty, 0, "", AllocaInsertPt);
55  return new llvm::AllocaInst(Ty, 0, Name, AllocaInsertPt);
56}
57
58void CodeGenFunction::InitTempAlloca(llvm::AllocaInst *Var,
59                                     llvm::Value *Init) {
60  llvm::StoreInst *Store = new llvm::StoreInst(Init, Var);
61  llvm::BasicBlock *Block = AllocaInsertPt->getParent();
62  Block->getInstList().insertAfter(&*AllocaInsertPt, Store);
63}
64
65llvm::AllocaInst *CodeGenFunction::CreateIRTemp(QualType Ty,
66                                                const Twine &Name) {
67  llvm::AllocaInst *Alloc = CreateTempAlloca(ConvertType(Ty), Name);
68  // FIXME: Should we prefer the preferred type alignment here?
69  CharUnits Align = getContext().getTypeAlignInChars(Ty);
70  Alloc->setAlignment(Align.getQuantity());
71  return Alloc;
72}
73
74llvm::AllocaInst *CodeGenFunction::CreateMemTemp(QualType Ty,
75                                                 const Twine &Name) {
76  llvm::AllocaInst *Alloc = CreateTempAlloca(ConvertTypeForMem(Ty), Name);
77  // FIXME: Should we prefer the preferred type alignment here?
78  CharUnits Align = getContext().getTypeAlignInChars(Ty);
79  Alloc->setAlignment(Align.getQuantity());
80  return Alloc;
81}
82
83/// EvaluateExprAsBool - Perform the usual unary conversions on the specified
84/// expression and compare the result against zero, returning an Int1Ty value.
85llvm::Value *CodeGenFunction::EvaluateExprAsBool(const Expr *E) {
86  if (const MemberPointerType *MPT = E->getType()->getAs<MemberPointerType>()) {
87    llvm::Value *MemPtr = EmitScalarExpr(E);
88    return CGM.getCXXABI().EmitMemberPointerIsNotNull(*this, MemPtr, MPT);
89  }
90
91  QualType BoolTy = getContext().BoolTy;
92  if (!E->getType()->isAnyComplexType())
93    return EmitScalarConversion(EmitScalarExpr(E), E->getType(), BoolTy);
94
95  return EmitComplexToScalarConversion(EmitComplexExpr(E), E->getType(),BoolTy);
96}
97
98/// EmitIgnoredExpr - Emit code to compute the specified expression,
99/// ignoring the result.
100void CodeGenFunction::EmitIgnoredExpr(const Expr *E) {
101  if (E->isRValue())
102    return (void) EmitAnyExpr(E, AggValueSlot::ignored(), true);
103
104  // Just emit it as an l-value and drop the result.
105  EmitLValue(E);
106}
107
108/// EmitAnyExpr - Emit code to compute the specified expression which
109/// can have any type.  The result is returned as an RValue struct.
110/// If this is an aggregate expression, AggSlot indicates where the
111/// result should be returned.
112RValue CodeGenFunction::EmitAnyExpr(const Expr *E, AggValueSlot AggSlot,
113                                    bool IgnoreResult) {
114  if (!hasAggregateLLVMType(E->getType()))
115    return RValue::get(EmitScalarExpr(E, IgnoreResult));
116  else if (E->getType()->isAnyComplexType())
117    return RValue::getComplex(EmitComplexExpr(E, IgnoreResult, IgnoreResult));
118
119  EmitAggExpr(E, AggSlot, IgnoreResult);
120  return AggSlot.asRValue();
121}
122
123/// EmitAnyExprToTemp - Similary to EmitAnyExpr(), however, the result will
124/// always be accessible even if no aggregate location is provided.
125RValue CodeGenFunction::EmitAnyExprToTemp(const Expr *E) {
126  AggValueSlot AggSlot = AggValueSlot::ignored();
127
128  if (hasAggregateLLVMType(E->getType()) &&
129      !E->getType()->isAnyComplexType())
130    AggSlot = CreateAggTemp(E->getType(), "agg.tmp");
131  return EmitAnyExpr(E, AggSlot);
132}
133
134/// EmitAnyExprToMem - Evaluate an expression into a given memory
135/// location.
136void CodeGenFunction::EmitAnyExprToMem(const Expr *E,
137                                       llvm::Value *Location,
138                                       Qualifiers Quals,
139                                       bool IsInit) {
140  // FIXME: This function should take an LValue as an argument.
141  if (E->getType()->isAnyComplexType()) {
142    EmitComplexExprIntoAddr(E, Location, Quals.hasVolatile());
143  } else if (hasAggregateLLVMType(E->getType())) {
144    CharUnits Alignment = getContext().getTypeAlignInChars(E->getType());
145    EmitAggExpr(E, AggValueSlot::forAddr(Location, Alignment, Quals,
146                                         AggValueSlot::IsDestructed_t(IsInit),
147                                         AggValueSlot::DoesNotNeedGCBarriers,
148                                         AggValueSlot::IsAliased_t(!IsInit)));
149  } else {
150    RValue RV = RValue::get(EmitScalarExpr(E, /*Ignore*/ false));
151    LValue LV = MakeAddrLValue(Location, E->getType());
152    EmitStoreThroughLValue(RV, LV);
153  }
154}
155
156namespace {
157/// \brief An adjustment to be made to the temporary created when emitting a
158/// reference binding, which accesses a particular subobject of that temporary.
159  struct SubobjectAdjustment {
160    enum {
161      DerivedToBaseAdjustment,
162      FieldAdjustment,
163      MemberPointerAdjustment
164    } Kind;
165
166    union {
167      struct {
168        const CastExpr *BasePath;
169        const CXXRecordDecl *DerivedClass;
170      } DerivedToBase;
171
172      FieldDecl *Field;
173
174      struct {
175        const MemberPointerType *MPT;
176        llvm::Value *Ptr;
177      } Ptr;
178    };
179
180    SubobjectAdjustment(const CastExpr *BasePath,
181                        const CXXRecordDecl *DerivedClass)
182      : Kind(DerivedToBaseAdjustment) {
183      DerivedToBase.BasePath = BasePath;
184      DerivedToBase.DerivedClass = DerivedClass;
185    }
186
187    SubobjectAdjustment(FieldDecl *Field)
188      : Kind(FieldAdjustment) {
189      this->Field = Field;
190    }
191
192    SubobjectAdjustment(const MemberPointerType *MPT, llvm::Value *Ptr)
193      : Kind(MemberPointerAdjustment) {
194      this->Ptr.MPT = MPT;
195      this->Ptr.Ptr = Ptr;
196    }
197  };
198}
199
200static llvm::Value *
201CreateReferenceTemporary(CodeGenFunction &CGF, QualType Type,
202                         const NamedDecl *InitializedDecl) {
203  if (const VarDecl *VD = dyn_cast_or_null<VarDecl>(InitializedDecl)) {
204    if (VD->hasGlobalStorage()) {
205      SmallString<256> Name;
206      llvm::raw_svector_ostream Out(Name);
207      CGF.CGM.getCXXABI().getMangleContext().mangleReferenceTemporary(VD, Out);
208      Out.flush();
209
210      llvm::Type *RefTempTy = CGF.ConvertTypeForMem(Type);
211
212      // Create the reference temporary.
213      llvm::GlobalValue *RefTemp =
214        new llvm::GlobalVariable(CGF.CGM.getModule(),
215                                 RefTempTy, /*isConstant=*/false,
216                                 llvm::GlobalValue::InternalLinkage,
217                                 llvm::Constant::getNullValue(RefTempTy),
218                                 Name.str());
219      return RefTemp;
220    }
221  }
222
223  return CGF.CreateMemTemp(Type, "ref.tmp");
224}
225
226static llvm::Value *
227EmitExprForReferenceBinding(CodeGenFunction &CGF, const Expr *E,
228                            llvm::Value *&ReferenceTemporary,
229                            const CXXDestructorDecl *&ReferenceTemporaryDtor,
230                            QualType &ObjCARCReferenceLifetimeType,
231                            const NamedDecl *InitializedDecl) {
232  // Look through single-element init lists that claim to be lvalues. They're
233  // just syntactic wrappers in this case.
234  if (const InitListExpr *ILE = dyn_cast<InitListExpr>(E)) {
235    if (ILE->getNumInits() == 1 && ILE->isGLValue())
236      E = ILE->getInit(0);
237  }
238
239  // Look through expressions for materialized temporaries (for now).
240  if (const MaterializeTemporaryExpr *M
241                                      = dyn_cast<MaterializeTemporaryExpr>(E)) {
242    // Objective-C++ ARC:
243    //   If we are binding a reference to a temporary that has ownership, we
244    //   need to perform retain/release operations on the temporary.
245    if (CGF.getContext().getLangOpts().ObjCAutoRefCount &&
246        E->getType()->isObjCLifetimeType() &&
247        (E->getType().getObjCLifetime() == Qualifiers::OCL_Strong ||
248         E->getType().getObjCLifetime() == Qualifiers::OCL_Weak ||
249         E->getType().getObjCLifetime() == Qualifiers::OCL_Autoreleasing))
250      ObjCARCReferenceLifetimeType = E->getType();
251
252    E = M->GetTemporaryExpr();
253  }
254
255  if (const CXXDefaultArgExpr *DAE = dyn_cast<CXXDefaultArgExpr>(E))
256    E = DAE->getExpr();
257
258  if (const ExprWithCleanups *EWC = dyn_cast<ExprWithCleanups>(E)) {
259    CGF.enterFullExpression(EWC);
260    CodeGenFunction::RunCleanupsScope Scope(CGF);
261
262    return EmitExprForReferenceBinding(CGF, EWC->getSubExpr(),
263                                       ReferenceTemporary,
264                                       ReferenceTemporaryDtor,
265                                       ObjCARCReferenceLifetimeType,
266                                       InitializedDecl);
267  }
268
269  RValue RV;
270  if (E->isGLValue()) {
271    // Emit the expression as an lvalue.
272    LValue LV = CGF.EmitLValue(E);
273
274    if (LV.isSimple())
275      return LV.getAddress();
276
277    // We have to load the lvalue.
278    RV = CGF.EmitLoadOfLValue(LV);
279  } else {
280    if (!ObjCARCReferenceLifetimeType.isNull()) {
281      ReferenceTemporary = CreateReferenceTemporary(CGF,
282                                                  ObjCARCReferenceLifetimeType,
283                                                    InitializedDecl);
284
285
286      LValue RefTempDst = CGF.MakeAddrLValue(ReferenceTemporary,
287                                             ObjCARCReferenceLifetimeType);
288
289      CGF.EmitScalarInit(E, dyn_cast_or_null<ValueDecl>(InitializedDecl),
290                         RefTempDst, false);
291
292      bool ExtendsLifeOfTemporary = false;
293      if (const VarDecl *Var = dyn_cast_or_null<VarDecl>(InitializedDecl)) {
294        if (Var->extendsLifetimeOfTemporary())
295          ExtendsLifeOfTemporary = true;
296      } else if (InitializedDecl && isa<FieldDecl>(InitializedDecl)) {
297        ExtendsLifeOfTemporary = true;
298      }
299
300      if (!ExtendsLifeOfTemporary) {
301        // Since the lifetime of this temporary isn't going to be extended,
302        // we need to clean it up ourselves at the end of the full expression.
303        switch (ObjCARCReferenceLifetimeType.getObjCLifetime()) {
304        case Qualifiers::OCL_None:
305        case Qualifiers::OCL_ExplicitNone:
306        case Qualifiers::OCL_Autoreleasing:
307          break;
308
309        case Qualifiers::OCL_Strong: {
310          assert(!ObjCARCReferenceLifetimeType->isArrayType());
311          CleanupKind cleanupKind = CGF.getARCCleanupKind();
312          CGF.pushDestroy(cleanupKind,
313                          ReferenceTemporary,
314                          ObjCARCReferenceLifetimeType,
315                          CodeGenFunction::destroyARCStrongImprecise,
316                          cleanupKind & EHCleanup);
317          break;
318        }
319
320        case Qualifiers::OCL_Weak:
321          assert(!ObjCARCReferenceLifetimeType->isArrayType());
322          CGF.pushDestroy(NormalAndEHCleanup,
323                          ReferenceTemporary,
324                          ObjCARCReferenceLifetimeType,
325                          CodeGenFunction::destroyARCWeak,
326                          /*useEHCleanupForArray*/ true);
327          break;
328        }
329
330        ObjCARCReferenceLifetimeType = QualType();
331      }
332
333      return ReferenceTemporary;
334    }
335
336    SmallVector<SubobjectAdjustment, 2> Adjustments;
337    while (true) {
338      E = E->IgnoreParens();
339
340      if (const CastExpr *CE = dyn_cast<CastExpr>(E)) {
341        if ((CE->getCastKind() == CK_DerivedToBase ||
342             CE->getCastKind() == CK_UncheckedDerivedToBase) &&
343            E->getType()->isRecordType()) {
344          E = CE->getSubExpr();
345          CXXRecordDecl *Derived
346            = cast<CXXRecordDecl>(E->getType()->getAs<RecordType>()->getDecl());
347          Adjustments.push_back(SubobjectAdjustment(CE, Derived));
348          continue;
349        }
350
351        if (CE->getCastKind() == CK_NoOp) {
352          E = CE->getSubExpr();
353          continue;
354        }
355      } else if (const MemberExpr *ME = dyn_cast<MemberExpr>(E)) {
356        if (!ME->isArrow() && ME->getBase()->isRValue()) {
357          assert(ME->getBase()->getType()->isRecordType());
358          if (FieldDecl *Field = dyn_cast<FieldDecl>(ME->getMemberDecl())) {
359            E = ME->getBase();
360            Adjustments.push_back(SubobjectAdjustment(Field));
361            continue;
362          }
363        }
364      } else if (const BinaryOperator *BO = dyn_cast<BinaryOperator>(E)) {
365        if (BO->isPtrMemOp()) {
366          assert(BO->getLHS()->isRValue());
367          E = BO->getLHS();
368          const MemberPointerType *MPT =
369              BO->getRHS()->getType()->getAs<MemberPointerType>();
370          llvm::Value *Ptr = CGF.EmitScalarExpr(BO->getRHS());
371          Adjustments.push_back(SubobjectAdjustment(MPT, Ptr));
372        }
373      }
374
375      if (const OpaqueValueExpr *opaque = dyn_cast<OpaqueValueExpr>(E))
376        if (opaque->getType()->isRecordType())
377          return CGF.EmitOpaqueValueLValue(opaque).getAddress();
378
379      // Nothing changed.
380      break;
381    }
382
383    // Create a reference temporary if necessary.
384    AggValueSlot AggSlot = AggValueSlot::ignored();
385    if (CGF.hasAggregateLLVMType(E->getType()) &&
386        !E->getType()->isAnyComplexType()) {
387      ReferenceTemporary = CreateReferenceTemporary(CGF, E->getType(),
388                                                    InitializedDecl);
389      CharUnits Alignment = CGF.getContext().getTypeAlignInChars(E->getType());
390      AggValueSlot::IsDestructed_t isDestructed
391        = AggValueSlot::IsDestructed_t(InitializedDecl != 0);
392      AggSlot = AggValueSlot::forAddr(ReferenceTemporary, Alignment,
393                                      Qualifiers(), isDestructed,
394                                      AggValueSlot::DoesNotNeedGCBarriers,
395                                      AggValueSlot::IsNotAliased);
396    }
397
398    if (InitializedDecl) {
399      // Get the destructor for the reference temporary.
400      if (const RecordType *RT = E->getType()->getAs<RecordType>()) {
401        CXXRecordDecl *ClassDecl = cast<CXXRecordDecl>(RT->getDecl());
402        if (!ClassDecl->hasTrivialDestructor())
403          ReferenceTemporaryDtor = ClassDecl->getDestructor();
404      }
405    }
406
407    RV = CGF.EmitAnyExpr(E, AggSlot);
408
409    // Check if need to perform derived-to-base casts and/or field accesses, to
410    // get from the temporary object we created (and, potentially, for which we
411    // extended the lifetime) to the subobject we're binding the reference to.
412    if (!Adjustments.empty()) {
413      llvm::Value *Object = RV.getAggregateAddr();
414      for (unsigned I = Adjustments.size(); I != 0; --I) {
415        SubobjectAdjustment &Adjustment = Adjustments[I-1];
416        switch (Adjustment.Kind) {
417        case SubobjectAdjustment::DerivedToBaseAdjustment:
418          Object =
419              CGF.GetAddressOfBaseClass(Object,
420                                        Adjustment.DerivedToBase.DerivedClass,
421                              Adjustment.DerivedToBase.BasePath->path_begin(),
422                              Adjustment.DerivedToBase.BasePath->path_end(),
423                                        /*NullCheckValue=*/false);
424          break;
425
426        case SubobjectAdjustment::FieldAdjustment: {
427          LValue LV = CGF.MakeAddrLValue(Object, E->getType());
428          LV = CGF.EmitLValueForField(LV, Adjustment.Field);
429          if (LV.isSimple()) {
430            Object = LV.getAddress();
431            break;
432          }
433
434          // For non-simple lvalues, we actually have to create a copy of
435          // the object we're binding to.
436          QualType T = Adjustment.Field->getType().getNonReferenceType()
437                                                  .getUnqualifiedType();
438          Object = CreateReferenceTemporary(CGF, T, InitializedDecl);
439          LValue TempLV = CGF.MakeAddrLValue(Object,
440                                             Adjustment.Field->getType());
441          CGF.EmitStoreThroughLValue(CGF.EmitLoadOfLValue(LV), TempLV);
442          break;
443        }
444
445        case SubobjectAdjustment::MemberPointerAdjustment: {
446          Object = CGF.CGM.getCXXABI().EmitMemberDataPointerAddress(
447                        CGF, Object, Adjustment.Ptr.Ptr, Adjustment.Ptr.MPT);
448          break;
449        }
450        }
451      }
452
453      return Object;
454    }
455  }
456
457  if (RV.isAggregate())
458    return RV.getAggregateAddr();
459
460  // Create a temporary variable that we can bind the reference to.
461  ReferenceTemporary = CreateReferenceTemporary(CGF, E->getType(),
462                                                InitializedDecl);
463
464
465  unsigned Alignment =
466    CGF.getContext().getTypeAlignInChars(E->getType()).getQuantity();
467  if (RV.isScalar())
468    CGF.EmitStoreOfScalar(RV.getScalarVal(), ReferenceTemporary,
469                          /*Volatile=*/false, Alignment, E->getType());
470  else
471    CGF.StoreComplexToAddr(RV.getComplexVal(), ReferenceTemporary,
472                           /*Volatile=*/false);
473  return ReferenceTemporary;
474}
475
476RValue
477CodeGenFunction::EmitReferenceBindingToExpr(const Expr *E,
478                                            const NamedDecl *InitializedDecl) {
479  llvm::Value *ReferenceTemporary = 0;
480  const CXXDestructorDecl *ReferenceTemporaryDtor = 0;
481  QualType ObjCARCReferenceLifetimeType;
482  llvm::Value *Value = EmitExprForReferenceBinding(*this, E, ReferenceTemporary,
483                                                   ReferenceTemporaryDtor,
484                                                   ObjCARCReferenceLifetimeType,
485                                                   InitializedDecl);
486  if (!ReferenceTemporaryDtor && ObjCARCReferenceLifetimeType.isNull())
487    return RValue::get(Value);
488
489  // Make sure to call the destructor for the reference temporary.
490  const VarDecl *VD = dyn_cast_or_null<VarDecl>(InitializedDecl);
491  if (VD && VD->hasGlobalStorage()) {
492    if (ReferenceTemporaryDtor) {
493      llvm::Constant *DtorFn =
494        CGM.GetAddrOfCXXDestructor(ReferenceTemporaryDtor, Dtor_Complete);
495      CGM.getCXXABI().registerGlobalDtor(*this, DtorFn,
496                                    cast<llvm::Constant>(ReferenceTemporary));
497    } else {
498      assert(!ObjCARCReferenceLifetimeType.isNull());
499      // Note: We intentionally do not register a global "destructor" to
500      // release the object.
501    }
502
503    return RValue::get(Value);
504  }
505
506  if (ReferenceTemporaryDtor)
507    PushDestructorCleanup(ReferenceTemporaryDtor, ReferenceTemporary);
508  else {
509    switch (ObjCARCReferenceLifetimeType.getObjCLifetime()) {
510    case Qualifiers::OCL_None:
511      llvm_unreachable(
512                      "Not a reference temporary that needs to be deallocated");
513    case Qualifiers::OCL_ExplicitNone:
514    case Qualifiers::OCL_Autoreleasing:
515      // Nothing to do.
516      break;
517
518    case Qualifiers::OCL_Strong: {
519      bool precise = VD && VD->hasAttr<ObjCPreciseLifetimeAttr>();
520      CleanupKind cleanupKind = getARCCleanupKind();
521      pushDestroy(cleanupKind, ReferenceTemporary, ObjCARCReferenceLifetimeType,
522                  precise ? destroyARCStrongPrecise : destroyARCStrongImprecise,
523                  cleanupKind & EHCleanup);
524      break;
525    }
526
527    case Qualifiers::OCL_Weak: {
528      // __weak objects always get EH cleanups; otherwise, exceptions
529      // could cause really nasty crashes instead of mere leaks.
530      pushDestroy(NormalAndEHCleanup, ReferenceTemporary,
531                  ObjCARCReferenceLifetimeType, destroyARCWeak, true);
532      break;
533    }
534    }
535  }
536
537  return RValue::get(Value);
538}
539
540
541/// getAccessedFieldNo - Given an encoded value and a result number, return the
542/// input field number being accessed.
543unsigned CodeGenFunction::getAccessedFieldNo(unsigned Idx,
544                                             const llvm::Constant *Elts) {
545  return cast<llvm::ConstantInt>(Elts->getAggregateElement(Idx))
546      ->getZExtValue();
547}
548
549void CodeGenFunction::EmitCheck(llvm::Value *Address, unsigned Size) {
550  if (!CatchUndefined)
551    return;
552
553  // This needs to be to the standard address space.
554  Address = Builder.CreateBitCast(Address, Int8PtrTy);
555
556  llvm::Value *F = CGM.getIntrinsic(llvm::Intrinsic::objectsize, IntPtrTy);
557
558  llvm::Value *Min = Builder.getFalse();
559  llvm::Value *C = Builder.CreateCall2(F, Address, Min);
560  llvm::BasicBlock *Cont = createBasicBlock();
561  Builder.CreateCondBr(Builder.CreateICmpUGE(C,
562                                        llvm::ConstantInt::get(IntPtrTy, Size)),
563                       Cont, getTrapBB());
564  EmitBlock(Cont);
565}
566
567
568CodeGenFunction::ComplexPairTy CodeGenFunction::
569EmitComplexPrePostIncDec(const UnaryOperator *E, LValue LV,
570                         bool isInc, bool isPre) {
571  ComplexPairTy InVal = LoadComplexFromAddr(LV.getAddress(),
572                                            LV.isVolatileQualified());
573
574  llvm::Value *NextVal;
575  if (isa<llvm::IntegerType>(InVal.first->getType())) {
576    uint64_t AmountVal = isInc ? 1 : -1;
577    NextVal = llvm::ConstantInt::get(InVal.first->getType(), AmountVal, true);
578
579    // Add the inc/dec to the real part.
580    NextVal = Builder.CreateAdd(InVal.first, NextVal, isInc ? "inc" : "dec");
581  } else {
582    QualType ElemTy = E->getType()->getAs<ComplexType>()->getElementType();
583    llvm::APFloat FVal(getContext().getFloatTypeSemantics(ElemTy), 1);
584    if (!isInc)
585      FVal.changeSign();
586    NextVal = llvm::ConstantFP::get(getLLVMContext(), FVal);
587
588    // Add the inc/dec to the real part.
589    NextVal = Builder.CreateFAdd(InVal.first, NextVal, isInc ? "inc" : "dec");
590  }
591
592  ComplexPairTy IncVal(NextVal, InVal.second);
593
594  // Store the updated result through the lvalue.
595  StoreComplexToAddr(IncVal, LV.getAddress(), LV.isVolatileQualified());
596
597  // If this is a postinc, return the value read from memory, otherwise use the
598  // updated value.
599  return isPre ? IncVal : InVal;
600}
601
602
603//===----------------------------------------------------------------------===//
604//                         LValue Expression Emission
605//===----------------------------------------------------------------------===//
606
607RValue CodeGenFunction::GetUndefRValue(QualType Ty) {
608  if (Ty->isVoidType())
609    return RValue::get(0);
610
611  if (const ComplexType *CTy = Ty->getAs<ComplexType>()) {
612    llvm::Type *EltTy = ConvertType(CTy->getElementType());
613    llvm::Value *U = llvm::UndefValue::get(EltTy);
614    return RValue::getComplex(std::make_pair(U, U));
615  }
616
617  // If this is a use of an undefined aggregate type, the aggregate must have an
618  // identifiable address.  Just because the contents of the value are undefined
619  // doesn't mean that the address can't be taken and compared.
620  if (hasAggregateLLVMType(Ty)) {
621    llvm::Value *DestPtr = CreateMemTemp(Ty, "undef.agg.tmp");
622    return RValue::getAggregate(DestPtr);
623  }
624
625  return RValue::get(llvm::UndefValue::get(ConvertType(Ty)));
626}
627
628RValue CodeGenFunction::EmitUnsupportedRValue(const Expr *E,
629                                              const char *Name) {
630  ErrorUnsupported(E, Name);
631  return GetUndefRValue(E->getType());
632}
633
634LValue CodeGenFunction::EmitUnsupportedLValue(const Expr *E,
635                                              const char *Name) {
636  ErrorUnsupported(E, Name);
637  llvm::Type *Ty = llvm::PointerType::getUnqual(ConvertType(E->getType()));
638  return MakeAddrLValue(llvm::UndefValue::get(Ty), E->getType());
639}
640
641LValue CodeGenFunction::EmitCheckedLValue(const Expr *E) {
642  LValue LV = EmitLValue(E);
643  if (!isa<DeclRefExpr>(E) && !LV.isBitField() && LV.isSimple())
644    EmitCheck(LV.getAddress(),
645              getContext().getTypeSizeInChars(E->getType()).getQuantity());
646  return LV;
647}
648
649/// EmitLValue - Emit code to compute a designator that specifies the location
650/// of the expression.
651///
652/// This can return one of two things: a simple address or a bitfield reference.
653/// In either case, the LLVM Value* in the LValue structure is guaranteed to be
654/// an LLVM pointer type.
655///
656/// If this returns a bitfield reference, nothing about the pointee type of the
657/// LLVM value is known: For example, it may not be a pointer to an integer.
658///
659/// If this returns a normal address, and if the lvalue's C type is fixed size,
660/// this method guarantees that the returned pointer type will point to an LLVM
661/// type of the same size of the lvalue's type.  If the lvalue has a variable
662/// length type, this is not possible.
663///
664LValue CodeGenFunction::EmitLValue(const Expr *E) {
665  switch (E->getStmtClass()) {
666  default: return EmitUnsupportedLValue(E, "l-value expression");
667
668  case Expr::ObjCPropertyRefExprClass:
669    llvm_unreachable("cannot emit a property reference directly");
670
671  case Expr::ObjCSelectorExprClass:
672  return EmitObjCSelectorLValue(cast<ObjCSelectorExpr>(E));
673  case Expr::ObjCIsaExprClass:
674    return EmitObjCIsaExpr(cast<ObjCIsaExpr>(E));
675  case Expr::BinaryOperatorClass:
676    return EmitBinaryOperatorLValue(cast<BinaryOperator>(E));
677  case Expr::CompoundAssignOperatorClass:
678    if (!E->getType()->isAnyComplexType())
679      return EmitCompoundAssignmentLValue(cast<CompoundAssignOperator>(E));
680    return EmitComplexCompoundAssignmentLValue(cast<CompoundAssignOperator>(E));
681  case Expr::CallExprClass:
682  case Expr::CXXMemberCallExprClass:
683  case Expr::CXXOperatorCallExprClass:
684  case Expr::UserDefinedLiteralClass:
685    return EmitCallExprLValue(cast<CallExpr>(E));
686  case Expr::VAArgExprClass:
687    return EmitVAArgExprLValue(cast<VAArgExpr>(E));
688  case Expr::DeclRefExprClass:
689    return EmitDeclRefLValue(cast<DeclRefExpr>(E));
690  case Expr::ParenExprClass:
691    return EmitLValue(cast<ParenExpr>(E)->getSubExpr());
692  case Expr::GenericSelectionExprClass:
693    return EmitLValue(cast<GenericSelectionExpr>(E)->getResultExpr());
694  case Expr::PredefinedExprClass:
695    return EmitPredefinedLValue(cast<PredefinedExpr>(E));
696  case Expr::StringLiteralClass:
697    return EmitStringLiteralLValue(cast<StringLiteral>(E));
698  case Expr::ObjCEncodeExprClass:
699    return EmitObjCEncodeExprLValue(cast<ObjCEncodeExpr>(E));
700  case Expr::PseudoObjectExprClass:
701    return EmitPseudoObjectLValue(cast<PseudoObjectExpr>(E));
702  case Expr::InitListExprClass:
703    return EmitInitListLValue(cast<InitListExpr>(E));
704  case Expr::CXXTemporaryObjectExprClass:
705  case Expr::CXXConstructExprClass:
706    return EmitCXXConstructLValue(cast<CXXConstructExpr>(E));
707  case Expr::CXXBindTemporaryExprClass:
708    return EmitCXXBindTemporaryLValue(cast<CXXBindTemporaryExpr>(E));
709  case Expr::LambdaExprClass:
710    return EmitLambdaLValue(cast<LambdaExpr>(E));
711
712  case Expr::ExprWithCleanupsClass: {
713    const ExprWithCleanups *cleanups = cast<ExprWithCleanups>(E);
714    enterFullExpression(cleanups);
715    RunCleanupsScope Scope(*this);
716    return EmitLValue(cleanups->getSubExpr());
717  }
718
719  case Expr::CXXScalarValueInitExprClass:
720    return EmitNullInitializationLValue(cast<CXXScalarValueInitExpr>(E));
721  case Expr::CXXDefaultArgExprClass:
722    return EmitLValue(cast<CXXDefaultArgExpr>(E)->getExpr());
723  case Expr::CXXTypeidExprClass:
724    return EmitCXXTypeidLValue(cast<CXXTypeidExpr>(E));
725
726  case Expr::ObjCMessageExprClass:
727    return EmitObjCMessageExprLValue(cast<ObjCMessageExpr>(E));
728  case Expr::ObjCIvarRefExprClass:
729    return EmitObjCIvarRefLValue(cast<ObjCIvarRefExpr>(E));
730  case Expr::StmtExprClass:
731    return EmitStmtExprLValue(cast<StmtExpr>(E));
732  case Expr::UnaryOperatorClass:
733    return EmitUnaryOpLValue(cast<UnaryOperator>(E));
734  case Expr::ArraySubscriptExprClass:
735    return EmitArraySubscriptExpr(cast<ArraySubscriptExpr>(E));
736  case Expr::ExtVectorElementExprClass:
737    return EmitExtVectorElementExpr(cast<ExtVectorElementExpr>(E));
738  case Expr::MemberExprClass:
739    return EmitMemberExpr(cast<MemberExpr>(E));
740  case Expr::CompoundLiteralExprClass:
741    return EmitCompoundLiteralLValue(cast<CompoundLiteralExpr>(E));
742  case Expr::ConditionalOperatorClass:
743    return EmitConditionalOperatorLValue(cast<ConditionalOperator>(E));
744  case Expr::BinaryConditionalOperatorClass:
745    return EmitConditionalOperatorLValue(cast<BinaryConditionalOperator>(E));
746  case Expr::ChooseExprClass:
747    return EmitLValue(cast<ChooseExpr>(E)->getChosenSubExpr(getContext()));
748  case Expr::OpaqueValueExprClass:
749    return EmitOpaqueValueLValue(cast<OpaqueValueExpr>(E));
750  case Expr::SubstNonTypeTemplateParmExprClass:
751    return EmitLValue(cast<SubstNonTypeTemplateParmExpr>(E)->getReplacement());
752  case Expr::ImplicitCastExprClass:
753  case Expr::CStyleCastExprClass:
754  case Expr::CXXFunctionalCastExprClass:
755  case Expr::CXXStaticCastExprClass:
756  case Expr::CXXDynamicCastExprClass:
757  case Expr::CXXReinterpretCastExprClass:
758  case Expr::CXXConstCastExprClass:
759  case Expr::ObjCBridgedCastExprClass:
760    return EmitCastLValue(cast<CastExpr>(E));
761
762  case Expr::MaterializeTemporaryExprClass:
763    return EmitMaterializeTemporaryExpr(cast<MaterializeTemporaryExpr>(E));
764  }
765}
766
767/// Given an object of the given canonical type, can we safely copy a
768/// value out of it based on its initializer?
769static bool isConstantEmittableObjectType(QualType type) {
770  assert(type.isCanonical());
771  assert(!type->isReferenceType());
772
773  // Must be const-qualified but non-volatile.
774  Qualifiers qs = type.getLocalQualifiers();
775  if (!qs.hasConst() || qs.hasVolatile()) return false;
776
777  // Otherwise, all object types satisfy this except C++ classes with
778  // mutable subobjects or non-trivial copy/destroy behavior.
779  if (const RecordType *RT = dyn_cast<RecordType>(type))
780    if (const CXXRecordDecl *RD = dyn_cast<CXXRecordDecl>(RT->getDecl()))
781      if (RD->hasMutableFields() || !RD->isTrivial())
782        return false;
783
784  return true;
785}
786
787/// Can we constant-emit a load of a reference to a variable of the
788/// given type?  This is different from predicates like
789/// Decl::isUsableInConstantExpressions because we do want it to apply
790/// in situations that don't necessarily satisfy the language's rules
791/// for this (e.g. C++'s ODR-use rules).  For example, we want to able
792/// to do this with const float variables even if those variables
793/// aren't marked 'constexpr'.
794enum ConstantEmissionKind {
795  CEK_None,
796  CEK_AsReferenceOnly,
797  CEK_AsValueOrReference,
798  CEK_AsValueOnly
799};
800static ConstantEmissionKind checkVarTypeForConstantEmission(QualType type) {
801  type = type.getCanonicalType();
802  if (const ReferenceType *ref = dyn_cast<ReferenceType>(type)) {
803    if (isConstantEmittableObjectType(ref->getPointeeType()))
804      return CEK_AsValueOrReference;
805    return CEK_AsReferenceOnly;
806  }
807  if (isConstantEmittableObjectType(type))
808    return CEK_AsValueOnly;
809  return CEK_None;
810}
811
812/// Try to emit a reference to the given value without producing it as
813/// an l-value.  This is actually more than an optimization: we can't
814/// produce an l-value for variables that we never actually captured
815/// in a block or lambda, which means const int variables or constexpr
816/// literals or similar.
817CodeGenFunction::ConstantEmission
818CodeGenFunction::tryEmitAsConstant(DeclRefExpr *refExpr) {
819  ValueDecl *value = refExpr->getDecl();
820
821  // The value needs to be an enum constant or a constant variable.
822  ConstantEmissionKind CEK;
823  if (isa<ParmVarDecl>(value)) {
824    CEK = CEK_None;
825  } else if (VarDecl *var = dyn_cast<VarDecl>(value)) {
826    CEK = checkVarTypeForConstantEmission(var->getType());
827  } else if (isa<EnumConstantDecl>(value)) {
828    CEK = CEK_AsValueOnly;
829  } else {
830    CEK = CEK_None;
831  }
832  if (CEK == CEK_None) return ConstantEmission();
833
834  Expr::EvalResult result;
835  bool resultIsReference;
836  QualType resultType;
837
838  // It's best to evaluate all the way as an r-value if that's permitted.
839  if (CEK != CEK_AsReferenceOnly &&
840      refExpr->EvaluateAsRValue(result, getContext())) {
841    resultIsReference = false;
842    resultType = refExpr->getType();
843
844  // Otherwise, try to evaluate as an l-value.
845  } else if (CEK != CEK_AsValueOnly &&
846             refExpr->EvaluateAsLValue(result, getContext())) {
847    resultIsReference = true;
848    resultType = value->getType();
849
850  // Failure.
851  } else {
852    return ConstantEmission();
853  }
854
855  // In any case, if the initializer has side-effects, abandon ship.
856  if (result.HasSideEffects)
857    return ConstantEmission();
858
859  // Emit as a constant.
860  llvm::Constant *C = CGM.EmitConstantValue(result.Val, resultType, this);
861
862  // Make sure we emit a debug reference to the global variable.
863  // This should probably fire even for
864  if (isa<VarDecl>(value)) {
865    if (!getContext().DeclMustBeEmitted(cast<VarDecl>(value)))
866      EmitDeclRefExprDbgValue(refExpr, C);
867  } else {
868    assert(isa<EnumConstantDecl>(value));
869    EmitDeclRefExprDbgValue(refExpr, C);
870  }
871
872  // If we emitted a reference constant, we need to dereference that.
873  if (resultIsReference)
874    return ConstantEmission::forReference(C);
875
876  return ConstantEmission::forValue(C);
877}
878
879llvm::Value *CodeGenFunction::EmitLoadOfScalar(LValue lvalue) {
880  return EmitLoadOfScalar(lvalue.getAddress(), lvalue.isVolatile(),
881                          lvalue.getAlignment().getQuantity(),
882                          lvalue.getType(), lvalue.getTBAAInfo());
883}
884
885static bool hasBooleanRepresentation(QualType Ty) {
886  if (Ty->isBooleanType())
887    return true;
888
889  if (const EnumType *ET = Ty->getAs<EnumType>())
890    return ET->getDecl()->getIntegerType()->isBooleanType();
891
892  if (const AtomicType *AT = Ty->getAs<AtomicType>())
893    return hasBooleanRepresentation(AT->getValueType());
894
895  return false;
896}
897
898llvm::MDNode *CodeGenFunction::getRangeForLoadFromType(QualType Ty) {
899  const EnumType *ET = Ty->getAs<EnumType>();
900  bool IsRegularCPlusPlusEnum = (getLangOpts().CPlusPlus && ET &&
901                                 CGM.getCodeGenOpts().StrictEnums &&
902                                 !ET->getDecl()->isFixed());
903  bool IsBool = hasBooleanRepresentation(Ty);
904  if (!IsBool && !IsRegularCPlusPlusEnum)
905    return NULL;
906
907  llvm::APInt Min;
908  llvm::APInt End;
909  if (IsBool) {
910    Min = llvm::APInt(8, 0);
911    End = llvm::APInt(8, 2);
912  } else {
913    const EnumDecl *ED = ET->getDecl();
914    llvm::Type *LTy = ConvertTypeForMem(ED->getIntegerType());
915    unsigned Bitwidth = LTy->getScalarSizeInBits();
916    unsigned NumNegativeBits = ED->getNumNegativeBits();
917    unsigned NumPositiveBits = ED->getNumPositiveBits();
918
919    if (NumNegativeBits) {
920      unsigned NumBits = std::max(NumNegativeBits, NumPositiveBits + 1);
921      assert(NumBits <= Bitwidth);
922      End = llvm::APInt(Bitwidth, 1) << (NumBits - 1);
923      Min = -End;
924    } else {
925      assert(NumPositiveBits <= Bitwidth);
926      End = llvm::APInt(Bitwidth, 1) << NumPositiveBits;
927      Min = llvm::APInt(Bitwidth, 0);
928    }
929  }
930
931  llvm::MDBuilder MDHelper(getLLVMContext());
932  return MDHelper.createRange(Min, End);
933}
934
935llvm::Value *CodeGenFunction::EmitLoadOfScalar(llvm::Value *Addr, bool Volatile,
936                                              unsigned Alignment, QualType Ty,
937                                              llvm::MDNode *TBAAInfo) {
938  llvm::LoadInst *Load = Builder.CreateLoad(Addr);
939  if (Volatile)
940    Load->setVolatile(true);
941  if (Alignment)
942    Load->setAlignment(Alignment);
943  if (TBAAInfo)
944    CGM.DecorateInstruction(Load, TBAAInfo);
945  // If this is an atomic type, all normal reads must be atomic
946  if (Ty->isAtomicType())
947    Load->setAtomic(llvm::SequentiallyConsistent);
948
949  if (CGM.getCodeGenOpts().OptimizationLevel > 0)
950    if (llvm::MDNode *RangeInfo = getRangeForLoadFromType(Ty))
951      Load->setMetadata(llvm::LLVMContext::MD_range, RangeInfo);
952
953  return EmitFromMemory(Load, Ty);
954}
955
956llvm::Value *CodeGenFunction::EmitToMemory(llvm::Value *Value, QualType Ty) {
957  // Bool has a different representation in memory than in registers.
958  if (hasBooleanRepresentation(Ty)) {
959    // This should really always be an i1, but sometimes it's already
960    // an i8, and it's awkward to track those cases down.
961    if (Value->getType()->isIntegerTy(1))
962      return Builder.CreateZExt(Value, Builder.getInt8Ty(), "frombool");
963    assert(Value->getType()->isIntegerTy(8) && "value rep of bool not i1/i8");
964  }
965
966  return Value;
967}
968
969llvm::Value *CodeGenFunction::EmitFromMemory(llvm::Value *Value, QualType Ty) {
970  // Bool has a different representation in memory than in registers.
971  if (hasBooleanRepresentation(Ty)) {
972    assert(Value->getType()->isIntegerTy(8) && "memory rep of bool not i8");
973    return Builder.CreateTrunc(Value, Builder.getInt1Ty(), "tobool");
974  }
975
976  return Value;
977}
978
979void CodeGenFunction::EmitStoreOfScalar(llvm::Value *Value, llvm::Value *Addr,
980                                        bool Volatile, unsigned Alignment,
981                                        QualType Ty,
982                                        llvm::MDNode *TBAAInfo,
983                                        bool isInit) {
984  Value = EmitToMemory(Value, Ty);
985
986  llvm::StoreInst *Store = Builder.CreateStore(Value, Addr, Volatile);
987  if (Alignment)
988    Store->setAlignment(Alignment);
989  if (TBAAInfo)
990    CGM.DecorateInstruction(Store, TBAAInfo);
991  if (!isInit && Ty->isAtomicType())
992    Store->setAtomic(llvm::SequentiallyConsistent);
993}
994
995void CodeGenFunction::EmitStoreOfScalar(llvm::Value *value, LValue lvalue,
996    bool isInit) {
997  EmitStoreOfScalar(value, lvalue.getAddress(), lvalue.isVolatile(),
998                    lvalue.getAlignment().getQuantity(), lvalue.getType(),
999                    lvalue.getTBAAInfo(), isInit);
1000}
1001
1002/// EmitLoadOfLValue - Given an expression that represents a value lvalue, this
1003/// method emits the address of the lvalue, then loads the result as an rvalue,
1004/// returning the rvalue.
1005RValue CodeGenFunction::EmitLoadOfLValue(LValue LV) {
1006  if (LV.isObjCWeak()) {
1007    // load of a __weak object.
1008    llvm::Value *AddrWeakObj = LV.getAddress();
1009    return RValue::get(CGM.getObjCRuntime().EmitObjCWeakRead(*this,
1010                                                             AddrWeakObj));
1011  }
1012  if (LV.getQuals().getObjCLifetime() == Qualifiers::OCL_Weak)
1013    return RValue::get(EmitARCLoadWeak(LV.getAddress()));
1014
1015  if (LV.isSimple()) {
1016    assert(!LV.getType()->isFunctionType());
1017
1018    // Everything needs a load.
1019    return RValue::get(EmitLoadOfScalar(LV));
1020  }
1021
1022  if (LV.isVectorElt()) {
1023    llvm::LoadInst *Load = Builder.CreateLoad(LV.getVectorAddr(),
1024                                              LV.isVolatileQualified());
1025    Load->setAlignment(LV.getAlignment().getQuantity());
1026    return RValue::get(Builder.CreateExtractElement(Load, LV.getVectorIdx(),
1027                                                    "vecext"));
1028  }
1029
1030  // If this is a reference to a subset of the elements of a vector, either
1031  // shuffle the input or extract/insert them as appropriate.
1032  if (LV.isExtVectorElt())
1033    return EmitLoadOfExtVectorElementLValue(LV);
1034
1035  assert(LV.isBitField() && "Unknown LValue type!");
1036  return EmitLoadOfBitfieldLValue(LV);
1037}
1038
1039RValue CodeGenFunction::EmitLoadOfBitfieldLValue(LValue LV) {
1040  const CGBitFieldInfo &Info = LV.getBitFieldInfo();
1041
1042  // Get the output type.
1043  llvm::Type *ResLTy = ConvertType(LV.getType());
1044  unsigned ResSizeInBits = CGM.getTargetData().getTypeSizeInBits(ResLTy);
1045
1046  // Compute the result as an OR of all of the individual component accesses.
1047  llvm::Value *Res = 0;
1048  for (unsigned i = 0, e = Info.getNumComponents(); i != e; ++i) {
1049    const CGBitFieldInfo::AccessInfo &AI = Info.getComponent(i);
1050
1051    // Get the field pointer.
1052    llvm::Value *Ptr = LV.getBitFieldBaseAddr();
1053
1054    // Only offset by the field index if used, so that incoming values are not
1055    // required to be structures.
1056    if (AI.FieldIndex)
1057      Ptr = Builder.CreateStructGEP(Ptr, AI.FieldIndex, "bf.field");
1058
1059    // Offset by the byte offset, if used.
1060    if (!AI.FieldByteOffset.isZero()) {
1061      Ptr = EmitCastToVoidPtr(Ptr);
1062      Ptr = Builder.CreateConstGEP1_32(Ptr, AI.FieldByteOffset.getQuantity(),
1063                                       "bf.field.offs");
1064    }
1065
1066    // Cast to the access type.
1067    llvm::Type *PTy = llvm::Type::getIntNPtrTy(getLLVMContext(), AI.AccessWidth,
1068                       CGM.getContext().getTargetAddressSpace(LV.getType()));
1069    Ptr = Builder.CreateBitCast(Ptr, PTy);
1070
1071    // Perform the load.
1072    llvm::LoadInst *Load = Builder.CreateLoad(Ptr, LV.isVolatileQualified());
1073    if (!AI.AccessAlignment.isZero())
1074      Load->setAlignment(AI.AccessAlignment.getQuantity());
1075
1076    // Shift out unused low bits and mask out unused high bits.
1077    llvm::Value *Val = Load;
1078    if (AI.FieldBitStart)
1079      Val = Builder.CreateLShr(Load, AI.FieldBitStart);
1080    Val = Builder.CreateAnd(Val, llvm::APInt::getLowBitsSet(AI.AccessWidth,
1081                                                            AI.TargetBitWidth),
1082                            "bf.clear");
1083
1084    // Extend or truncate to the target size.
1085    if (AI.AccessWidth < ResSizeInBits)
1086      Val = Builder.CreateZExt(Val, ResLTy);
1087    else if (AI.AccessWidth > ResSizeInBits)
1088      Val = Builder.CreateTrunc(Val, ResLTy);
1089
1090    // Shift into place, and OR into the result.
1091    if (AI.TargetBitOffset)
1092      Val = Builder.CreateShl(Val, AI.TargetBitOffset);
1093    Res = Res ? Builder.CreateOr(Res, Val) : Val;
1094  }
1095
1096  // If the bit-field is signed, perform the sign-extension.
1097  //
1098  // FIXME: This can easily be folded into the load of the high bits, which
1099  // could also eliminate the mask of high bits in some situations.
1100  if (Info.isSigned()) {
1101    unsigned ExtraBits = ResSizeInBits - Info.getSize();
1102    if (ExtraBits)
1103      Res = Builder.CreateAShr(Builder.CreateShl(Res, ExtraBits),
1104                               ExtraBits, "bf.val.sext");
1105  }
1106
1107  return RValue::get(Res);
1108}
1109
1110// If this is a reference to a subset of the elements of a vector, create an
1111// appropriate shufflevector.
1112RValue CodeGenFunction::EmitLoadOfExtVectorElementLValue(LValue LV) {
1113  llvm::LoadInst *Load = Builder.CreateLoad(LV.getExtVectorAddr(),
1114                                            LV.isVolatileQualified());
1115  Load->setAlignment(LV.getAlignment().getQuantity());
1116  llvm::Value *Vec = Load;
1117
1118  const llvm::Constant *Elts = LV.getExtVectorElts();
1119
1120  // If the result of the expression is a non-vector type, we must be extracting
1121  // a single element.  Just codegen as an extractelement.
1122  const VectorType *ExprVT = LV.getType()->getAs<VectorType>();
1123  if (!ExprVT) {
1124    unsigned InIdx = getAccessedFieldNo(0, Elts);
1125    llvm::Value *Elt = llvm::ConstantInt::get(Int32Ty, InIdx);
1126    return RValue::get(Builder.CreateExtractElement(Vec, Elt));
1127  }
1128
1129  // Always use shuffle vector to try to retain the original program structure
1130  unsigned NumResultElts = ExprVT->getNumElements();
1131
1132  SmallVector<llvm::Constant*, 4> Mask;
1133  for (unsigned i = 0; i != NumResultElts; ++i)
1134    Mask.push_back(Builder.getInt32(getAccessedFieldNo(i, Elts)));
1135
1136  llvm::Value *MaskV = llvm::ConstantVector::get(Mask);
1137  Vec = Builder.CreateShuffleVector(Vec, llvm::UndefValue::get(Vec->getType()),
1138                                    MaskV);
1139  return RValue::get(Vec);
1140}
1141
1142
1143
1144/// EmitStoreThroughLValue - Store the specified rvalue into the specified
1145/// lvalue, where both are guaranteed to the have the same type, and that type
1146/// is 'Ty'.
1147void CodeGenFunction::EmitStoreThroughLValue(RValue Src, LValue Dst, bool isInit) {
1148  if (!Dst.isSimple()) {
1149    if (Dst.isVectorElt()) {
1150      // Read/modify/write the vector, inserting the new element.
1151      llvm::LoadInst *Load = Builder.CreateLoad(Dst.getVectorAddr(),
1152                                                Dst.isVolatileQualified());
1153      Load->setAlignment(Dst.getAlignment().getQuantity());
1154      llvm::Value *Vec = Load;
1155      Vec = Builder.CreateInsertElement(Vec, Src.getScalarVal(),
1156                                        Dst.getVectorIdx(), "vecins");
1157      llvm::StoreInst *Store = Builder.CreateStore(Vec, Dst.getVectorAddr(),
1158                                                   Dst.isVolatileQualified());
1159      Store->setAlignment(Dst.getAlignment().getQuantity());
1160      return;
1161    }
1162
1163    // If this is an update of extended vector elements, insert them as
1164    // appropriate.
1165    if (Dst.isExtVectorElt())
1166      return EmitStoreThroughExtVectorComponentLValue(Src, Dst);
1167
1168    assert(Dst.isBitField() && "Unknown LValue type");
1169    return EmitStoreThroughBitfieldLValue(Src, Dst);
1170  }
1171
1172  // There's special magic for assigning into an ARC-qualified l-value.
1173  if (Qualifiers::ObjCLifetime Lifetime = Dst.getQuals().getObjCLifetime()) {
1174    switch (Lifetime) {
1175    case Qualifiers::OCL_None:
1176      llvm_unreachable("present but none");
1177
1178    case Qualifiers::OCL_ExplicitNone:
1179      // nothing special
1180      break;
1181
1182    case Qualifiers::OCL_Strong:
1183      EmitARCStoreStrong(Dst, Src.getScalarVal(), /*ignore*/ true);
1184      return;
1185
1186    case Qualifiers::OCL_Weak:
1187      EmitARCStoreWeak(Dst.getAddress(), Src.getScalarVal(), /*ignore*/ true);
1188      return;
1189
1190    case Qualifiers::OCL_Autoreleasing:
1191      Src = RValue::get(EmitObjCExtendObjectLifetime(Dst.getType(),
1192                                                     Src.getScalarVal()));
1193      // fall into the normal path
1194      break;
1195    }
1196  }
1197
1198  if (Dst.isObjCWeak() && !Dst.isNonGC()) {
1199    // load of a __weak object.
1200    llvm::Value *LvalueDst = Dst.getAddress();
1201    llvm::Value *src = Src.getScalarVal();
1202     CGM.getObjCRuntime().EmitObjCWeakAssign(*this, src, LvalueDst);
1203    return;
1204  }
1205
1206  if (Dst.isObjCStrong() && !Dst.isNonGC()) {
1207    // load of a __strong object.
1208    llvm::Value *LvalueDst = Dst.getAddress();
1209    llvm::Value *src = Src.getScalarVal();
1210    if (Dst.isObjCIvar()) {
1211      assert(Dst.getBaseIvarExp() && "BaseIvarExp is NULL");
1212      llvm::Type *ResultType = ConvertType(getContext().LongTy);
1213      llvm::Value *RHS = EmitScalarExpr(Dst.getBaseIvarExp());
1214      llvm::Value *dst = RHS;
1215      RHS = Builder.CreatePtrToInt(RHS, ResultType, "sub.ptr.rhs.cast");
1216      llvm::Value *LHS =
1217        Builder.CreatePtrToInt(LvalueDst, ResultType, "sub.ptr.lhs.cast");
1218      llvm::Value *BytesBetween = Builder.CreateSub(LHS, RHS, "ivar.offset");
1219      CGM.getObjCRuntime().EmitObjCIvarAssign(*this, src, dst,
1220                                              BytesBetween);
1221    } else if (Dst.isGlobalObjCRef()) {
1222      CGM.getObjCRuntime().EmitObjCGlobalAssign(*this, src, LvalueDst,
1223                                                Dst.isThreadLocalRef());
1224    }
1225    else
1226      CGM.getObjCRuntime().EmitObjCStrongCastAssign(*this, src, LvalueDst);
1227    return;
1228  }
1229
1230  assert(Src.isScalar() && "Can't emit an agg store with this method");
1231  EmitStoreOfScalar(Src.getScalarVal(), Dst, isInit);
1232}
1233
1234void CodeGenFunction::EmitStoreThroughBitfieldLValue(RValue Src, LValue Dst,
1235                                                     llvm::Value **Result) {
1236  const CGBitFieldInfo &Info = Dst.getBitFieldInfo();
1237
1238  // Get the output type.
1239  llvm::Type *ResLTy = ConvertTypeForMem(Dst.getType());
1240  unsigned ResSizeInBits = CGM.getTargetData().getTypeSizeInBits(ResLTy);
1241
1242  // Get the source value, truncated to the width of the bit-field.
1243  llvm::Value *SrcVal = Src.getScalarVal();
1244
1245  if (hasBooleanRepresentation(Dst.getType()))
1246    SrcVal = Builder.CreateIntCast(SrcVal, ResLTy, /*IsSigned=*/false);
1247
1248  SrcVal = Builder.CreateAnd(SrcVal, llvm::APInt::getLowBitsSet(ResSizeInBits,
1249                                                                Info.getSize()),
1250                             "bf.value");
1251
1252  // Return the new value of the bit-field, if requested.
1253  if (Result) {
1254    // Cast back to the proper type for result.
1255    llvm::Type *SrcTy = Src.getScalarVal()->getType();
1256    llvm::Value *ReloadVal = Builder.CreateIntCast(SrcVal, SrcTy, false,
1257                                                   "bf.reload.val");
1258
1259    // Sign extend if necessary.
1260    if (Info.isSigned()) {
1261      unsigned ExtraBits = ResSizeInBits - Info.getSize();
1262      if (ExtraBits)
1263        ReloadVal = Builder.CreateAShr(Builder.CreateShl(ReloadVal, ExtraBits),
1264                                       ExtraBits, "bf.reload.sext");
1265    }
1266
1267    *Result = ReloadVal;
1268  }
1269
1270  // Iterate over the components, writing each piece to memory.
1271  for (unsigned i = 0, e = Info.getNumComponents(); i != e; ++i) {
1272    const CGBitFieldInfo::AccessInfo &AI = Info.getComponent(i);
1273
1274    // Get the field pointer.
1275    llvm::Value *Ptr = Dst.getBitFieldBaseAddr();
1276    unsigned addressSpace =
1277      cast<llvm::PointerType>(Ptr->getType())->getAddressSpace();
1278
1279    // Only offset by the field index if used, so that incoming values are not
1280    // required to be structures.
1281    if (AI.FieldIndex)
1282      Ptr = Builder.CreateStructGEP(Ptr, AI.FieldIndex, "bf.field");
1283
1284    // Offset by the byte offset, if used.
1285    if (!AI.FieldByteOffset.isZero()) {
1286      Ptr = EmitCastToVoidPtr(Ptr);
1287      Ptr = Builder.CreateConstGEP1_32(Ptr, AI.FieldByteOffset.getQuantity(),
1288                                       "bf.field.offs");
1289    }
1290
1291    // Cast to the access type.
1292    llvm::Type *AccessLTy =
1293      llvm::Type::getIntNTy(getLLVMContext(), AI.AccessWidth);
1294
1295    llvm::Type *PTy = AccessLTy->getPointerTo(addressSpace);
1296    Ptr = Builder.CreateBitCast(Ptr, PTy);
1297
1298    // Extract the piece of the bit-field value to write in this access, limited
1299    // to the values that are part of this access.
1300    llvm::Value *Val = SrcVal;
1301    if (AI.TargetBitOffset)
1302      Val = Builder.CreateLShr(Val, AI.TargetBitOffset);
1303    Val = Builder.CreateAnd(Val, llvm::APInt::getLowBitsSet(ResSizeInBits,
1304                                                            AI.TargetBitWidth));
1305
1306    // Extend or truncate to the access size.
1307    if (ResSizeInBits < AI.AccessWidth)
1308      Val = Builder.CreateZExt(Val, AccessLTy);
1309    else if (ResSizeInBits > AI.AccessWidth)
1310      Val = Builder.CreateTrunc(Val, AccessLTy);
1311
1312    // Shift into the position in memory.
1313    if (AI.FieldBitStart)
1314      Val = Builder.CreateShl(Val, AI.FieldBitStart);
1315
1316    // If necessary, load and OR in bits that are outside of the bit-field.
1317    if (AI.TargetBitWidth != AI.AccessWidth) {
1318      llvm::LoadInst *Load = Builder.CreateLoad(Ptr, Dst.isVolatileQualified());
1319      if (!AI.AccessAlignment.isZero())
1320        Load->setAlignment(AI.AccessAlignment.getQuantity());
1321
1322      // Compute the mask for zeroing the bits that are part of the bit-field.
1323      llvm::APInt InvMask =
1324        ~llvm::APInt::getBitsSet(AI.AccessWidth, AI.FieldBitStart,
1325                                 AI.FieldBitStart + AI.TargetBitWidth);
1326
1327      // Apply the mask and OR in to the value to write.
1328      Val = Builder.CreateOr(Builder.CreateAnd(Load, InvMask), Val);
1329    }
1330
1331    // Write the value.
1332    llvm::StoreInst *Store = Builder.CreateStore(Val, Ptr,
1333                                                 Dst.isVolatileQualified());
1334    if (!AI.AccessAlignment.isZero())
1335      Store->setAlignment(AI.AccessAlignment.getQuantity());
1336  }
1337}
1338
1339void CodeGenFunction::EmitStoreThroughExtVectorComponentLValue(RValue Src,
1340                                                               LValue Dst) {
1341  // This access turns into a read/modify/write of the vector.  Load the input
1342  // value now.
1343  llvm::LoadInst *Load = Builder.CreateLoad(Dst.getExtVectorAddr(),
1344                                            Dst.isVolatileQualified());
1345  Load->setAlignment(Dst.getAlignment().getQuantity());
1346  llvm::Value *Vec = Load;
1347  const llvm::Constant *Elts = Dst.getExtVectorElts();
1348
1349  llvm::Value *SrcVal = Src.getScalarVal();
1350
1351  if (const VectorType *VTy = Dst.getType()->getAs<VectorType>()) {
1352    unsigned NumSrcElts = VTy->getNumElements();
1353    unsigned NumDstElts =
1354       cast<llvm::VectorType>(Vec->getType())->getNumElements();
1355    if (NumDstElts == NumSrcElts) {
1356      // Use shuffle vector is the src and destination are the same number of
1357      // elements and restore the vector mask since it is on the side it will be
1358      // stored.
1359      SmallVector<llvm::Constant*, 4> Mask(NumDstElts);
1360      for (unsigned i = 0; i != NumSrcElts; ++i)
1361        Mask[getAccessedFieldNo(i, Elts)] = Builder.getInt32(i);
1362
1363      llvm::Value *MaskV = llvm::ConstantVector::get(Mask);
1364      Vec = Builder.CreateShuffleVector(SrcVal,
1365                                        llvm::UndefValue::get(Vec->getType()),
1366                                        MaskV);
1367    } else if (NumDstElts > NumSrcElts) {
1368      // Extended the source vector to the same length and then shuffle it
1369      // into the destination.
1370      // FIXME: since we're shuffling with undef, can we just use the indices
1371      //        into that?  This could be simpler.
1372      SmallVector<llvm::Constant*, 4> ExtMask;
1373      for (unsigned i = 0; i != NumSrcElts; ++i)
1374        ExtMask.push_back(Builder.getInt32(i));
1375      ExtMask.resize(NumDstElts, llvm::UndefValue::get(Int32Ty));
1376      llvm::Value *ExtMaskV = llvm::ConstantVector::get(ExtMask);
1377      llvm::Value *ExtSrcVal =
1378        Builder.CreateShuffleVector(SrcVal,
1379                                    llvm::UndefValue::get(SrcVal->getType()),
1380                                    ExtMaskV);
1381      // build identity
1382      SmallVector<llvm::Constant*, 4> Mask;
1383      for (unsigned i = 0; i != NumDstElts; ++i)
1384        Mask.push_back(Builder.getInt32(i));
1385
1386      // modify when what gets shuffled in
1387      for (unsigned i = 0; i != NumSrcElts; ++i)
1388        Mask[getAccessedFieldNo(i, Elts)] = Builder.getInt32(i+NumDstElts);
1389      llvm::Value *MaskV = llvm::ConstantVector::get(Mask);
1390      Vec = Builder.CreateShuffleVector(Vec, ExtSrcVal, MaskV);
1391    } else {
1392      // We should never shorten the vector
1393      llvm_unreachable("unexpected shorten vector length");
1394    }
1395  } else {
1396    // If the Src is a scalar (not a vector) it must be updating one element.
1397    unsigned InIdx = getAccessedFieldNo(0, Elts);
1398    llvm::Value *Elt = llvm::ConstantInt::get(Int32Ty, InIdx);
1399    Vec = Builder.CreateInsertElement(Vec, SrcVal, Elt);
1400  }
1401
1402  llvm::StoreInst *Store = Builder.CreateStore(Vec, Dst.getExtVectorAddr(),
1403                                               Dst.isVolatileQualified());
1404  Store->setAlignment(Dst.getAlignment().getQuantity());
1405}
1406
1407// setObjCGCLValueClass - sets class of he lvalue for the purpose of
1408// generating write-barries API. It is currently a global, ivar,
1409// or neither.
1410static void setObjCGCLValueClass(const ASTContext &Ctx, const Expr *E,
1411                                 LValue &LV,
1412                                 bool IsMemberAccess=false) {
1413  if (Ctx.getLangOpts().getGC() == LangOptions::NonGC)
1414    return;
1415
1416  if (isa<ObjCIvarRefExpr>(E)) {
1417    QualType ExpTy = E->getType();
1418    if (IsMemberAccess && ExpTy->isPointerType()) {
1419      // If ivar is a structure pointer, assigning to field of
1420      // this struct follows gcc's behavior and makes it a non-ivar
1421      // writer-barrier conservatively.
1422      ExpTy = ExpTy->getAs<PointerType>()->getPointeeType();
1423      if (ExpTy->isRecordType()) {
1424        LV.setObjCIvar(false);
1425        return;
1426      }
1427    }
1428    LV.setObjCIvar(true);
1429    ObjCIvarRefExpr *Exp = cast<ObjCIvarRefExpr>(const_cast<Expr*>(E));
1430    LV.setBaseIvarExp(Exp->getBase());
1431    LV.setObjCArray(E->getType()->isArrayType());
1432    return;
1433  }
1434
1435  if (const DeclRefExpr *Exp = dyn_cast<DeclRefExpr>(E)) {
1436    if (const VarDecl *VD = dyn_cast<VarDecl>(Exp->getDecl())) {
1437      if (VD->hasGlobalStorage()) {
1438        LV.setGlobalObjCRef(true);
1439        LV.setThreadLocalRef(VD->isThreadSpecified());
1440      }
1441    }
1442    LV.setObjCArray(E->getType()->isArrayType());
1443    return;
1444  }
1445
1446  if (const UnaryOperator *Exp = dyn_cast<UnaryOperator>(E)) {
1447    setObjCGCLValueClass(Ctx, Exp->getSubExpr(), LV, IsMemberAccess);
1448    return;
1449  }
1450
1451  if (const ParenExpr *Exp = dyn_cast<ParenExpr>(E)) {
1452    setObjCGCLValueClass(Ctx, Exp->getSubExpr(), LV, IsMemberAccess);
1453    if (LV.isObjCIvar()) {
1454      // If cast is to a structure pointer, follow gcc's behavior and make it
1455      // a non-ivar write-barrier.
1456      QualType ExpTy = E->getType();
1457      if (ExpTy->isPointerType())
1458        ExpTy = ExpTy->getAs<PointerType>()->getPointeeType();
1459      if (ExpTy->isRecordType())
1460        LV.setObjCIvar(false);
1461    }
1462    return;
1463  }
1464
1465  if (const GenericSelectionExpr *Exp = dyn_cast<GenericSelectionExpr>(E)) {
1466    setObjCGCLValueClass(Ctx, Exp->getResultExpr(), LV);
1467    return;
1468  }
1469
1470  if (const ImplicitCastExpr *Exp = dyn_cast<ImplicitCastExpr>(E)) {
1471    setObjCGCLValueClass(Ctx, Exp->getSubExpr(), LV, IsMemberAccess);
1472    return;
1473  }
1474
1475  if (const CStyleCastExpr *Exp = dyn_cast<CStyleCastExpr>(E)) {
1476    setObjCGCLValueClass(Ctx, Exp->getSubExpr(), LV, IsMemberAccess);
1477    return;
1478  }
1479
1480  if (const ObjCBridgedCastExpr *Exp = dyn_cast<ObjCBridgedCastExpr>(E)) {
1481    setObjCGCLValueClass(Ctx, Exp->getSubExpr(), LV, IsMemberAccess);
1482    return;
1483  }
1484
1485  if (const ArraySubscriptExpr *Exp = dyn_cast<ArraySubscriptExpr>(E)) {
1486    setObjCGCLValueClass(Ctx, Exp->getBase(), LV);
1487    if (LV.isObjCIvar() && !LV.isObjCArray())
1488      // Using array syntax to assigning to what an ivar points to is not
1489      // same as assigning to the ivar itself. {id *Names;} Names[i] = 0;
1490      LV.setObjCIvar(false);
1491    else if (LV.isGlobalObjCRef() && !LV.isObjCArray())
1492      // Using array syntax to assigning to what global points to is not
1493      // same as assigning to the global itself. {id *G;} G[i] = 0;
1494      LV.setGlobalObjCRef(false);
1495    return;
1496  }
1497
1498  if (const MemberExpr *Exp = dyn_cast<MemberExpr>(E)) {
1499    setObjCGCLValueClass(Ctx, Exp->getBase(), LV, true);
1500    // We don't know if member is an 'ivar', but this flag is looked at
1501    // only in the context of LV.isObjCIvar().
1502    LV.setObjCArray(E->getType()->isArrayType());
1503    return;
1504  }
1505}
1506
1507static llvm::Value *
1508EmitBitCastOfLValueToProperType(CodeGenFunction &CGF,
1509                                llvm::Value *V, llvm::Type *IRType,
1510                                StringRef Name = StringRef()) {
1511  unsigned AS = cast<llvm::PointerType>(V->getType())->getAddressSpace();
1512  return CGF.Builder.CreateBitCast(V, IRType->getPointerTo(AS), Name);
1513}
1514
1515static LValue EmitGlobalVarDeclLValue(CodeGenFunction &CGF,
1516                                      const Expr *E, const VarDecl *VD) {
1517  assert((VD->hasExternalStorage() || VD->isFileVarDecl()) &&
1518         "Var decl must have external storage or be a file var decl!");
1519
1520  llvm::Value *V = CGF.CGM.GetAddrOfGlobalVar(VD);
1521  llvm::Type *RealVarTy = CGF.getTypes().ConvertTypeForMem(VD->getType());
1522  V = EmitBitCastOfLValueToProperType(CGF, V, RealVarTy);
1523  CharUnits Alignment = CGF.getContext().getDeclAlign(VD);
1524  QualType T = E->getType();
1525  LValue LV;
1526  if (VD->getType()->isReferenceType()) {
1527    llvm::LoadInst *LI = CGF.Builder.CreateLoad(V);
1528    LI->setAlignment(Alignment.getQuantity());
1529    V = LI;
1530    LV = CGF.MakeNaturalAlignAddrLValue(V, T);
1531  } else {
1532    LV = CGF.MakeAddrLValue(V, E->getType(), Alignment);
1533  }
1534  setObjCGCLValueClass(CGF.getContext(), E, LV);
1535  return LV;
1536}
1537
1538static LValue EmitFunctionDeclLValue(CodeGenFunction &CGF,
1539                                     const Expr *E, const FunctionDecl *FD) {
1540  llvm::Value *V = CGF.CGM.GetAddrOfFunction(FD);
1541  if (!FD->hasPrototype()) {
1542    if (const FunctionProtoType *Proto =
1543            FD->getType()->getAs<FunctionProtoType>()) {
1544      // Ugly case: for a K&R-style definition, the type of the definition
1545      // isn't the same as the type of a use.  Correct for this with a
1546      // bitcast.
1547      QualType NoProtoType =
1548          CGF.getContext().getFunctionNoProtoType(Proto->getResultType());
1549      NoProtoType = CGF.getContext().getPointerType(NoProtoType);
1550      V = CGF.Builder.CreateBitCast(V, CGF.ConvertType(NoProtoType));
1551    }
1552  }
1553  CharUnits Alignment = CGF.getContext().getDeclAlign(FD);
1554  return CGF.MakeAddrLValue(V, E->getType(), Alignment);
1555}
1556
1557LValue CodeGenFunction::EmitDeclRefLValue(const DeclRefExpr *E) {
1558  const NamedDecl *ND = E->getDecl();
1559  CharUnits Alignment = getContext().getDeclAlign(ND);
1560  QualType T = E->getType();
1561
1562  // FIXME: We should be able to assert this for FunctionDecls as well!
1563  // FIXME: We should be able to assert this for all DeclRefExprs, not just
1564  // those with a valid source location.
1565  assert((ND->isUsed(false) || !isa<VarDecl>(ND) ||
1566          !E->getLocation().isValid()) &&
1567         "Should not use decl without marking it used!");
1568
1569  if (ND->hasAttr<WeakRefAttr>()) {
1570    const ValueDecl *VD = cast<ValueDecl>(ND);
1571    llvm::Constant *Aliasee = CGM.GetWeakRefReference(VD);
1572    return MakeAddrLValue(Aliasee, E->getType(), Alignment);
1573  }
1574
1575  if (const VarDecl *VD = dyn_cast<VarDecl>(ND)) {
1576    // Check if this is a global variable.
1577    if (VD->hasExternalStorage() || VD->isFileVarDecl())
1578      return EmitGlobalVarDeclLValue(*this, E, VD);
1579
1580    bool isBlockVariable = VD->hasAttr<BlocksAttr>();
1581
1582    bool NonGCable = VD->hasLocalStorage() &&
1583                     !VD->getType()->isReferenceType() &&
1584                     !isBlockVariable;
1585
1586    llvm::Value *V = LocalDeclMap[VD];
1587    if (!V && VD->isStaticLocal())
1588      V = CGM.getStaticLocalDeclAddress(VD);
1589
1590    // Use special handling for lambdas.
1591    if (!V) {
1592      if (FieldDecl *FD = LambdaCaptureFields.lookup(VD)) {
1593        QualType LambdaTagType = getContext().getTagDeclType(FD->getParent());
1594        LValue LambdaLV = MakeNaturalAlignAddrLValue(CXXABIThisValue,
1595                                                     LambdaTagType);
1596        return EmitLValueForField(LambdaLV, FD);
1597      }
1598
1599      assert(isa<BlockDecl>(CurCodeDecl) && E->refersToEnclosingLocal());
1600      CharUnits alignment = getContext().getDeclAlign(VD);
1601      return MakeAddrLValue(GetAddrOfBlockDecl(VD, isBlockVariable),
1602                            E->getType(), alignment);
1603    }
1604
1605    assert(V && "DeclRefExpr not entered in LocalDeclMap?");
1606
1607    if (isBlockVariable)
1608      V = BuildBlockByrefAddress(V, VD);
1609
1610    LValue LV;
1611    if (VD->getType()->isReferenceType()) {
1612      llvm::LoadInst *LI = Builder.CreateLoad(V);
1613      LI->setAlignment(Alignment.getQuantity());
1614      V = LI;
1615      LV = MakeNaturalAlignAddrLValue(V, T);
1616    } else {
1617      LV = MakeAddrLValue(V, T, Alignment);
1618    }
1619
1620    if (NonGCable) {
1621      LV.getQuals().removeObjCGCAttr();
1622      LV.setNonGC(true);
1623    }
1624    setObjCGCLValueClass(getContext(), E, LV);
1625    return LV;
1626  }
1627
1628  if (const FunctionDecl *fn = dyn_cast<FunctionDecl>(ND))
1629    return EmitFunctionDeclLValue(*this, E, fn);
1630
1631  llvm_unreachable("Unhandled DeclRefExpr");
1632}
1633
1634LValue CodeGenFunction::EmitUnaryOpLValue(const UnaryOperator *E) {
1635  // __extension__ doesn't affect lvalue-ness.
1636  if (E->getOpcode() == UO_Extension)
1637    return EmitLValue(E->getSubExpr());
1638
1639  QualType ExprTy = getContext().getCanonicalType(E->getSubExpr()->getType());
1640  switch (E->getOpcode()) {
1641  default: llvm_unreachable("Unknown unary operator lvalue!");
1642  case UO_Deref: {
1643    QualType T = E->getSubExpr()->getType()->getPointeeType();
1644    assert(!T.isNull() && "CodeGenFunction::EmitUnaryOpLValue: Illegal type");
1645
1646    LValue LV = MakeNaturalAlignAddrLValue(EmitScalarExpr(E->getSubExpr()), T);
1647    LV.getQuals().setAddressSpace(ExprTy.getAddressSpace());
1648
1649    // We should not generate __weak write barrier on indirect reference
1650    // of a pointer to object; as in void foo (__weak id *param); *param = 0;
1651    // But, we continue to generate __strong write barrier on indirect write
1652    // into a pointer to object.
1653    if (getContext().getLangOpts().ObjC1 &&
1654        getContext().getLangOpts().getGC() != LangOptions::NonGC &&
1655        LV.isObjCWeak())
1656      LV.setNonGC(!E->isOBJCGCCandidate(getContext()));
1657    return LV;
1658  }
1659  case UO_Real:
1660  case UO_Imag: {
1661    LValue LV = EmitLValue(E->getSubExpr());
1662    assert(LV.isSimple() && "real/imag on non-ordinary l-value");
1663    llvm::Value *Addr = LV.getAddress();
1664
1665    // __real is valid on scalars.  This is a faster way of testing that.
1666    // __imag can only produce an rvalue on scalars.
1667    if (E->getOpcode() == UO_Real &&
1668        !cast<llvm::PointerType>(Addr->getType())
1669           ->getElementType()->isStructTy()) {
1670      assert(E->getSubExpr()->getType()->isArithmeticType());
1671      return LV;
1672    }
1673
1674    assert(E->getSubExpr()->getType()->isAnyComplexType());
1675
1676    unsigned Idx = E->getOpcode() == UO_Imag;
1677    return MakeAddrLValue(Builder.CreateStructGEP(LV.getAddress(),
1678                                                  Idx, "idx"),
1679                          ExprTy);
1680  }
1681  case UO_PreInc:
1682  case UO_PreDec: {
1683    LValue LV = EmitLValue(E->getSubExpr());
1684    bool isInc = E->getOpcode() == UO_PreInc;
1685
1686    if (E->getType()->isAnyComplexType())
1687      EmitComplexPrePostIncDec(E, LV, isInc, true/*isPre*/);
1688    else
1689      EmitScalarPrePostIncDec(E, LV, isInc, true/*isPre*/);
1690    return LV;
1691  }
1692  }
1693}
1694
1695LValue CodeGenFunction::EmitStringLiteralLValue(const StringLiteral *E) {
1696  return MakeAddrLValue(CGM.GetAddrOfConstantStringFromLiteral(E),
1697                        E->getType());
1698}
1699
1700LValue CodeGenFunction::EmitObjCEncodeExprLValue(const ObjCEncodeExpr *E) {
1701  return MakeAddrLValue(CGM.GetAddrOfConstantStringFromObjCEncode(E),
1702                        E->getType());
1703}
1704
1705static llvm::Constant*
1706GetAddrOfConstantWideString(StringRef Str,
1707                            const char *GlobalName,
1708                            ASTContext &Context,
1709                            QualType Ty, SourceLocation Loc,
1710                            CodeGenModule &CGM) {
1711
1712  StringLiteral *SL = StringLiteral::Create(Context,
1713                                            Str,
1714                                            StringLiteral::Wide,
1715                                            /*Pascal = */false,
1716                                            Ty, Loc);
1717  llvm::Constant *C = CGM.GetConstantArrayFromStringLiteral(SL);
1718  llvm::GlobalVariable *GV =
1719    new llvm::GlobalVariable(CGM.getModule(), C->getType(),
1720                             !CGM.getLangOpts().WritableStrings,
1721                             llvm::GlobalValue::PrivateLinkage,
1722                             C, GlobalName);
1723  const unsigned WideAlignment =
1724    Context.getTypeAlignInChars(Ty).getQuantity();
1725  GV->setAlignment(WideAlignment);
1726  return GV;
1727}
1728
1729// FIXME: Mostly copied from StringLiteralParser::CopyStringFragment
1730static void ConvertUTF8ToWideString(unsigned CharByteWidth, StringRef Source,
1731                                    SmallString<32>& Target) {
1732  Target.resize(CharByteWidth * (Source.size() + 1));
1733  char* ResultPtr = &Target[0];
1734
1735  assert(CharByteWidth==1 || CharByteWidth==2 || CharByteWidth==4);
1736  ConversionResult result = conversionOK;
1737  // Copy the character span over.
1738  if (CharByteWidth == 1) {
1739    if (!isLegalUTF8String(reinterpret_cast<const UTF8*>(&*Source.begin()),
1740                           reinterpret_cast<const UTF8*>(&*Source.end())))
1741      result = sourceIllegal;
1742    memcpy(ResultPtr, Source.data(), Source.size());
1743    ResultPtr += Source.size();
1744  } else if (CharByteWidth == 2) {
1745    UTF8 const *sourceStart = (UTF8 const *)Source.data();
1746    // FIXME: Make the type of the result buffer correct instead of
1747    // using reinterpret_cast.
1748    UTF16 *targetStart = reinterpret_cast<UTF16*>(ResultPtr);
1749    ConversionFlags flags = strictConversion;
1750    result = ConvertUTF8toUTF16(
1751      &sourceStart,sourceStart + Source.size(),
1752        &targetStart,targetStart + 2*Source.size(),flags);
1753    if (result==conversionOK)
1754      ResultPtr = reinterpret_cast<char*>(targetStart);
1755  } else if (CharByteWidth == 4) {
1756    UTF8 const *sourceStart = (UTF8 const *)Source.data();
1757    // FIXME: Make the type of the result buffer correct instead of
1758    // using reinterpret_cast.
1759    UTF32 *targetStart = reinterpret_cast<UTF32*>(ResultPtr);
1760    ConversionFlags flags = strictConversion;
1761    result = ConvertUTF8toUTF32(
1762        &sourceStart,sourceStart + Source.size(),
1763        &targetStart,targetStart + 4*Source.size(),flags);
1764    if (result==conversionOK)
1765      ResultPtr = reinterpret_cast<char*>(targetStart);
1766  }
1767  assert((result != targetExhausted)
1768         && "ConvertUTF8toUTFXX exhausted target buffer");
1769  assert(result == conversionOK);
1770  Target.resize(ResultPtr - &Target[0]);
1771}
1772
1773LValue CodeGenFunction::EmitPredefinedLValue(const PredefinedExpr *E) {
1774  switch (E->getIdentType()) {
1775  default:
1776    return EmitUnsupportedLValue(E, "predefined expression");
1777
1778  case PredefinedExpr::Func:
1779  case PredefinedExpr::Function:
1780  case PredefinedExpr::LFunction:
1781  case PredefinedExpr::PrettyFunction: {
1782    unsigned IdentType = E->getIdentType();
1783    std::string GlobalVarName;
1784
1785    switch (IdentType) {
1786    default: llvm_unreachable("Invalid type");
1787    case PredefinedExpr::Func:
1788      GlobalVarName = "__func__.";
1789      break;
1790    case PredefinedExpr::Function:
1791      GlobalVarName = "__FUNCTION__.";
1792      break;
1793    case PredefinedExpr::LFunction:
1794      GlobalVarName = "L__FUNCTION__.";
1795      break;
1796    case PredefinedExpr::PrettyFunction:
1797      GlobalVarName = "__PRETTY_FUNCTION__.";
1798      break;
1799    }
1800
1801    StringRef FnName = CurFn->getName();
1802    if (FnName.startswith("\01"))
1803      FnName = FnName.substr(1);
1804    GlobalVarName += FnName;
1805
1806    const Decl *CurDecl = CurCodeDecl;
1807    if (CurDecl == 0)
1808      CurDecl = getContext().getTranslationUnitDecl();
1809
1810    std::string FunctionName =
1811        (isa<BlockDecl>(CurDecl)
1812         ? FnName.str()
1813         : PredefinedExpr::ComputeName((PredefinedExpr::IdentType)IdentType,
1814                                       CurDecl));
1815
1816    const Type* ElemType = E->getType()->getArrayElementTypeNoTypeQual();
1817    llvm::Constant *C;
1818    if (ElemType->isWideCharType()) {
1819      SmallString<32> RawChars;
1820      ConvertUTF8ToWideString(
1821          getContext().getTypeSizeInChars(ElemType).getQuantity(),
1822          FunctionName, RawChars);
1823      C = GetAddrOfConstantWideString(RawChars,
1824                                      GlobalVarName.c_str(),
1825                                      getContext(),
1826                                      E->getType(),
1827                                      E->getLocation(),
1828                                      CGM);
1829    } else {
1830      C = CGM.GetAddrOfConstantCString(FunctionName,
1831                                       GlobalVarName.c_str(),
1832                                       1);
1833    }
1834    return MakeAddrLValue(C, E->getType());
1835  }
1836  }
1837}
1838
1839llvm::BasicBlock *CodeGenFunction::getTrapBB() {
1840  const CodeGenOptions &GCO = CGM.getCodeGenOpts();
1841
1842  // If we are not optimzing, don't collapse all calls to trap in the function
1843  // to the same call, that way, in the debugger they can see which operation
1844  // did in fact fail.  If we are optimizing, we collapse all calls to trap down
1845  // to just one per function to save on codesize.
1846  if (GCO.OptimizationLevel && TrapBB)
1847    return TrapBB;
1848
1849  llvm::BasicBlock *Cont = 0;
1850  if (HaveInsertPoint()) {
1851    Cont = createBasicBlock("cont");
1852    EmitBranch(Cont);
1853  }
1854  TrapBB = createBasicBlock("trap");
1855  EmitBlock(TrapBB);
1856
1857  llvm::Value *F = CGM.getIntrinsic(llvm::Intrinsic::trap);
1858  llvm::CallInst *TrapCall = Builder.CreateCall(F);
1859  TrapCall->setDoesNotReturn();
1860  TrapCall->setDoesNotThrow();
1861  Builder.CreateUnreachable();
1862
1863  if (Cont)
1864    EmitBlock(Cont);
1865  return TrapBB;
1866}
1867
1868/// isSimpleArrayDecayOperand - If the specified expr is a simple decay from an
1869/// array to pointer, return the array subexpression.
1870static const Expr *isSimpleArrayDecayOperand(const Expr *E) {
1871  // If this isn't just an array->pointer decay, bail out.
1872  const CastExpr *CE = dyn_cast<CastExpr>(E);
1873  if (CE == 0 || CE->getCastKind() != CK_ArrayToPointerDecay)
1874    return 0;
1875
1876  // If this is a decay from variable width array, bail out.
1877  const Expr *SubExpr = CE->getSubExpr();
1878  if (SubExpr->getType()->isVariableArrayType())
1879    return 0;
1880
1881  return SubExpr;
1882}
1883
1884LValue CodeGenFunction::EmitArraySubscriptExpr(const ArraySubscriptExpr *E) {
1885  // The index must always be an integer, which is not an aggregate.  Emit it.
1886  llvm::Value *Idx = EmitScalarExpr(E->getIdx());
1887  QualType IdxTy  = E->getIdx()->getType();
1888  bool IdxSigned = IdxTy->isSignedIntegerOrEnumerationType();
1889
1890  // If the base is a vector type, then we are forming a vector element lvalue
1891  // with this subscript.
1892  if (E->getBase()->getType()->isVectorType()) {
1893    // Emit the vector as an lvalue to get its address.
1894    LValue LHS = EmitLValue(E->getBase());
1895    assert(LHS.isSimple() && "Can only subscript lvalue vectors here!");
1896    Idx = Builder.CreateIntCast(Idx, Int32Ty, IdxSigned, "vidx");
1897    return LValue::MakeVectorElt(LHS.getAddress(), Idx,
1898                                 E->getBase()->getType(), LHS.getAlignment());
1899  }
1900
1901  // Extend or truncate the index type to 32 or 64-bits.
1902  if (Idx->getType() != IntPtrTy)
1903    Idx = Builder.CreateIntCast(Idx, IntPtrTy, IdxSigned, "idxprom");
1904
1905  // We know that the pointer points to a type of the correct size, unless the
1906  // size is a VLA or Objective-C interface.
1907  llvm::Value *Address = 0;
1908  CharUnits ArrayAlignment;
1909  if (const VariableArrayType *vla =
1910        getContext().getAsVariableArrayType(E->getType())) {
1911    // The base must be a pointer, which is not an aggregate.  Emit
1912    // it.  It needs to be emitted first in case it's what captures
1913    // the VLA bounds.
1914    Address = EmitScalarExpr(E->getBase());
1915
1916    // The element count here is the total number of non-VLA elements.
1917    llvm::Value *numElements = getVLASize(vla).first;
1918
1919    // Effectively, the multiply by the VLA size is part of the GEP.
1920    // GEP indexes are signed, and scaling an index isn't permitted to
1921    // signed-overflow, so we use the same semantics for our explicit
1922    // multiply.  We suppress this if overflow is not undefined behavior.
1923    if (getLangOpts().isSignedOverflowDefined()) {
1924      Idx = Builder.CreateMul(Idx, numElements);
1925      Address = Builder.CreateGEP(Address, Idx, "arrayidx");
1926    } else {
1927      Idx = Builder.CreateNSWMul(Idx, numElements);
1928      Address = Builder.CreateInBoundsGEP(Address, Idx, "arrayidx");
1929    }
1930  } else if (const ObjCObjectType *OIT = E->getType()->getAs<ObjCObjectType>()){
1931    // Indexing over an interface, as in "NSString *P; P[4];"
1932    llvm::Value *InterfaceSize =
1933      llvm::ConstantInt::get(Idx->getType(),
1934          getContext().getTypeSizeInChars(OIT).getQuantity());
1935
1936    Idx = Builder.CreateMul(Idx, InterfaceSize);
1937
1938    // The base must be a pointer, which is not an aggregate.  Emit it.
1939    llvm::Value *Base = EmitScalarExpr(E->getBase());
1940    Address = EmitCastToVoidPtr(Base);
1941    Address = Builder.CreateGEP(Address, Idx, "arrayidx");
1942    Address = Builder.CreateBitCast(Address, Base->getType());
1943  } else if (const Expr *Array = isSimpleArrayDecayOperand(E->getBase())) {
1944    // If this is A[i] where A is an array, the frontend will have decayed the
1945    // base to be a ArrayToPointerDecay implicit cast.  While correct, it is
1946    // inefficient at -O0 to emit a "gep A, 0, 0" when codegen'ing it, then a
1947    // "gep x, i" here.  Emit one "gep A, 0, i".
1948    assert(Array->getType()->isArrayType() &&
1949           "Array to pointer decay must have array source type!");
1950    LValue ArrayLV = EmitLValue(Array);
1951    llvm::Value *ArrayPtr = ArrayLV.getAddress();
1952    llvm::Value *Zero = llvm::ConstantInt::get(Int32Ty, 0);
1953    llvm::Value *Args[] = { Zero, Idx };
1954
1955    // Propagate the alignment from the array itself to the result.
1956    ArrayAlignment = ArrayLV.getAlignment();
1957
1958    if (getContext().getLangOpts().isSignedOverflowDefined())
1959      Address = Builder.CreateGEP(ArrayPtr, Args, "arrayidx");
1960    else
1961      Address = Builder.CreateInBoundsGEP(ArrayPtr, Args, "arrayidx");
1962  } else {
1963    // The base must be a pointer, which is not an aggregate.  Emit it.
1964    llvm::Value *Base = EmitScalarExpr(E->getBase());
1965    if (getContext().getLangOpts().isSignedOverflowDefined())
1966      Address = Builder.CreateGEP(Base, Idx, "arrayidx");
1967    else
1968      Address = Builder.CreateInBoundsGEP(Base, Idx, "arrayidx");
1969  }
1970
1971  QualType T = E->getBase()->getType()->getPointeeType();
1972  assert(!T.isNull() &&
1973         "CodeGenFunction::EmitArraySubscriptExpr(): Illegal base type");
1974
1975
1976  // Limit the alignment to that of the result type.
1977  LValue LV;
1978  if (!ArrayAlignment.isZero()) {
1979    CharUnits Align = getContext().getTypeAlignInChars(T);
1980    ArrayAlignment = std::min(Align, ArrayAlignment);
1981    LV = MakeAddrLValue(Address, T, ArrayAlignment);
1982  } else {
1983    LV = MakeNaturalAlignAddrLValue(Address, T);
1984  }
1985
1986  LV.getQuals().setAddressSpace(E->getBase()->getType().getAddressSpace());
1987
1988  if (getContext().getLangOpts().ObjC1 &&
1989      getContext().getLangOpts().getGC() != LangOptions::NonGC) {
1990    LV.setNonGC(!E->isOBJCGCCandidate(getContext()));
1991    setObjCGCLValueClass(getContext(), E, LV);
1992  }
1993  return LV;
1994}
1995
1996static
1997llvm::Constant *GenerateConstantVector(CGBuilderTy &Builder,
1998                                       SmallVector<unsigned, 4> &Elts) {
1999  SmallVector<llvm::Constant*, 4> CElts;
2000  for (unsigned i = 0, e = Elts.size(); i != e; ++i)
2001    CElts.push_back(Builder.getInt32(Elts[i]));
2002
2003  return llvm::ConstantVector::get(CElts);
2004}
2005
2006LValue CodeGenFunction::
2007EmitExtVectorElementExpr(const ExtVectorElementExpr *E) {
2008  // Emit the base vector as an l-value.
2009  LValue Base;
2010
2011  // ExtVectorElementExpr's base can either be a vector or pointer to vector.
2012  if (E->isArrow()) {
2013    // If it is a pointer to a vector, emit the address and form an lvalue with
2014    // it.
2015    llvm::Value *Ptr = EmitScalarExpr(E->getBase());
2016    const PointerType *PT = E->getBase()->getType()->getAs<PointerType>();
2017    Base = MakeAddrLValue(Ptr, PT->getPointeeType());
2018    Base.getQuals().removeObjCGCAttr();
2019  } else if (E->getBase()->isGLValue()) {
2020    // Otherwise, if the base is an lvalue ( as in the case of foo.x.x),
2021    // emit the base as an lvalue.
2022    assert(E->getBase()->getType()->isVectorType());
2023    Base = EmitLValue(E->getBase());
2024  } else {
2025    // Otherwise, the base is a normal rvalue (as in (V+V).x), emit it as such.
2026    assert(E->getBase()->getType()->isVectorType() &&
2027           "Result must be a vector");
2028    llvm::Value *Vec = EmitScalarExpr(E->getBase());
2029
2030    // Store the vector to memory (because LValue wants an address).
2031    llvm::Value *VecMem = CreateMemTemp(E->getBase()->getType());
2032    Builder.CreateStore(Vec, VecMem);
2033    Base = MakeAddrLValue(VecMem, E->getBase()->getType());
2034  }
2035
2036  QualType type =
2037    E->getType().withCVRQualifiers(Base.getQuals().getCVRQualifiers());
2038
2039  // Encode the element access list into a vector of unsigned indices.
2040  SmallVector<unsigned, 4> Indices;
2041  E->getEncodedElementAccess(Indices);
2042
2043  if (Base.isSimple()) {
2044    llvm::Constant *CV = GenerateConstantVector(Builder, Indices);
2045    return LValue::MakeExtVectorElt(Base.getAddress(), CV, type,
2046                                    Base.getAlignment());
2047  }
2048  assert(Base.isExtVectorElt() && "Can only subscript lvalue vec elts here!");
2049
2050  llvm::Constant *BaseElts = Base.getExtVectorElts();
2051  SmallVector<llvm::Constant *, 4> CElts;
2052
2053  for (unsigned i = 0, e = Indices.size(); i != e; ++i)
2054    CElts.push_back(BaseElts->getAggregateElement(Indices[i]));
2055  llvm::Constant *CV = llvm::ConstantVector::get(CElts);
2056  return LValue::MakeExtVectorElt(Base.getExtVectorAddr(), CV, type,
2057                                  Base.getAlignment());
2058}
2059
2060LValue CodeGenFunction::EmitMemberExpr(const MemberExpr *E) {
2061  Expr *BaseExpr = E->getBase();
2062
2063  // If this is s.x, emit s as an lvalue.  If it is s->x, emit s as a scalar.
2064  LValue BaseLV;
2065  if (E->isArrow())
2066    BaseLV = MakeNaturalAlignAddrLValue(EmitScalarExpr(BaseExpr),
2067                                        BaseExpr->getType()->getPointeeType());
2068  else
2069    BaseLV = EmitLValue(BaseExpr);
2070
2071  NamedDecl *ND = E->getMemberDecl();
2072  if (FieldDecl *Field = dyn_cast<FieldDecl>(ND)) {
2073    LValue LV = EmitLValueForField(BaseLV, Field);
2074    setObjCGCLValueClass(getContext(), E, LV);
2075    return LV;
2076  }
2077
2078  if (VarDecl *VD = dyn_cast<VarDecl>(ND))
2079    return EmitGlobalVarDeclLValue(*this, E, VD);
2080
2081  if (const FunctionDecl *FD = dyn_cast<FunctionDecl>(ND))
2082    return EmitFunctionDeclLValue(*this, E, FD);
2083
2084  llvm_unreachable("Unhandled member declaration!");
2085}
2086
2087LValue CodeGenFunction::EmitLValueForBitfield(llvm::Value *BaseValue,
2088                                              const FieldDecl *Field,
2089                                              unsigned CVRQualifiers) {
2090  const CGRecordLayout &RL =
2091    CGM.getTypes().getCGRecordLayout(Field->getParent());
2092  const CGBitFieldInfo &Info = RL.getBitFieldInfo(Field);
2093  return LValue::MakeBitfield(BaseValue, Info,
2094                          Field->getType().withCVRQualifiers(CVRQualifiers));
2095}
2096
2097/// EmitLValueForAnonRecordField - Given that the field is a member of
2098/// an anonymous struct or union buried inside a record, and given
2099/// that the base value is a pointer to the enclosing record, derive
2100/// an lvalue for the ultimate field.
2101LValue CodeGenFunction::EmitLValueForAnonRecordField(llvm::Value *BaseValue,
2102                                             const IndirectFieldDecl *Field,
2103                                                     unsigned CVRQualifiers) {
2104  IndirectFieldDecl::chain_iterator I = Field->chain_begin(),
2105    IEnd = Field->chain_end();
2106  while (true) {
2107    QualType RecordTy =
2108        getContext().getTypeDeclType(cast<FieldDecl>(*I)->getParent());
2109    LValue LV = EmitLValueForField(MakeAddrLValue(BaseValue, RecordTy),
2110                                   cast<FieldDecl>(*I));
2111    if (++I == IEnd) return LV;
2112
2113    assert(LV.isSimple());
2114    BaseValue = LV.getAddress();
2115    CVRQualifiers |= LV.getVRQualifiers();
2116  }
2117}
2118
2119LValue CodeGenFunction::EmitLValueForField(LValue base,
2120                                           const FieldDecl *field) {
2121  if (field->isBitField())
2122    return EmitLValueForBitfield(base.getAddress(), field,
2123                                 base.getVRQualifiers());
2124
2125  const RecordDecl *rec = field->getParent();
2126  QualType type = field->getType();
2127  CharUnits alignment = getContext().getDeclAlign(field);
2128
2129  // FIXME: It should be impossible to have an LValue without alignment for a
2130  // complete type.
2131  if (!base.getAlignment().isZero())
2132    alignment = std::min(alignment, base.getAlignment());
2133
2134  bool mayAlias = rec->hasAttr<MayAliasAttr>();
2135
2136  llvm::Value *addr = base.getAddress();
2137  unsigned cvr = base.getVRQualifiers();
2138  if (rec->isUnion()) {
2139    // For unions, there is no pointer adjustment.
2140    assert(!type->isReferenceType() && "union has reference member");
2141  } else {
2142    // For structs, we GEP to the field that the record layout suggests.
2143    unsigned idx = CGM.getTypes().getCGRecordLayout(rec).getLLVMFieldNo(field);
2144    addr = Builder.CreateStructGEP(addr, idx, field->getName());
2145
2146    // If this is a reference field, load the reference right now.
2147    if (const ReferenceType *refType = type->getAs<ReferenceType>()) {
2148      llvm::LoadInst *load = Builder.CreateLoad(addr, "ref");
2149      if (cvr & Qualifiers::Volatile) load->setVolatile(true);
2150      load->setAlignment(alignment.getQuantity());
2151
2152      if (CGM.shouldUseTBAA()) {
2153        llvm::MDNode *tbaa;
2154        if (mayAlias)
2155          tbaa = CGM.getTBAAInfo(getContext().CharTy);
2156        else
2157          tbaa = CGM.getTBAAInfo(type);
2158        CGM.DecorateInstruction(load, tbaa);
2159      }
2160
2161      addr = load;
2162      mayAlias = false;
2163      type = refType->getPointeeType();
2164      if (type->isIncompleteType())
2165        alignment = CharUnits();
2166      else
2167        alignment = getContext().getTypeAlignInChars(type);
2168      cvr = 0; // qualifiers don't recursively apply to referencee
2169    }
2170  }
2171
2172  // Make sure that the address is pointing to the right type.  This is critical
2173  // for both unions and structs.  A union needs a bitcast, a struct element
2174  // will need a bitcast if the LLVM type laid out doesn't match the desired
2175  // type.
2176  addr = EmitBitCastOfLValueToProperType(*this, addr,
2177                                         CGM.getTypes().ConvertTypeForMem(type),
2178                                         field->getName());
2179
2180  if (field->hasAttr<AnnotateAttr>())
2181    addr = EmitFieldAnnotations(field, addr);
2182
2183  LValue LV = MakeAddrLValue(addr, type, alignment);
2184  LV.getQuals().addCVRQualifiers(cvr);
2185
2186  // __weak attribute on a field is ignored.
2187  if (LV.getQuals().getObjCGCAttr() == Qualifiers::Weak)
2188    LV.getQuals().removeObjCGCAttr();
2189
2190  // Fields of may_alias structs act like 'char' for TBAA purposes.
2191  // FIXME: this should get propagated down through anonymous structs
2192  // and unions.
2193  if (mayAlias && LV.getTBAAInfo())
2194    LV.setTBAAInfo(CGM.getTBAAInfo(getContext().CharTy));
2195
2196  return LV;
2197}
2198
2199LValue
2200CodeGenFunction::EmitLValueForFieldInitialization(LValue Base,
2201                                                  const FieldDecl *Field) {
2202  QualType FieldType = Field->getType();
2203
2204  if (!FieldType->isReferenceType())
2205    return EmitLValueForField(Base, Field);
2206
2207  const CGRecordLayout &RL =
2208    CGM.getTypes().getCGRecordLayout(Field->getParent());
2209  unsigned idx = RL.getLLVMFieldNo(Field);
2210  llvm::Value *V = Builder.CreateStructGEP(Base.getAddress(), idx);
2211  assert(!FieldType.getObjCGCAttr() && "fields cannot have GC attrs");
2212
2213  // Make sure that the address is pointing to the right type.  This is critical
2214  // for both unions and structs.  A union needs a bitcast, a struct element
2215  // will need a bitcast if the LLVM type laid out doesn't match the desired
2216  // type.
2217  llvm::Type *llvmType = ConvertTypeForMem(FieldType);
2218  V = EmitBitCastOfLValueToProperType(*this, V, llvmType, Field->getName());
2219
2220  CharUnits Alignment = getContext().getDeclAlign(Field);
2221
2222  // FIXME: It should be impossible to have an LValue without alignment for a
2223  // complete type.
2224  if (!Base.getAlignment().isZero())
2225    Alignment = std::min(Alignment, Base.getAlignment());
2226
2227  return MakeAddrLValue(V, FieldType, Alignment);
2228}
2229
2230LValue CodeGenFunction::EmitCompoundLiteralLValue(const CompoundLiteralExpr *E){
2231  if (E->isFileScope()) {
2232    llvm::Value *GlobalPtr = CGM.GetAddrOfConstantCompoundLiteral(E);
2233    return MakeAddrLValue(GlobalPtr, E->getType());
2234  }
2235  if (E->getType()->isVariablyModifiedType())
2236    // make sure to emit the VLA size.
2237    EmitVariablyModifiedType(E->getType());
2238
2239  llvm::Value *DeclPtr = CreateMemTemp(E->getType(), ".compoundliteral");
2240  const Expr *InitExpr = E->getInitializer();
2241  LValue Result = MakeAddrLValue(DeclPtr, E->getType());
2242
2243  EmitAnyExprToMem(InitExpr, DeclPtr, E->getType().getQualifiers(),
2244                   /*Init*/ true);
2245
2246  return Result;
2247}
2248
2249LValue CodeGenFunction::EmitInitListLValue(const InitListExpr *E) {
2250  if (!E->isGLValue())
2251    // Initializing an aggregate temporary in C++11: T{...}.
2252    return EmitAggExprToLValue(E);
2253
2254  // An lvalue initializer list must be initializing a reference.
2255  assert(E->getNumInits() == 1 && "reference init with multiple values");
2256  return EmitLValue(E->getInit(0));
2257}
2258
2259LValue CodeGenFunction::
2260EmitConditionalOperatorLValue(const AbstractConditionalOperator *expr) {
2261  if (!expr->isGLValue()) {
2262    // ?: here should be an aggregate.
2263    assert((hasAggregateLLVMType(expr->getType()) &&
2264            !expr->getType()->isAnyComplexType()) &&
2265           "Unexpected conditional operator!");
2266    return EmitAggExprToLValue(expr);
2267  }
2268
2269  OpaqueValueMapping binding(*this, expr);
2270
2271  const Expr *condExpr = expr->getCond();
2272  bool CondExprBool;
2273  if (ConstantFoldsToSimpleInteger(condExpr, CondExprBool)) {
2274    const Expr *live = expr->getTrueExpr(), *dead = expr->getFalseExpr();
2275    if (!CondExprBool) std::swap(live, dead);
2276
2277    if (!ContainsLabel(dead))
2278      return EmitLValue(live);
2279  }
2280
2281  llvm::BasicBlock *lhsBlock = createBasicBlock("cond.true");
2282  llvm::BasicBlock *rhsBlock = createBasicBlock("cond.false");
2283  llvm::BasicBlock *contBlock = createBasicBlock("cond.end");
2284
2285  ConditionalEvaluation eval(*this);
2286  EmitBranchOnBoolExpr(condExpr, lhsBlock, rhsBlock);
2287
2288  // Any temporaries created here are conditional.
2289  EmitBlock(lhsBlock);
2290  eval.begin(*this);
2291  LValue lhs = EmitLValue(expr->getTrueExpr());
2292  eval.end(*this);
2293
2294  if (!lhs.isSimple())
2295    return EmitUnsupportedLValue(expr, "conditional operator");
2296
2297  lhsBlock = Builder.GetInsertBlock();
2298  Builder.CreateBr(contBlock);
2299
2300  // Any temporaries created here are conditional.
2301  EmitBlock(rhsBlock);
2302  eval.begin(*this);
2303  LValue rhs = EmitLValue(expr->getFalseExpr());
2304  eval.end(*this);
2305  if (!rhs.isSimple())
2306    return EmitUnsupportedLValue(expr, "conditional operator");
2307  rhsBlock = Builder.GetInsertBlock();
2308
2309  EmitBlock(contBlock);
2310
2311  llvm::PHINode *phi = Builder.CreatePHI(lhs.getAddress()->getType(), 2,
2312                                         "cond-lvalue");
2313  phi->addIncoming(lhs.getAddress(), lhsBlock);
2314  phi->addIncoming(rhs.getAddress(), rhsBlock);
2315  return MakeAddrLValue(phi, expr->getType());
2316}
2317
2318/// EmitCastLValue - Casts are never lvalues unless that cast is to a reference
2319/// type. If the cast is to a reference, we can have the usual lvalue result,
2320/// otherwise if a cast is needed by the code generator in an lvalue context,
2321/// then it must mean that we need the address of an aggregate in order to
2322/// access one of its members.  This can happen for all the reasons that casts
2323/// are permitted with aggregate result, including noop aggregate casts, and
2324/// cast from scalar to union.
2325LValue CodeGenFunction::EmitCastLValue(const CastExpr *E) {
2326  switch (E->getCastKind()) {
2327  case CK_ToVoid:
2328    return EmitUnsupportedLValue(E, "unexpected cast lvalue");
2329
2330  case CK_Dependent:
2331    llvm_unreachable("dependent cast kind in IR gen!");
2332
2333  // These two casts are currently treated as no-ops, although they could
2334  // potentially be real operations depending on the target's ABI.
2335  case CK_NonAtomicToAtomic:
2336  case CK_AtomicToNonAtomic:
2337
2338  case CK_NoOp:
2339  case CK_LValueToRValue:
2340    if (!E->getSubExpr()->Classify(getContext()).isPRValue()
2341        || E->getType()->isRecordType())
2342      return EmitLValue(E->getSubExpr());
2343    // Fall through to synthesize a temporary.
2344
2345  case CK_BitCast:
2346  case CK_ArrayToPointerDecay:
2347  case CK_FunctionToPointerDecay:
2348  case CK_NullToMemberPointer:
2349  case CK_NullToPointer:
2350  case CK_IntegralToPointer:
2351  case CK_PointerToIntegral:
2352  case CK_PointerToBoolean:
2353  case CK_VectorSplat:
2354  case CK_IntegralCast:
2355  case CK_IntegralToBoolean:
2356  case CK_IntegralToFloating:
2357  case CK_FloatingToIntegral:
2358  case CK_FloatingToBoolean:
2359  case CK_FloatingCast:
2360  case CK_FloatingRealToComplex:
2361  case CK_FloatingComplexToReal:
2362  case CK_FloatingComplexToBoolean:
2363  case CK_FloatingComplexCast:
2364  case CK_FloatingComplexToIntegralComplex:
2365  case CK_IntegralRealToComplex:
2366  case CK_IntegralComplexToReal:
2367  case CK_IntegralComplexToBoolean:
2368  case CK_IntegralComplexCast:
2369  case CK_IntegralComplexToFloatingComplex:
2370  case CK_DerivedToBaseMemberPointer:
2371  case CK_BaseToDerivedMemberPointer:
2372  case CK_MemberPointerToBoolean:
2373  case CK_ReinterpretMemberPointer:
2374  case CK_AnyPointerToBlockPointerCast:
2375  case CK_ARCProduceObject:
2376  case CK_ARCConsumeObject:
2377  case CK_ARCReclaimReturnedObject:
2378  case CK_ARCExtendBlockObject:
2379  case CK_CopyAndAutoreleaseBlockObject: {
2380    // These casts only produce lvalues when we're binding a reference to a
2381    // temporary realized from a (converted) pure rvalue. Emit the expression
2382    // as a value, copy it into a temporary, and return an lvalue referring to
2383    // that temporary.
2384    llvm::Value *V = CreateMemTemp(E->getType(), "ref.temp");
2385    EmitAnyExprToMem(E, V, E->getType().getQualifiers(), false);
2386    return MakeAddrLValue(V, E->getType());
2387  }
2388
2389  case CK_Dynamic: {
2390    LValue LV = EmitLValue(E->getSubExpr());
2391    llvm::Value *V = LV.getAddress();
2392    const CXXDynamicCastExpr *DCE = cast<CXXDynamicCastExpr>(E);
2393    return MakeAddrLValue(EmitDynamicCast(V, DCE), E->getType());
2394  }
2395
2396  case CK_ConstructorConversion:
2397  case CK_UserDefinedConversion:
2398  case CK_CPointerToObjCPointerCast:
2399  case CK_BlockPointerToObjCPointerCast:
2400    return EmitLValue(E->getSubExpr());
2401
2402  case CK_UncheckedDerivedToBase:
2403  case CK_DerivedToBase: {
2404    const RecordType *DerivedClassTy =
2405      E->getSubExpr()->getType()->getAs<RecordType>();
2406    CXXRecordDecl *DerivedClassDecl =
2407      cast<CXXRecordDecl>(DerivedClassTy->getDecl());
2408
2409    LValue LV = EmitLValue(E->getSubExpr());
2410    llvm::Value *This = LV.getAddress();
2411
2412    // Perform the derived-to-base conversion
2413    llvm::Value *Base =
2414      GetAddressOfBaseClass(This, DerivedClassDecl,
2415                            E->path_begin(), E->path_end(),
2416                            /*NullCheckValue=*/false);
2417
2418    return MakeAddrLValue(Base, E->getType());
2419  }
2420  case CK_ToUnion:
2421    return EmitAggExprToLValue(E);
2422  case CK_BaseToDerived: {
2423    const RecordType *DerivedClassTy = E->getType()->getAs<RecordType>();
2424    CXXRecordDecl *DerivedClassDecl =
2425      cast<CXXRecordDecl>(DerivedClassTy->getDecl());
2426
2427    LValue LV = EmitLValue(E->getSubExpr());
2428
2429    // Perform the base-to-derived conversion
2430    llvm::Value *Derived =
2431      GetAddressOfDerivedClass(LV.getAddress(), DerivedClassDecl,
2432                               E->path_begin(), E->path_end(),
2433                               /*NullCheckValue=*/false);
2434
2435    return MakeAddrLValue(Derived, E->getType());
2436  }
2437  case CK_LValueBitCast: {
2438    // This must be a reinterpret_cast (or c-style equivalent).
2439    const ExplicitCastExpr *CE = cast<ExplicitCastExpr>(E);
2440
2441    LValue LV = EmitLValue(E->getSubExpr());
2442    llvm::Value *V = Builder.CreateBitCast(LV.getAddress(),
2443                                           ConvertType(CE->getTypeAsWritten()));
2444    return MakeAddrLValue(V, E->getType());
2445  }
2446  case CK_ObjCObjectLValueCast: {
2447    LValue LV = EmitLValue(E->getSubExpr());
2448    QualType ToType = getContext().getLValueReferenceType(E->getType());
2449    llvm::Value *V = Builder.CreateBitCast(LV.getAddress(),
2450                                           ConvertType(ToType));
2451    return MakeAddrLValue(V, E->getType());
2452  }
2453  }
2454
2455  llvm_unreachable("Unhandled lvalue cast kind?");
2456}
2457
2458LValue CodeGenFunction::EmitNullInitializationLValue(
2459                                              const CXXScalarValueInitExpr *E) {
2460  QualType Ty = E->getType();
2461  LValue LV = MakeAddrLValue(CreateMemTemp(Ty), Ty);
2462  EmitNullInitialization(LV.getAddress(), Ty);
2463  return LV;
2464}
2465
2466LValue CodeGenFunction::EmitOpaqueValueLValue(const OpaqueValueExpr *e) {
2467  assert(OpaqueValueMappingData::shouldBindAsLValue(e));
2468  return getOpaqueLValueMapping(e);
2469}
2470
2471LValue CodeGenFunction::EmitMaterializeTemporaryExpr(
2472                                           const MaterializeTemporaryExpr *E) {
2473  RValue RV = EmitReferenceBindingToExpr(E, /*InitializedDecl=*/0);
2474  return MakeAddrLValue(RV.getScalarVal(), E->getType());
2475}
2476
2477RValue CodeGenFunction::EmitRValueForField(LValue LV,
2478                                           const FieldDecl *FD) {
2479  QualType FT = FD->getType();
2480  LValue FieldLV = EmitLValueForField(LV, FD);
2481  if (FT->isAnyComplexType())
2482    return RValue::getComplex(
2483        LoadComplexFromAddr(FieldLV.getAddress(),
2484                            FieldLV.isVolatileQualified()));
2485  else if (CodeGenFunction::hasAggregateLLVMType(FT))
2486    return FieldLV.asAggregateRValue();
2487
2488  return EmitLoadOfLValue(FieldLV);
2489}
2490
2491//===--------------------------------------------------------------------===//
2492//                             Expression Emission
2493//===--------------------------------------------------------------------===//
2494
2495RValue CodeGenFunction::EmitCallExpr(const CallExpr *E,
2496                                     ReturnValueSlot ReturnValue) {
2497  if (CGDebugInfo *DI = getDebugInfo())
2498    DI->EmitLocation(Builder, E->getLocStart());
2499
2500  // Builtins never have block type.
2501  if (E->getCallee()->getType()->isBlockPointerType())
2502    return EmitBlockCallExpr(E, ReturnValue);
2503
2504  if (const CXXMemberCallExpr *CE = dyn_cast<CXXMemberCallExpr>(E))
2505    return EmitCXXMemberCallExpr(CE, ReturnValue);
2506
2507  if (const CUDAKernelCallExpr *CE = dyn_cast<CUDAKernelCallExpr>(E))
2508    return EmitCUDAKernelCallExpr(CE, ReturnValue);
2509
2510  const Decl *TargetDecl = E->getCalleeDecl();
2511  if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(TargetDecl)) {
2512    if (unsigned builtinID = FD->getBuiltinID())
2513      return EmitBuiltinExpr(FD, builtinID, E);
2514  }
2515
2516  if (const CXXOperatorCallExpr *CE = dyn_cast<CXXOperatorCallExpr>(E))
2517    if (const CXXMethodDecl *MD = dyn_cast_or_null<CXXMethodDecl>(TargetDecl))
2518      return EmitCXXOperatorMemberCallExpr(CE, MD, ReturnValue);
2519
2520  if (const CXXPseudoDestructorExpr *PseudoDtor
2521          = dyn_cast<CXXPseudoDestructorExpr>(E->getCallee()->IgnoreParens())) {
2522    QualType DestroyedType = PseudoDtor->getDestroyedType();
2523    if (getContext().getLangOpts().ObjCAutoRefCount &&
2524        DestroyedType->isObjCLifetimeType() &&
2525        (DestroyedType.getObjCLifetime() == Qualifiers::OCL_Strong ||
2526         DestroyedType.getObjCLifetime() == Qualifiers::OCL_Weak)) {
2527      // Automatic Reference Counting:
2528      //   If the pseudo-expression names a retainable object with weak or
2529      //   strong lifetime, the object shall be released.
2530      Expr *BaseExpr = PseudoDtor->getBase();
2531      llvm::Value *BaseValue = NULL;
2532      Qualifiers BaseQuals;
2533
2534      // If this is s.x, emit s as an lvalue. If it is s->x, emit s as a scalar.
2535      if (PseudoDtor->isArrow()) {
2536        BaseValue = EmitScalarExpr(BaseExpr);
2537        const PointerType *PTy = BaseExpr->getType()->getAs<PointerType>();
2538        BaseQuals = PTy->getPointeeType().getQualifiers();
2539      } else {
2540        LValue BaseLV = EmitLValue(BaseExpr);
2541        BaseValue = BaseLV.getAddress();
2542        QualType BaseTy = BaseExpr->getType();
2543        BaseQuals = BaseTy.getQualifiers();
2544      }
2545
2546      switch (PseudoDtor->getDestroyedType().getObjCLifetime()) {
2547      case Qualifiers::OCL_None:
2548      case Qualifiers::OCL_ExplicitNone:
2549      case Qualifiers::OCL_Autoreleasing:
2550        break;
2551
2552      case Qualifiers::OCL_Strong:
2553        EmitARCRelease(Builder.CreateLoad(BaseValue,
2554                          PseudoDtor->getDestroyedType().isVolatileQualified()),
2555                       /*precise*/ true);
2556        break;
2557
2558      case Qualifiers::OCL_Weak:
2559        EmitARCDestroyWeak(BaseValue);
2560        break;
2561      }
2562    } else {
2563      // C++ [expr.pseudo]p1:
2564      //   The result shall only be used as the operand for the function call
2565      //   operator (), and the result of such a call has type void. The only
2566      //   effect is the evaluation of the postfix-expression before the dot or
2567      //   arrow.
2568      EmitScalarExpr(E->getCallee());
2569    }
2570
2571    return RValue::get(0);
2572  }
2573
2574  llvm::Value *Callee = EmitScalarExpr(E->getCallee());
2575  return EmitCall(E->getCallee()->getType(), Callee, ReturnValue,
2576                  E->arg_begin(), E->arg_end(), TargetDecl);
2577}
2578
2579LValue CodeGenFunction::EmitBinaryOperatorLValue(const BinaryOperator *E) {
2580  // Comma expressions just emit their LHS then their RHS as an l-value.
2581  if (E->getOpcode() == BO_Comma) {
2582    EmitIgnoredExpr(E->getLHS());
2583    EnsureInsertPoint();
2584    return EmitLValue(E->getRHS());
2585  }
2586
2587  if (E->getOpcode() == BO_PtrMemD ||
2588      E->getOpcode() == BO_PtrMemI)
2589    return EmitPointerToDataMemberBinaryExpr(E);
2590
2591  assert(E->getOpcode() == BO_Assign && "unexpected binary l-value");
2592
2593  // Note that in all of these cases, __block variables need the RHS
2594  // evaluated first just in case the variable gets moved by the RHS.
2595
2596  if (!hasAggregateLLVMType(E->getType())) {
2597    switch (E->getLHS()->getType().getObjCLifetime()) {
2598    case Qualifiers::OCL_Strong:
2599      return EmitARCStoreStrong(E, /*ignored*/ false).first;
2600
2601    case Qualifiers::OCL_Autoreleasing:
2602      return EmitARCStoreAutoreleasing(E).first;
2603
2604    // No reason to do any of these differently.
2605    case Qualifiers::OCL_None:
2606    case Qualifiers::OCL_ExplicitNone:
2607    case Qualifiers::OCL_Weak:
2608      break;
2609    }
2610
2611    RValue RV = EmitAnyExpr(E->getRHS());
2612    LValue LV = EmitLValue(E->getLHS());
2613    EmitStoreThroughLValue(RV, LV);
2614    return LV;
2615  }
2616
2617  if (E->getType()->isAnyComplexType())
2618    return EmitComplexAssignmentLValue(E);
2619
2620  return EmitAggExprToLValue(E);
2621}
2622
2623LValue CodeGenFunction::EmitCallExprLValue(const CallExpr *E) {
2624  RValue RV = EmitCallExpr(E);
2625
2626  if (!RV.isScalar())
2627    return MakeAddrLValue(RV.getAggregateAddr(), E->getType());
2628
2629  assert(E->getCallReturnType()->isReferenceType() &&
2630         "Can't have a scalar return unless the return type is a "
2631         "reference type!");
2632
2633  return MakeAddrLValue(RV.getScalarVal(), E->getType());
2634}
2635
2636LValue CodeGenFunction::EmitVAArgExprLValue(const VAArgExpr *E) {
2637  // FIXME: This shouldn't require another copy.
2638  return EmitAggExprToLValue(E);
2639}
2640
2641LValue CodeGenFunction::EmitCXXConstructLValue(const CXXConstructExpr *E) {
2642  assert(E->getType()->getAsCXXRecordDecl()->hasTrivialDestructor()
2643         && "binding l-value to type which needs a temporary");
2644  AggValueSlot Slot = CreateAggTemp(E->getType());
2645  EmitCXXConstructExpr(E, Slot);
2646  return MakeAddrLValue(Slot.getAddr(), E->getType());
2647}
2648
2649LValue
2650CodeGenFunction::EmitCXXTypeidLValue(const CXXTypeidExpr *E) {
2651  return MakeAddrLValue(EmitCXXTypeidExpr(E), E->getType());
2652}
2653
2654LValue
2655CodeGenFunction::EmitCXXBindTemporaryLValue(const CXXBindTemporaryExpr *E) {
2656  AggValueSlot Slot = CreateAggTemp(E->getType(), "temp.lvalue");
2657  Slot.setExternallyDestructed();
2658  EmitAggExpr(E->getSubExpr(), Slot);
2659  EmitCXXTemporary(E->getTemporary(), E->getType(), Slot.getAddr());
2660  return MakeAddrLValue(Slot.getAddr(), E->getType());
2661}
2662
2663LValue
2664CodeGenFunction::EmitLambdaLValue(const LambdaExpr *E) {
2665  AggValueSlot Slot = CreateAggTemp(E->getType(), "temp.lvalue");
2666  EmitLambdaExpr(E, Slot);
2667  return MakeAddrLValue(Slot.getAddr(), E->getType());
2668}
2669
2670LValue CodeGenFunction::EmitObjCMessageExprLValue(const ObjCMessageExpr *E) {
2671  RValue RV = EmitObjCMessageExpr(E);
2672
2673  if (!RV.isScalar())
2674    return MakeAddrLValue(RV.getAggregateAddr(), E->getType());
2675
2676  assert(E->getMethodDecl()->getResultType()->isReferenceType() &&
2677         "Can't have a scalar return unless the return type is a "
2678         "reference type!");
2679
2680  return MakeAddrLValue(RV.getScalarVal(), E->getType());
2681}
2682
2683LValue CodeGenFunction::EmitObjCSelectorLValue(const ObjCSelectorExpr *E) {
2684  llvm::Value *V =
2685    CGM.getObjCRuntime().GetSelector(Builder, E->getSelector(), true);
2686  return MakeAddrLValue(V, E->getType());
2687}
2688
2689llvm::Value *CodeGenFunction::EmitIvarOffset(const ObjCInterfaceDecl *Interface,
2690                                             const ObjCIvarDecl *Ivar) {
2691  return CGM.getObjCRuntime().EmitIvarOffset(*this, Interface, Ivar);
2692}
2693
2694LValue CodeGenFunction::EmitLValueForIvar(QualType ObjectTy,
2695                                          llvm::Value *BaseValue,
2696                                          const ObjCIvarDecl *Ivar,
2697                                          unsigned CVRQualifiers) {
2698  return CGM.getObjCRuntime().EmitObjCValueForIvar(*this, ObjectTy, BaseValue,
2699                                                   Ivar, CVRQualifiers);
2700}
2701
2702LValue CodeGenFunction::EmitObjCIvarRefLValue(const ObjCIvarRefExpr *E) {
2703  // FIXME: A lot of the code below could be shared with EmitMemberExpr.
2704  llvm::Value *BaseValue = 0;
2705  const Expr *BaseExpr = E->getBase();
2706  Qualifiers BaseQuals;
2707  QualType ObjectTy;
2708  if (E->isArrow()) {
2709    BaseValue = EmitScalarExpr(BaseExpr);
2710    ObjectTy = BaseExpr->getType()->getPointeeType();
2711    BaseQuals = ObjectTy.getQualifiers();
2712  } else {
2713    LValue BaseLV = EmitLValue(BaseExpr);
2714    // FIXME: this isn't right for bitfields.
2715    BaseValue = BaseLV.getAddress();
2716    ObjectTy = BaseExpr->getType();
2717    BaseQuals = ObjectTy.getQualifiers();
2718  }
2719
2720  LValue LV =
2721    EmitLValueForIvar(ObjectTy, BaseValue, E->getDecl(),
2722                      BaseQuals.getCVRQualifiers());
2723  setObjCGCLValueClass(getContext(), E, LV);
2724  return LV;
2725}
2726
2727LValue CodeGenFunction::EmitStmtExprLValue(const StmtExpr *E) {
2728  // Can only get l-value for message expression returning aggregate type
2729  RValue RV = EmitAnyExprToTemp(E);
2730  return MakeAddrLValue(RV.getAggregateAddr(), E->getType());
2731}
2732
2733RValue CodeGenFunction::EmitCall(QualType CalleeType, llvm::Value *Callee,
2734                                 ReturnValueSlot ReturnValue,
2735                                 CallExpr::const_arg_iterator ArgBeg,
2736                                 CallExpr::const_arg_iterator ArgEnd,
2737                                 const Decl *TargetDecl) {
2738  // Get the actual function type. The callee type will always be a pointer to
2739  // function type or a block pointer type.
2740  assert(CalleeType->isFunctionPointerType() &&
2741         "Call must have function pointer type!");
2742
2743  CalleeType = getContext().getCanonicalType(CalleeType);
2744
2745  const FunctionType *FnType
2746    = cast<FunctionType>(cast<PointerType>(CalleeType)->getPointeeType());
2747
2748  CallArgList Args;
2749  EmitCallArgs(Args, dyn_cast<FunctionProtoType>(FnType), ArgBeg, ArgEnd);
2750
2751  const CGFunctionInfo &FnInfo =
2752    CGM.getTypes().arrangeFunctionCall(Args, FnType);
2753
2754  // C99 6.5.2.2p6:
2755  //   If the expression that denotes the called function has a type
2756  //   that does not include a prototype, [the default argument
2757  //   promotions are performed]. If the number of arguments does not
2758  //   equal the number of parameters, the behavior is undefined. If
2759  //   the function is defined with a type that includes a prototype,
2760  //   and either the prototype ends with an ellipsis (, ...) or the
2761  //   types of the arguments after promotion are not compatible with
2762  //   the types of the parameters, the behavior is undefined. If the
2763  //   function is defined with a type that does not include a
2764  //   prototype, and the types of the arguments after promotion are
2765  //   not compatible with those of the parameters after promotion,
2766  //   the behavior is undefined [except in some trivial cases].
2767  // That is, in the general case, we should assume that a call
2768  // through an unprototyped function type works like a *non-variadic*
2769  // call.  The way we make this work is to cast to the exact type
2770  // of the promoted arguments.
2771  if (isa<FunctionNoProtoType>(FnType) && !FnInfo.isVariadic()) {
2772    llvm::Type *CalleeTy = getTypes().GetFunctionType(FnInfo);
2773    CalleeTy = CalleeTy->getPointerTo();
2774    Callee = Builder.CreateBitCast(Callee, CalleeTy, "callee.knr.cast");
2775  }
2776
2777  return EmitCall(FnInfo, Callee, ReturnValue, Args, TargetDecl);
2778}
2779
2780LValue CodeGenFunction::
2781EmitPointerToDataMemberBinaryExpr(const BinaryOperator *E) {
2782  llvm::Value *BaseV;
2783  if (E->getOpcode() == BO_PtrMemI)
2784    BaseV = EmitScalarExpr(E->getLHS());
2785  else
2786    BaseV = EmitLValue(E->getLHS()).getAddress();
2787
2788  llvm::Value *OffsetV = EmitScalarExpr(E->getRHS());
2789
2790  const MemberPointerType *MPT
2791    = E->getRHS()->getType()->getAs<MemberPointerType>();
2792
2793  llvm::Value *AddV =
2794    CGM.getCXXABI().EmitMemberDataPointerAddress(*this, BaseV, OffsetV, MPT);
2795
2796  return MakeAddrLValue(AddV, MPT->getPointeeType());
2797}
2798
2799static void
2800EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *E, llvm::Value *Dest,
2801             llvm::Value *Ptr, llvm::Value *Val1, llvm::Value *Val2,
2802             uint64_t Size, unsigned Align, llvm::AtomicOrdering Order) {
2803  llvm::AtomicRMWInst::BinOp Op = llvm::AtomicRMWInst::Add;
2804  llvm::Instruction::BinaryOps PostOp = (llvm::Instruction::BinaryOps)0;
2805
2806  switch (E->getOp()) {
2807  case AtomicExpr::AO__c11_atomic_init:
2808    llvm_unreachable("Already handled!");
2809
2810  case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
2811  case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
2812  case AtomicExpr::AO__atomic_compare_exchange:
2813  case AtomicExpr::AO__atomic_compare_exchange_n: {
2814    // Note that cmpxchg only supports specifying one ordering and
2815    // doesn't support weak cmpxchg, at least at the moment.
2816    llvm::LoadInst *LoadVal1 = CGF.Builder.CreateLoad(Val1);
2817    LoadVal1->setAlignment(Align);
2818    llvm::LoadInst *LoadVal2 = CGF.Builder.CreateLoad(Val2);
2819    LoadVal2->setAlignment(Align);
2820    llvm::AtomicCmpXchgInst *CXI =
2821        CGF.Builder.CreateAtomicCmpXchg(Ptr, LoadVal1, LoadVal2, Order);
2822    CXI->setVolatile(E->isVolatile());
2823    llvm::StoreInst *StoreVal1 = CGF.Builder.CreateStore(CXI, Val1);
2824    StoreVal1->setAlignment(Align);
2825    llvm::Value *Cmp = CGF.Builder.CreateICmpEQ(CXI, LoadVal1);
2826    CGF.EmitStoreOfScalar(Cmp, CGF.MakeAddrLValue(Dest, E->getType()));
2827    return;
2828  }
2829
2830  case AtomicExpr::AO__c11_atomic_load:
2831  case AtomicExpr::AO__atomic_load_n:
2832  case AtomicExpr::AO__atomic_load: {
2833    llvm::LoadInst *Load = CGF.Builder.CreateLoad(Ptr);
2834    Load->setAtomic(Order);
2835    Load->setAlignment(Size);
2836    Load->setVolatile(E->isVolatile());
2837    llvm::StoreInst *StoreDest = CGF.Builder.CreateStore(Load, Dest);
2838    StoreDest->setAlignment(Align);
2839    return;
2840  }
2841
2842  case AtomicExpr::AO__c11_atomic_store:
2843  case AtomicExpr::AO__atomic_store:
2844  case AtomicExpr::AO__atomic_store_n: {
2845    assert(!Dest && "Store does not return a value");
2846    llvm::LoadInst *LoadVal1 = CGF.Builder.CreateLoad(Val1);
2847    LoadVal1->setAlignment(Align);
2848    llvm::StoreInst *Store = CGF.Builder.CreateStore(LoadVal1, Ptr);
2849    Store->setAtomic(Order);
2850    Store->setAlignment(Size);
2851    Store->setVolatile(E->isVolatile());
2852    return;
2853  }
2854
2855  case AtomicExpr::AO__c11_atomic_exchange:
2856  case AtomicExpr::AO__atomic_exchange_n:
2857  case AtomicExpr::AO__atomic_exchange:
2858    Op = llvm::AtomicRMWInst::Xchg;
2859    break;
2860
2861  case AtomicExpr::AO__atomic_add_fetch:
2862    PostOp = llvm::Instruction::Add;
2863    // Fall through.
2864  case AtomicExpr::AO__c11_atomic_fetch_add:
2865  case AtomicExpr::AO__atomic_fetch_add:
2866    Op = llvm::AtomicRMWInst::Add;
2867    break;
2868
2869  case AtomicExpr::AO__atomic_sub_fetch:
2870    PostOp = llvm::Instruction::Sub;
2871    // Fall through.
2872  case AtomicExpr::AO__c11_atomic_fetch_sub:
2873  case AtomicExpr::AO__atomic_fetch_sub:
2874    Op = llvm::AtomicRMWInst::Sub;
2875    break;
2876
2877  case AtomicExpr::AO__atomic_and_fetch:
2878    PostOp = llvm::Instruction::And;
2879    // Fall through.
2880  case AtomicExpr::AO__c11_atomic_fetch_and:
2881  case AtomicExpr::AO__atomic_fetch_and:
2882    Op = llvm::AtomicRMWInst::And;
2883    break;
2884
2885  case AtomicExpr::AO__atomic_or_fetch:
2886    PostOp = llvm::Instruction::Or;
2887    // Fall through.
2888  case AtomicExpr::AO__c11_atomic_fetch_or:
2889  case AtomicExpr::AO__atomic_fetch_or:
2890    Op = llvm::AtomicRMWInst::Or;
2891    break;
2892
2893  case AtomicExpr::AO__atomic_xor_fetch:
2894    PostOp = llvm::Instruction::Xor;
2895    // Fall through.
2896  case AtomicExpr::AO__c11_atomic_fetch_xor:
2897  case AtomicExpr::AO__atomic_fetch_xor:
2898    Op = llvm::AtomicRMWInst::Xor;
2899    break;
2900
2901  case AtomicExpr::AO__atomic_nand_fetch:
2902    PostOp = llvm::Instruction::And;
2903    // Fall through.
2904  case AtomicExpr::AO__atomic_fetch_nand:
2905    Op = llvm::AtomicRMWInst::Nand;
2906    break;
2907  }
2908
2909  llvm::LoadInst *LoadVal1 = CGF.Builder.CreateLoad(Val1);
2910  LoadVal1->setAlignment(Align);
2911  llvm::AtomicRMWInst *RMWI =
2912      CGF.Builder.CreateAtomicRMW(Op, Ptr, LoadVal1, Order);
2913  RMWI->setVolatile(E->isVolatile());
2914
2915  // For __atomic_*_fetch operations, perform the operation again to
2916  // determine the value which was written.
2917  llvm::Value *Result = RMWI;
2918  if (PostOp)
2919    Result = CGF.Builder.CreateBinOp(PostOp, RMWI, LoadVal1);
2920  if (E->getOp() == AtomicExpr::AO__atomic_nand_fetch)
2921    Result = CGF.Builder.CreateNot(Result);
2922  llvm::StoreInst *StoreDest = CGF.Builder.CreateStore(Result, Dest);
2923  StoreDest->setAlignment(Align);
2924}
2925
2926// This function emits any expression (scalar, complex, or aggregate)
2927// into a temporary alloca.
2928static llvm::Value *
2929EmitValToTemp(CodeGenFunction &CGF, Expr *E) {
2930  llvm::Value *DeclPtr = CGF.CreateMemTemp(E->getType(), ".atomictmp");
2931  CGF.EmitAnyExprToMem(E, DeclPtr, E->getType().getQualifiers(),
2932                       /*Init*/ true);
2933  return DeclPtr;
2934}
2935
2936static RValue ConvertTempToRValue(CodeGenFunction &CGF, QualType Ty,
2937                                  llvm::Value *Dest) {
2938  if (Ty->isAnyComplexType())
2939    return RValue::getComplex(CGF.LoadComplexFromAddr(Dest, false));
2940  if (CGF.hasAggregateLLVMType(Ty))
2941    return RValue::getAggregate(Dest);
2942  return RValue::get(CGF.EmitLoadOfScalar(CGF.MakeAddrLValue(Dest, Ty)));
2943}
2944
2945RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E, llvm::Value *Dest) {
2946  QualType AtomicTy = E->getPtr()->getType()->getPointeeType();
2947  QualType MemTy = AtomicTy;
2948  if (const AtomicType *AT = AtomicTy->getAs<AtomicType>())
2949    MemTy = AT->getValueType();
2950  CharUnits sizeChars = getContext().getTypeSizeInChars(AtomicTy);
2951  uint64_t Size = sizeChars.getQuantity();
2952  CharUnits alignChars = getContext().getTypeAlignInChars(AtomicTy);
2953  unsigned Align = alignChars.getQuantity();
2954  unsigned MaxInlineWidth =
2955      getContext().getTargetInfo().getMaxAtomicInlineWidth();
2956  bool UseLibcall = (Size != Align || Size > MaxInlineWidth);
2957
2958
2959
2960  llvm::Value *Ptr, *Order, *OrderFail = 0, *Val1 = 0, *Val2 = 0;
2961  Ptr = EmitScalarExpr(E->getPtr());
2962
2963  if (E->getOp() == AtomicExpr::AO__c11_atomic_init) {
2964    assert(!Dest && "Init does not return a value");
2965    if (!hasAggregateLLVMType(E->getVal1()->getType())) {
2966      QualType PointeeType
2967        = E->getPtr()->getType()->getAs<PointerType>()->getPointeeType();
2968      EmitScalarInit(EmitScalarExpr(E->getVal1()),
2969                     LValue::MakeAddr(Ptr, PointeeType, alignChars,
2970                                      getContext()));
2971    } else if (E->getType()->isAnyComplexType()) {
2972      EmitComplexExprIntoAddr(E->getVal1(), Ptr, E->isVolatile());
2973    } else {
2974      AggValueSlot Slot = AggValueSlot::forAddr(Ptr, alignChars,
2975                                        AtomicTy.getQualifiers(),
2976                                        AggValueSlot::IsNotDestructed,
2977                                        AggValueSlot::DoesNotNeedGCBarriers,
2978                                        AggValueSlot::IsNotAliased);
2979      EmitAggExpr(E->getVal1(), Slot);
2980    }
2981    return RValue::get(0);
2982  }
2983
2984  Order = EmitScalarExpr(E->getOrder());
2985
2986  switch (E->getOp()) {
2987  case AtomicExpr::AO__c11_atomic_init:
2988    llvm_unreachable("Already handled!");
2989
2990  case AtomicExpr::AO__c11_atomic_load:
2991  case AtomicExpr::AO__atomic_load_n:
2992    break;
2993
2994  case AtomicExpr::AO__atomic_load:
2995    Dest = EmitScalarExpr(E->getVal1());
2996    break;
2997
2998  case AtomicExpr::AO__atomic_store:
2999    Val1 = EmitScalarExpr(E->getVal1());
3000    break;
3001
3002  case AtomicExpr::AO__atomic_exchange:
3003    Val1 = EmitScalarExpr(E->getVal1());
3004    Dest = EmitScalarExpr(E->getVal2());
3005    break;
3006
3007  case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
3008  case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
3009  case AtomicExpr::AO__atomic_compare_exchange_n:
3010  case AtomicExpr::AO__atomic_compare_exchange:
3011    Val1 = EmitScalarExpr(E->getVal1());
3012    if (E->getOp() == AtomicExpr::AO__atomic_compare_exchange)
3013      Val2 = EmitScalarExpr(E->getVal2());
3014    else
3015      Val2 = EmitValToTemp(*this, E->getVal2());
3016    OrderFail = EmitScalarExpr(E->getOrderFail());
3017    // Evaluate and discard the 'weak' argument.
3018    if (E->getNumSubExprs() == 6)
3019      EmitScalarExpr(E->getWeak());
3020    break;
3021
3022  case AtomicExpr::AO__c11_atomic_fetch_add:
3023  case AtomicExpr::AO__c11_atomic_fetch_sub:
3024    if (MemTy->isPointerType()) {
3025      // For pointer arithmetic, we're required to do a bit of math:
3026      // adding 1 to an int* is not the same as adding 1 to a uintptr_t.
3027      // ... but only for the C11 builtins. The GNU builtins expect the
3028      // user to multiply by sizeof(T).
3029      QualType Val1Ty = E->getVal1()->getType();
3030      llvm::Value *Val1Scalar = EmitScalarExpr(E->getVal1());
3031      CharUnits PointeeIncAmt =
3032          getContext().getTypeSizeInChars(MemTy->getPointeeType());
3033      Val1Scalar = Builder.CreateMul(Val1Scalar, CGM.getSize(PointeeIncAmt));
3034      Val1 = CreateMemTemp(Val1Ty, ".atomictmp");
3035      EmitStoreOfScalar(Val1Scalar, MakeAddrLValue(Val1, Val1Ty));
3036      break;
3037    }
3038    // Fall through.
3039  case AtomicExpr::AO__atomic_fetch_add:
3040  case AtomicExpr::AO__atomic_fetch_sub:
3041  case AtomicExpr::AO__atomic_add_fetch:
3042  case AtomicExpr::AO__atomic_sub_fetch:
3043  case AtomicExpr::AO__c11_atomic_store:
3044  case AtomicExpr::AO__c11_atomic_exchange:
3045  case AtomicExpr::AO__atomic_store_n:
3046  case AtomicExpr::AO__atomic_exchange_n:
3047  case AtomicExpr::AO__c11_atomic_fetch_and:
3048  case AtomicExpr::AO__c11_atomic_fetch_or:
3049  case AtomicExpr::AO__c11_atomic_fetch_xor:
3050  case AtomicExpr::AO__atomic_fetch_and:
3051  case AtomicExpr::AO__atomic_fetch_or:
3052  case AtomicExpr::AO__atomic_fetch_xor:
3053  case AtomicExpr::AO__atomic_fetch_nand:
3054  case AtomicExpr::AO__atomic_and_fetch:
3055  case AtomicExpr::AO__atomic_or_fetch:
3056  case AtomicExpr::AO__atomic_xor_fetch:
3057  case AtomicExpr::AO__atomic_nand_fetch:
3058    Val1 = EmitValToTemp(*this, E->getVal1());
3059    break;
3060  }
3061
3062  if (!E->getType()->isVoidType() && !Dest)
3063    Dest = CreateMemTemp(E->getType(), ".atomicdst");
3064
3065  // Use a library call.  See: http://gcc.gnu.org/wiki/Atomic/GCCMM/LIbrary .
3066  if (UseLibcall) {
3067
3068    llvm::SmallVector<QualType, 5> Params;
3069    CallArgList Args;
3070    // Size is always the first parameter
3071    Args.add(RValue::get(llvm::ConstantInt::get(SizeTy, Size)),
3072             getContext().getSizeType());
3073    // Atomic address is always the second parameter
3074    Args.add(RValue::get(EmitCastToVoidPtr(Ptr)),
3075             getContext().VoidPtrTy);
3076
3077    const char* LibCallName;
3078    QualType RetTy = getContext().VoidTy;
3079    switch (E->getOp()) {
3080    // There is only one libcall for compare an exchange, because there is no
3081    // optimisation benefit possible from a libcall version of a weak compare
3082    // and exchange.
3083    // bool __atomic_compare_exchange(size_t size, void *obj, void *expected,
3084    //                                void *desired, int success, int failure)
3085    case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
3086    case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
3087    case AtomicExpr::AO__atomic_compare_exchange:
3088    case AtomicExpr::AO__atomic_compare_exchange_n:
3089      LibCallName = "__atomic_compare_exchange";
3090      RetTy = getContext().BoolTy;
3091      Args.add(RValue::get(EmitCastToVoidPtr(Val1)),
3092               getContext().VoidPtrTy);
3093      Args.add(RValue::get(EmitCastToVoidPtr(Val2)),
3094               getContext().VoidPtrTy);
3095      Args.add(RValue::get(Order),
3096               getContext().IntTy);
3097      Order = OrderFail;
3098      break;
3099    // void __atomic_exchange(size_t size, void *mem, void *val, void *return,
3100    //                        int order)
3101    case AtomicExpr::AO__c11_atomic_exchange:
3102    case AtomicExpr::AO__atomic_exchange_n:
3103    case AtomicExpr::AO__atomic_exchange:
3104      LibCallName = "__atomic_exchange";
3105      Args.add(RValue::get(EmitCastToVoidPtr(Val1)),
3106               getContext().VoidPtrTy);
3107      Args.add(RValue::get(EmitCastToVoidPtr(Dest)),
3108               getContext().VoidPtrTy);
3109      break;
3110    // void __atomic_store(size_t size, void *mem, void *val, int order)
3111    case AtomicExpr::AO__c11_atomic_store:
3112    case AtomicExpr::AO__atomic_store:
3113    case AtomicExpr::AO__atomic_store_n:
3114      LibCallName = "__atomic_store";
3115      Args.add(RValue::get(EmitCastToVoidPtr(Val1)),
3116               getContext().VoidPtrTy);
3117      break;
3118    // void __atomic_load(size_t size, void *mem, void *return, int order)
3119    case AtomicExpr::AO__c11_atomic_load:
3120    case AtomicExpr::AO__atomic_load:
3121    case AtomicExpr::AO__atomic_load_n:
3122      LibCallName = "__atomic_load";
3123      Args.add(RValue::get(EmitCastToVoidPtr(Dest)),
3124               getContext().VoidPtrTy);
3125      break;
3126#if 0
3127    // These are only defined for 1-16 byte integers.  It is not clear what
3128    // their semantics would be on anything else...
3129    case AtomicExpr::Add:   LibCallName = "__atomic_fetch_add_generic"; break;
3130    case AtomicExpr::Sub:   LibCallName = "__atomic_fetch_sub_generic"; break;
3131    case AtomicExpr::And:   LibCallName = "__atomic_fetch_and_generic"; break;
3132    case AtomicExpr::Or:    LibCallName = "__atomic_fetch_or_generic"; break;
3133    case AtomicExpr::Xor:   LibCallName = "__atomic_fetch_xor_generic"; break;
3134#endif
3135    default: return EmitUnsupportedRValue(E, "atomic library call");
3136    }
3137    // order is always the last parameter
3138    Args.add(RValue::get(Order),
3139             getContext().IntTy);
3140
3141    const CGFunctionInfo &FuncInfo =
3142        CGM.getTypes().arrangeFunctionCall(RetTy, Args,
3143            FunctionType::ExtInfo(), RequiredArgs::All);
3144    llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(FuncInfo);
3145    llvm::Constant *Func = CGM.CreateRuntimeFunction(FTy, LibCallName);
3146    RValue Res = EmitCall(FuncInfo, Func, ReturnValueSlot(), Args);
3147    if (E->isCmpXChg())
3148      return Res;
3149    if (E->getType()->isVoidType())
3150      return RValue::get(0);
3151    return ConvertTempToRValue(*this, E->getType(), Dest);
3152  }
3153
3154  llvm::Type *IPtrTy =
3155      llvm::IntegerType::get(getLLVMContext(), Size * 8)->getPointerTo();
3156  llvm::Value *OrigDest = Dest;
3157  Ptr = Builder.CreateBitCast(Ptr, IPtrTy);
3158  if (Val1) Val1 = Builder.CreateBitCast(Val1, IPtrTy);
3159  if (Val2) Val2 = Builder.CreateBitCast(Val2, IPtrTy);
3160  if (Dest && !E->isCmpXChg()) Dest = Builder.CreateBitCast(Dest, IPtrTy);
3161
3162  if (isa<llvm::ConstantInt>(Order)) {
3163    int ord = cast<llvm::ConstantInt>(Order)->getZExtValue();
3164    switch (ord) {
3165    case 0:  // memory_order_relaxed
3166      EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, Size, Align,
3167                   llvm::Monotonic);
3168      break;
3169    case 1:  // memory_order_consume
3170    case 2:  // memory_order_acquire
3171      EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, Size, Align,
3172                   llvm::Acquire);
3173      break;
3174    case 3:  // memory_order_release
3175      EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, Size, Align,
3176                   llvm::Release);
3177      break;
3178    case 4:  // memory_order_acq_rel
3179      EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, Size, Align,
3180                   llvm::AcquireRelease);
3181      break;
3182    case 5:  // memory_order_seq_cst
3183      EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, Size, Align,
3184                   llvm::SequentiallyConsistent);
3185      break;
3186    default: // invalid order
3187      // We should not ever get here normally, but it's hard to
3188      // enforce that in general.
3189      break;
3190    }
3191    if (E->getType()->isVoidType())
3192      return RValue::get(0);
3193    return ConvertTempToRValue(*this, E->getType(), OrigDest);
3194  }
3195
3196  // Long case, when Order isn't obviously constant.
3197
3198  bool IsStore = E->getOp() == AtomicExpr::AO__c11_atomic_store ||
3199                 E->getOp() == AtomicExpr::AO__atomic_store ||
3200                 E->getOp() == AtomicExpr::AO__atomic_store_n;
3201  bool IsLoad = E->getOp() == AtomicExpr::AO__c11_atomic_load ||
3202                E->getOp() == AtomicExpr::AO__atomic_load ||
3203                E->getOp() == AtomicExpr::AO__atomic_load_n;
3204
3205  // Create all the relevant BB's
3206  llvm::BasicBlock *MonotonicBB = 0, *AcquireBB = 0, *ReleaseBB = 0,
3207                   *AcqRelBB = 0, *SeqCstBB = 0;
3208  MonotonicBB = createBasicBlock("monotonic", CurFn);
3209  if (!IsStore)
3210    AcquireBB = createBasicBlock("acquire", CurFn);
3211  if (!IsLoad)
3212    ReleaseBB = createBasicBlock("release", CurFn);
3213  if (!IsLoad && !IsStore)
3214    AcqRelBB = createBasicBlock("acqrel", CurFn);
3215  SeqCstBB = createBasicBlock("seqcst", CurFn);
3216  llvm::BasicBlock *ContBB = createBasicBlock("atomic.continue", CurFn);
3217
3218  // Create the switch for the split
3219  // MonotonicBB is arbitrarily chosen as the default case; in practice, this
3220  // doesn't matter unless someone is crazy enough to use something that
3221  // doesn't fold to a constant for the ordering.
3222  Order = Builder.CreateIntCast(Order, Builder.getInt32Ty(), false);
3223  llvm::SwitchInst *SI = Builder.CreateSwitch(Order, MonotonicBB);
3224
3225  // Emit all the different atomics
3226  Builder.SetInsertPoint(MonotonicBB);
3227  EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, Size, Align,
3228               llvm::Monotonic);
3229  Builder.CreateBr(ContBB);
3230  if (!IsStore) {
3231    Builder.SetInsertPoint(AcquireBB);
3232    EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, Size, Align,
3233                 llvm::Acquire);
3234    Builder.CreateBr(ContBB);
3235    SI->addCase(Builder.getInt32(1), AcquireBB);
3236    SI->addCase(Builder.getInt32(2), AcquireBB);
3237  }
3238  if (!IsLoad) {
3239    Builder.SetInsertPoint(ReleaseBB);
3240    EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, Size, Align,
3241                 llvm::Release);
3242    Builder.CreateBr(ContBB);
3243    SI->addCase(Builder.getInt32(3), ReleaseBB);
3244  }
3245  if (!IsLoad && !IsStore) {
3246    Builder.SetInsertPoint(AcqRelBB);
3247    EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, Size, Align,
3248                 llvm::AcquireRelease);
3249    Builder.CreateBr(ContBB);
3250    SI->addCase(Builder.getInt32(4), AcqRelBB);
3251  }
3252  Builder.SetInsertPoint(SeqCstBB);
3253  EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, Size, Align,
3254               llvm::SequentiallyConsistent);
3255  Builder.CreateBr(ContBB);
3256  SI->addCase(Builder.getInt32(5), SeqCstBB);
3257
3258  // Cleanup and return
3259  Builder.SetInsertPoint(ContBB);
3260  if (E->getType()->isVoidType())
3261    return RValue::get(0);
3262  return ConvertTempToRValue(*this, E->getType(), OrigDest);
3263}
3264
3265void CodeGenFunction::SetFPAccuracy(llvm::Value *Val, float Accuracy) {
3266  assert(Val->getType()->isFPOrFPVectorTy());
3267  if (Accuracy == 0.0 || !isa<llvm::Instruction>(Val))
3268    return;
3269
3270  llvm::MDBuilder MDHelper(getLLVMContext());
3271  llvm::MDNode *Node = MDHelper.createFPMath(Accuracy);
3272
3273  cast<llvm::Instruction>(Val)->setMetadata(llvm::LLVMContext::MD_fpmath, Node);
3274}
3275
3276namespace {
3277  struct LValueOrRValue {
3278    LValue LV;
3279    RValue RV;
3280  };
3281}
3282
3283static LValueOrRValue emitPseudoObjectExpr(CodeGenFunction &CGF,
3284                                           const PseudoObjectExpr *E,
3285                                           bool forLValue,
3286                                           AggValueSlot slot) {
3287  llvm::SmallVector<CodeGenFunction::OpaqueValueMappingData, 4> opaques;
3288
3289  // Find the result expression, if any.
3290  const Expr *resultExpr = E->getResultExpr();
3291  LValueOrRValue result;
3292
3293  for (PseudoObjectExpr::const_semantics_iterator
3294         i = E->semantics_begin(), e = E->semantics_end(); i != e; ++i) {
3295    const Expr *semantic = *i;
3296
3297    // If this semantic expression is an opaque value, bind it
3298    // to the result of its source expression.
3299    if (const OpaqueValueExpr *ov = dyn_cast<OpaqueValueExpr>(semantic)) {
3300
3301      // If this is the result expression, we may need to evaluate
3302      // directly into the slot.
3303      typedef CodeGenFunction::OpaqueValueMappingData OVMA;
3304      OVMA opaqueData;
3305      if (ov == resultExpr && ov->isRValue() && !forLValue &&
3306          CodeGenFunction::hasAggregateLLVMType(ov->getType()) &&
3307          !ov->getType()->isAnyComplexType()) {
3308        CGF.EmitAggExpr(ov->getSourceExpr(), slot);
3309
3310        LValue LV = CGF.MakeAddrLValue(slot.getAddr(), ov->getType());
3311        opaqueData = OVMA::bind(CGF, ov, LV);
3312        result.RV = slot.asRValue();
3313
3314      // Otherwise, emit as normal.
3315      } else {
3316        opaqueData = OVMA::bind(CGF, ov, ov->getSourceExpr());
3317
3318        // If this is the result, also evaluate the result now.
3319        if (ov == resultExpr) {
3320          if (forLValue)
3321            result.LV = CGF.EmitLValue(ov);
3322          else
3323            result.RV = CGF.EmitAnyExpr(ov, slot);
3324        }
3325      }
3326
3327      opaques.push_back(opaqueData);
3328
3329    // Otherwise, if the expression is the result, evaluate it
3330    // and remember the result.
3331    } else if (semantic == resultExpr) {
3332      if (forLValue)
3333        result.LV = CGF.EmitLValue(semantic);
3334      else
3335        result.RV = CGF.EmitAnyExpr(semantic, slot);
3336
3337    // Otherwise, evaluate the expression in an ignored context.
3338    } else {
3339      CGF.EmitIgnoredExpr(semantic);
3340    }
3341  }
3342
3343  // Unbind all the opaques now.
3344  for (unsigned i = 0, e = opaques.size(); i != e; ++i)
3345    opaques[i].unbind(CGF);
3346
3347  return result;
3348}
3349
3350RValue CodeGenFunction::EmitPseudoObjectRValue(const PseudoObjectExpr *E,
3351                                               AggValueSlot slot) {
3352  return emitPseudoObjectExpr(*this, E, false, slot).RV;
3353}
3354
3355LValue CodeGenFunction::EmitPseudoObjectLValue(const PseudoObjectExpr *E) {
3356  return emitPseudoObjectExpr(*this, E, true, AggValueSlot::ignored()).LV;
3357}
3358