CGExpr.cpp revision 7ac9ef1c82c779a5348ed11b3d10e01c58803b35 
1//===--- CGExpr.cpp - Emit LLVM Code from Expressions ---------------------===//
2//
3//                     The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10// This contains code to emit Expr nodes as LLVM code.
11//
12//===----------------------------------------------------------------------===//
13
14#include "CodeGenFunction.h"
15#include "CodeGenModule.h"
16#include "CGCall.h"
17#include "CGCXXABI.h"
18#include "CGDebugInfo.h"
19#include "CGRecordLayout.h"
20#include "CGObjCRuntime.h"
21#include "TargetInfo.h"
22#include "clang/AST/ASTContext.h"
23#include "clang/AST/DeclObjC.h"
24#include "clang/Basic/ConvertUTF.h"
25#include "clang/Frontend/CodeGenOptions.h"
26#include "llvm/Intrinsics.h"
27#include "llvm/LLVMContext.h"
28#include "llvm/MDBuilder.h"
29#include "llvm/Target/TargetData.h"
30using namespace clang;
31using namespace CodeGen;
32
33//===--------------------------------------------------------------------===//
34//                        Miscellaneous Helper Methods
35//===--------------------------------------------------------------------===//
36
37llvm::Value *CodeGenFunction::EmitCastToVoidPtr(llvm::Value *value) {
38  unsigned addressSpace =
39    cast<llvm::PointerType>(value->getType())->getAddressSpace();
40
41  llvm::PointerType *destType = Int8PtrTy;
42  if (addressSpace)
43    destType = llvm::Type::getInt8PtrTy(getLLVMContext(), addressSpace);
44
45  if (value->getType() == destType) return value;
46  return Builder.CreateBitCast(value, destType);
47}
48
49/// CreateTempAlloca - This creates a alloca and inserts it into the entry
50/// block.
51llvm::AllocaInst *CodeGenFunction::CreateTempAlloca(llvm::Type *Ty,
52                                                    const Twine &Name) {
53  if (!Builder.isNamePreserving())
54    return new llvm::AllocaInst(Ty, 0, "", AllocaInsertPt);
55  return new llvm::AllocaInst(Ty, 0, Name, AllocaInsertPt);
56}
57
58void CodeGenFunction::InitTempAlloca(llvm::AllocaInst *Var,
59                                     llvm::Value *Init) {
60  llvm::StoreInst *Store = new llvm::StoreInst(Init, Var);
61  llvm::BasicBlock *Block = AllocaInsertPt->getParent();
62  Block->getInstList().insertAfter(&*AllocaInsertPt, Store);
63}
64
65llvm::AllocaInst *CodeGenFunction::CreateIRTemp(QualType Ty,
66                                                const Twine &Name) {
67  llvm::AllocaInst *Alloc = CreateTempAlloca(ConvertType(Ty), Name);
68  // FIXME: Should we prefer the preferred type alignment here?
69  CharUnits Align = getContext().getTypeAlignInChars(Ty);
70  Alloc->setAlignment(Align.getQuantity());
71  return Alloc;
72}
73
74llvm::AllocaInst *CodeGenFunction::CreateMemTemp(QualType Ty,
75                                                 const Twine &Name) {
76  llvm::AllocaInst *Alloc = CreateTempAlloca(ConvertTypeForMem(Ty), Name);
77  // FIXME: Should we prefer the preferred type alignment here?
78  CharUnits Align = getContext().getTypeAlignInChars(Ty);
79  Alloc->setAlignment(Align.getQuantity());
80  return Alloc;
81}
82
83/// EvaluateExprAsBool - Perform the usual unary conversions on the specified
84/// expression and compare the result against zero, returning an Int1Ty value.
85llvm::Value *CodeGenFunction::EvaluateExprAsBool(const Expr *E) {
86  if (const MemberPointerType *MPT = E->getType()->getAs<MemberPointerType>()) {
87    llvm::Value *MemPtr = EmitScalarExpr(E);
88    return CGM.getCXXABI().EmitMemberPointerIsNotNull(*this, MemPtr, MPT);
89  }
90
91  QualType BoolTy = getContext().BoolTy;
92  if (!E->getType()->isAnyComplexType())
93    return EmitScalarConversion(EmitScalarExpr(E), E->getType(), BoolTy);
94
95  return EmitComplexToScalarConversion(EmitComplexExpr(E), E->getType(),BoolTy);
96}
97
98/// EmitIgnoredExpr - Emit code to compute the specified expression,
99/// ignoring the result.
100void CodeGenFunction::EmitIgnoredExpr(const Expr *E) {
101  if (E->isRValue())
102    return (void) EmitAnyExpr(E, AggValueSlot::ignored(), true);
103
104  // Just emit it as an l-value and drop the result.
105  EmitLValue(E);
106}
107
108/// EmitAnyExpr - Emit code to compute the specified expression which
109/// can have any type.  The result is returned as an RValue struct.
110/// If this is an aggregate expression, AggSlot indicates where the
111/// result should be returned.
112RValue CodeGenFunction::EmitAnyExpr(const Expr *E,
113                                    AggValueSlot aggSlot,
114                                    bool ignoreResult) {
115  if (!hasAggregateLLVMType(E->getType()))
116    return RValue::get(EmitScalarExpr(E, ignoreResult));
117  else if (E->getType()->isAnyComplexType())
118    return RValue::getComplex(EmitComplexExpr(E, ignoreResult, ignoreResult));
119
120  if (!ignoreResult && aggSlot.isIgnored())
121    aggSlot = CreateAggTemp(E->getType(), "agg-temp");
122  EmitAggExpr(E, aggSlot);
123  return aggSlot.asRValue();
124}
125
126/// EmitAnyExprToTemp - Similary to EmitAnyExpr(), however, the result will
127/// always be accessible even if no aggregate location is provided.
128RValue CodeGenFunction::EmitAnyExprToTemp(const Expr *E) {
129  AggValueSlot AggSlot = AggValueSlot::ignored();
130
131  if (hasAggregateLLVMType(E->getType()) &&
132      !E->getType()->isAnyComplexType())
133    AggSlot = CreateAggTemp(E->getType(), "agg.tmp");
134  return EmitAnyExpr(E, AggSlot);
135}
136
137/// EmitAnyExprToMem - Evaluate an expression into a given memory
138/// location.
139void CodeGenFunction::EmitAnyExprToMem(const Expr *E,
140                                       llvm::Value *Location,
141                                       Qualifiers Quals,
142                                       bool IsInit) {
143  // FIXME: This function should take an LValue as an argument.
144  if (E->getType()->isAnyComplexType()) {
145    EmitComplexExprIntoAddr(E, Location, Quals.hasVolatile());
146  } else if (hasAggregateLLVMType(E->getType())) {
147    CharUnits Alignment = getContext().getTypeAlignInChars(E->getType());
148    EmitAggExpr(E, AggValueSlot::forAddr(Location, Alignment, Quals,
149                                         AggValueSlot::IsDestructed_t(IsInit),
150                                         AggValueSlot::DoesNotNeedGCBarriers,
151                                         AggValueSlot::IsAliased_t(!IsInit)));
152  } else {
153    RValue RV = RValue::get(EmitScalarExpr(E, /*Ignore*/ false));
154    LValue LV = MakeAddrLValue(Location, E->getType());
155    EmitStoreThroughLValue(RV, LV);
156  }
157}
158
159namespace {
160/// \brief An adjustment to be made to the temporary created when emitting a
161/// reference binding, which accesses a particular subobject of that temporary.
162  struct SubobjectAdjustment {
163    enum {
164      DerivedToBaseAdjustment,
165      FieldAdjustment,
166      MemberPointerAdjustment
167    } Kind;
168
169    union {
170      struct {
171        const CastExpr *BasePath;
172        const CXXRecordDecl *DerivedClass;
173      } DerivedToBase;
174
175      FieldDecl *Field;
176
177      struct {
178        const MemberPointerType *MPT;
179        llvm::Value *Ptr;
180      } Ptr;
181    };
182
183    SubobjectAdjustment(const CastExpr *BasePath,
184                        const CXXRecordDecl *DerivedClass)
185      : Kind(DerivedToBaseAdjustment) {
186      DerivedToBase.BasePath = BasePath;
187      DerivedToBase.DerivedClass = DerivedClass;
188    }
189
190    SubobjectAdjustment(FieldDecl *Field)
191      : Kind(FieldAdjustment) {
192      this->Field = Field;
193    }
194
195    SubobjectAdjustment(const MemberPointerType *MPT, llvm::Value *Ptr)
196      : Kind(MemberPointerAdjustment) {
197      this->Ptr.MPT = MPT;
198      this->Ptr.Ptr = Ptr;
199    }
200  };
201}
202
203static llvm::Value *
204CreateReferenceTemporary(CodeGenFunction &CGF, QualType Type,
205                         const NamedDecl *InitializedDecl) {
206  if (const VarDecl *VD = dyn_cast_or_null<VarDecl>(InitializedDecl)) {
207    if (VD->hasGlobalStorage()) {
208      SmallString<256> Name;
209      llvm::raw_svector_ostream Out(Name);
210      CGF.CGM.getCXXABI().getMangleContext().mangleReferenceTemporary(VD, Out);
211      Out.flush();
212
213      llvm::Type *RefTempTy = CGF.ConvertTypeForMem(Type);
214
215      // Create the reference temporary.
216      llvm::GlobalValue *RefTemp =
217        new llvm::GlobalVariable(CGF.CGM.getModule(),
218                                 RefTempTy, /*isConstant=*/false,
219                                 llvm::GlobalValue::InternalLinkage,
220                                 llvm::Constant::getNullValue(RefTempTy),
221                                 Name.str());
222      return RefTemp;
223    }
224  }
225
226  return CGF.CreateMemTemp(Type, "ref.tmp");
227}
228
229static llvm::Value *
230EmitExprForReferenceBinding(CodeGenFunction &CGF, const Expr *E,
231                            llvm::Value *&ReferenceTemporary,
232                            const CXXDestructorDecl *&ReferenceTemporaryDtor,
233                            QualType &ObjCARCReferenceLifetimeType,
234                            const NamedDecl *InitializedDecl) {
235  // Look through single-element init lists that claim to be lvalues. They're
236  // just syntactic wrappers in this case.
237  if (const InitListExpr *ILE = dyn_cast<InitListExpr>(E)) {
238    if (ILE->getNumInits() == 1 && ILE->isGLValue())
239      E = ILE->getInit(0);
240  }
241
242  // Look through expressions for materialized temporaries (for now).
243  if (const MaterializeTemporaryExpr *M
244                                      = dyn_cast<MaterializeTemporaryExpr>(E)) {
245    // Objective-C++ ARC:
246    //   If we are binding a reference to a temporary that has ownership, we
247    //   need to perform retain/release operations on the temporary.
248    if (CGF.getContext().getLangOpts().ObjCAutoRefCount &&
249        E->getType()->isObjCLifetimeType() &&
250        (E->getType().getObjCLifetime() == Qualifiers::OCL_Strong ||
251         E->getType().getObjCLifetime() == Qualifiers::OCL_Weak ||
252         E->getType().getObjCLifetime() == Qualifiers::OCL_Autoreleasing))
253      ObjCARCReferenceLifetimeType = E->getType();
254
255    E = M->GetTemporaryExpr();
256  }
257
258  if (const CXXDefaultArgExpr *DAE = dyn_cast<CXXDefaultArgExpr>(E))
259    E = DAE->getExpr();
260
261  if (const ExprWithCleanups *EWC = dyn_cast<ExprWithCleanups>(E)) {
262    CGF.enterFullExpression(EWC);
263    CodeGenFunction::RunCleanupsScope Scope(CGF);
264
265    return EmitExprForReferenceBinding(CGF, EWC->getSubExpr(),
266                                       ReferenceTemporary,
267                                       ReferenceTemporaryDtor,
268                                       ObjCARCReferenceLifetimeType,
269                                       InitializedDecl);
270  }
271
272  RValue RV;
273  if (E->isGLValue()) {
274    // Emit the expression as an lvalue.
275    LValue LV = CGF.EmitLValue(E);
276
277    if (LV.isSimple())
278      return LV.getAddress();
279
280    // We have to load the lvalue.
281    RV = CGF.EmitLoadOfLValue(LV);
282  } else {
283    if (!ObjCARCReferenceLifetimeType.isNull()) {
284      ReferenceTemporary = CreateReferenceTemporary(CGF,
285                                                  ObjCARCReferenceLifetimeType,
286                                                    InitializedDecl);
287
288
289      LValue RefTempDst = CGF.MakeAddrLValue(ReferenceTemporary,
290                                             ObjCARCReferenceLifetimeType);
291
292      CGF.EmitScalarInit(E, dyn_cast_or_null<ValueDecl>(InitializedDecl),
293                         RefTempDst, false);
294
295      bool ExtendsLifeOfTemporary = false;
296      if (const VarDecl *Var = dyn_cast_or_null<VarDecl>(InitializedDecl)) {
297        if (Var->extendsLifetimeOfTemporary())
298          ExtendsLifeOfTemporary = true;
299      } else if (InitializedDecl && isa<FieldDecl>(InitializedDecl)) {
300        ExtendsLifeOfTemporary = true;
301      }
302
303      if (!ExtendsLifeOfTemporary) {
304        // Since the lifetime of this temporary isn't going to be extended,
305        // we need to clean it up ourselves at the end of the full expression.
306        switch (ObjCARCReferenceLifetimeType.getObjCLifetime()) {
307        case Qualifiers::OCL_None:
308        case Qualifiers::OCL_ExplicitNone:
309        case Qualifiers::OCL_Autoreleasing:
310          break;
311
312        case Qualifiers::OCL_Strong: {
313          assert(!ObjCARCReferenceLifetimeType->isArrayType());
314          CleanupKind cleanupKind = CGF.getARCCleanupKind();
315          CGF.pushDestroy(cleanupKind,
316                          ReferenceTemporary,
317                          ObjCARCReferenceLifetimeType,
318                          CodeGenFunction::destroyARCStrongImprecise,
319                          cleanupKind & EHCleanup);
320          break;
321        }
322
323        case Qualifiers::OCL_Weak:
324          assert(!ObjCARCReferenceLifetimeType->isArrayType());
325          CGF.pushDestroy(NormalAndEHCleanup,
326                          ReferenceTemporary,
327                          ObjCARCReferenceLifetimeType,
328                          CodeGenFunction::destroyARCWeak,
329                          /*useEHCleanupForArray*/ true);
330          break;
331        }
332
333        ObjCARCReferenceLifetimeType = QualType();
334      }
335
336      return ReferenceTemporary;
337    }
338
339    SmallVector<SubobjectAdjustment, 2> Adjustments;
340    while (true) {
341      E = E->IgnoreParens();
342
343      if (const CastExpr *CE = dyn_cast<CastExpr>(E)) {
344        if ((CE->getCastKind() == CK_DerivedToBase ||
345             CE->getCastKind() == CK_UncheckedDerivedToBase) &&
346            E->getType()->isRecordType()) {
347          E = CE->getSubExpr();
348          CXXRecordDecl *Derived
349            = cast<CXXRecordDecl>(E->getType()->getAs<RecordType>()->getDecl());
350          Adjustments.push_back(SubobjectAdjustment(CE, Derived));
351          continue;
352        }
353
354        if (CE->getCastKind() == CK_NoOp) {
355          E = CE->getSubExpr();
356          continue;
357        }
358      } else if (const MemberExpr *ME = dyn_cast<MemberExpr>(E)) {
359        if (!ME->isArrow() && ME->getBase()->isRValue()) {
360          assert(ME->getBase()->getType()->isRecordType());
361          if (FieldDecl *Field = dyn_cast<FieldDecl>(ME->getMemberDecl())) {
362            E = ME->getBase();
363            Adjustments.push_back(SubobjectAdjustment(Field));
364            continue;
365          }
366        }
367      } else if (const BinaryOperator *BO = dyn_cast<BinaryOperator>(E)) {
368        if (BO->isPtrMemOp()) {
369          assert(BO->getLHS()->isRValue());
370          E = BO->getLHS();
371          const MemberPointerType *MPT =
372              BO->getRHS()->getType()->getAs<MemberPointerType>();
373          llvm::Value *Ptr = CGF.EmitScalarExpr(BO->getRHS());
374          Adjustments.push_back(SubobjectAdjustment(MPT, Ptr));
375        }
376      }
377
378      if (const OpaqueValueExpr *opaque = dyn_cast<OpaqueValueExpr>(E))
379        if (opaque->getType()->isRecordType())
380          return CGF.EmitOpaqueValueLValue(opaque).getAddress();
381
382      // Nothing changed.
383      break;
384    }
385
386    // Create a reference temporary if necessary.
387    AggValueSlot AggSlot = AggValueSlot::ignored();
388    if (CGF.hasAggregateLLVMType(E->getType()) &&
389        !E->getType()->isAnyComplexType()) {
390      ReferenceTemporary = CreateReferenceTemporary(CGF, E->getType(),
391                                                    InitializedDecl);
392      CharUnits Alignment = CGF.getContext().getTypeAlignInChars(E->getType());
393      AggValueSlot::IsDestructed_t isDestructed
394        = AggValueSlot::IsDestructed_t(InitializedDecl != 0);
395      AggSlot = AggValueSlot::forAddr(ReferenceTemporary, Alignment,
396                                      Qualifiers(), isDestructed,
397                                      AggValueSlot::DoesNotNeedGCBarriers,
398                                      AggValueSlot::IsNotAliased);
399    }
400
401    if (InitializedDecl) {
402      // Get the destructor for the reference temporary.
403      if (const RecordType *RT = E->getType()->getAs<RecordType>()) {
404        CXXRecordDecl *ClassDecl = cast<CXXRecordDecl>(RT->getDecl());
405        if (!ClassDecl->hasTrivialDestructor())
406          ReferenceTemporaryDtor = ClassDecl->getDestructor();
407      }
408    }
409
410    RV = CGF.EmitAnyExpr(E, AggSlot);
411
412    // Check if need to perform derived-to-base casts and/or field accesses, to
413    // get from the temporary object we created (and, potentially, for which we
414    // extended the lifetime) to the subobject we're binding the reference to.
415    if (!Adjustments.empty()) {
416      llvm::Value *Object = RV.getAggregateAddr();
417      for (unsigned I = Adjustments.size(); I != 0; --I) {
418        SubobjectAdjustment &Adjustment = Adjustments[I-1];
419        switch (Adjustment.Kind) {
420        case SubobjectAdjustment::DerivedToBaseAdjustment:
421          Object =
422              CGF.GetAddressOfBaseClass(Object,
423                                        Adjustment.DerivedToBase.DerivedClass,
424                              Adjustment.DerivedToBase.BasePath->path_begin(),
425                              Adjustment.DerivedToBase.BasePath->path_end(),
426                                        /*NullCheckValue=*/false);
427          break;
428
429        case SubobjectAdjustment::FieldAdjustment: {
430          LValue LV = CGF.MakeAddrLValue(Object, E->getType());
431          LV = CGF.EmitLValueForField(LV, Adjustment.Field);
432          if (LV.isSimple()) {
433            Object = LV.getAddress();
434            break;
435          }
436
437          // For non-simple lvalues, we actually have to create a copy of
438          // the object we're binding to.
439          QualType T = Adjustment.Field->getType().getNonReferenceType()
440                                                  .getUnqualifiedType();
441          Object = CreateReferenceTemporary(CGF, T, InitializedDecl);
442          LValue TempLV = CGF.MakeAddrLValue(Object,
443                                             Adjustment.Field->getType());
444          CGF.EmitStoreThroughLValue(CGF.EmitLoadOfLValue(LV), TempLV);
445          break;
446        }
447
448        case SubobjectAdjustment::MemberPointerAdjustment: {
449          Object = CGF.CGM.getCXXABI().EmitMemberDataPointerAddress(
450                        CGF, Object, Adjustment.Ptr.Ptr, Adjustment.Ptr.MPT);
451          break;
452        }
453        }
454      }
455
456      return Object;
457    }
458  }
459
460  if (RV.isAggregate())
461    return RV.getAggregateAddr();
462
463  // Create a temporary variable that we can bind the reference to.
464  ReferenceTemporary = CreateReferenceTemporary(CGF, E->getType(),
465                                                InitializedDecl);
466
467
468  unsigned Alignment =
469    CGF.getContext().getTypeAlignInChars(E->getType()).getQuantity();
470  if (RV.isScalar())
471    CGF.EmitStoreOfScalar(RV.getScalarVal(), ReferenceTemporary,
472                          /*Volatile=*/false, Alignment, E->getType());
473  else
474    CGF.StoreComplexToAddr(RV.getComplexVal(), ReferenceTemporary,
475                           /*Volatile=*/false);
476  return ReferenceTemporary;
477}
478
479RValue
480CodeGenFunction::EmitReferenceBindingToExpr(const Expr *E,
481                                            const NamedDecl *InitializedDecl) {
482  llvm::Value *ReferenceTemporary = 0;
483  const CXXDestructorDecl *ReferenceTemporaryDtor = 0;
484  QualType ObjCARCReferenceLifetimeType;
485  llvm::Value *Value = EmitExprForReferenceBinding(*this, E, ReferenceTemporary,
486                                                   ReferenceTemporaryDtor,
487                                                   ObjCARCReferenceLifetimeType,
488                                                   InitializedDecl);
489  if (CatchUndefined && !E->getType()->isFunctionType()) {
490    // C++11 [dcl.ref]p5 (as amended by core issue 453):
491    //   If a glvalue to which a reference is directly bound designates neither
492    //   an existing object or function of an appropriate type nor a region of
493    //   storage of suitable size and alignment to contain an object of the
494    //   reference's type, the behavior is undefined.
495    QualType Ty = E->getType();
496    EmitTypeCheck(TCK_ReferenceBinding, Value, Ty);
497  }
498  if (!ReferenceTemporaryDtor && ObjCARCReferenceLifetimeType.isNull())
499    return RValue::get(Value);
500
501  // Make sure to call the destructor for the reference temporary.
502  const VarDecl *VD = dyn_cast_or_null<VarDecl>(InitializedDecl);
503  if (VD && VD->hasGlobalStorage()) {
504    if (ReferenceTemporaryDtor) {
505      llvm::Constant *DtorFn =
506        CGM.GetAddrOfCXXDestructor(ReferenceTemporaryDtor, Dtor_Complete);
507      CGM.getCXXABI().registerGlobalDtor(*this, DtorFn,
508                                    cast<llvm::Constant>(ReferenceTemporary));
509    } else {
510      assert(!ObjCARCReferenceLifetimeType.isNull());
511      // Note: We intentionally do not register a global "destructor" to
512      // release the object.
513    }
514
515    return RValue::get(Value);
516  }
517
518  if (ReferenceTemporaryDtor)
519    PushDestructorCleanup(ReferenceTemporaryDtor, ReferenceTemporary);
520  else {
521    switch (ObjCARCReferenceLifetimeType.getObjCLifetime()) {
522    case Qualifiers::OCL_None:
523      llvm_unreachable(
524                      "Not a reference temporary that needs to be deallocated");
525    case Qualifiers::OCL_ExplicitNone:
526    case Qualifiers::OCL_Autoreleasing:
527      // Nothing to do.
528      break;
529
530    case Qualifiers::OCL_Strong: {
531      bool precise = VD && VD->hasAttr<ObjCPreciseLifetimeAttr>();
532      CleanupKind cleanupKind = getARCCleanupKind();
533      pushDestroy(cleanupKind, ReferenceTemporary, ObjCARCReferenceLifetimeType,
534                  precise ? destroyARCStrongPrecise : destroyARCStrongImprecise,
535                  cleanupKind & EHCleanup);
536      break;
537    }
538
539    case Qualifiers::OCL_Weak: {
540      // __weak objects always get EH cleanups; otherwise, exceptions
541      // could cause really nasty crashes instead of mere leaks.
542      pushDestroy(NormalAndEHCleanup, ReferenceTemporary,
543                  ObjCARCReferenceLifetimeType, destroyARCWeak, true);
544      break;
545    }
546    }
547  }
548
549  return RValue::get(Value);
550}
551
552
553/// getAccessedFieldNo - Given an encoded value and a result number, return the
554/// input field number being accessed.
555unsigned CodeGenFunction::getAccessedFieldNo(unsigned Idx,
556                                             const llvm::Constant *Elts) {
557  return cast<llvm::ConstantInt>(Elts->getAggregateElement(Idx))
558      ->getZExtValue();
559}
560
561void CodeGenFunction::EmitTypeCheck(TypeCheckKind TCK, llvm::Value *Address,
562                                    QualType Ty, CharUnits Alignment) {
563  if (!CatchUndefined)
564    return;
565
566  llvm::Value *Cond = 0;
567
568  if (TCK != TCK_Load && TCK != TCK_Store) {
569    // The glvalue must not be an empty glvalue. Don't bother checking this for
570    // loads and stores, because we will get a segfault anyway (if the operation
571    // isn't optimized out).
572    Cond = Builder.CreateICmpNE(
573        Address, llvm::Constant::getNullValue(Address->getType()));
574  }
575
576  if (!Ty->isIncompleteType()) {
577    uint64_t Size = getContext().getTypeSizeInChars(Ty).getQuantity();
578    uint64_t AlignVal = Alignment.getQuantity();
579    if (!AlignVal)
580      AlignVal = getContext().getTypeAlignInChars(Ty).getQuantity();
581
582    // This needs to be to the standard address space.
583    Address = Builder.CreateBitCast(Address, Int8PtrTy);
584
585    // The glvalue must refer to a large enough storage region.
586    // FIXME: If -faddress-sanitizer is enabled, insert dynamic instrumentation
587    //        to check this.
588    llvm::Value *F = CGM.getIntrinsic(llvm::Intrinsic::objectsize, IntPtrTy);
589    llvm::Value *Min = Builder.getFalse();
590    llvm::Value *LargeEnough =
591        Builder.CreateICmpUGE(Builder.CreateCall2(F, Address, Min),
592                              llvm::ConstantInt::get(IntPtrTy, Size));
593    Cond = Cond ? Builder.CreateAnd(Cond, LargeEnough) : LargeEnough;
594
595    // The glvalue must be suitably aligned.
596    llvm::Value *Align =
597        Builder.CreateAnd(Builder.CreatePtrToInt(Address, IntPtrTy),
598                          llvm::ConstantInt::get(IntPtrTy, AlignVal - 1));
599    Cond = Builder.CreateAnd(Cond,
600        Builder.CreateICmpEQ(Align, llvm::ConstantInt::get(IntPtrTy, 0)));
601  }
602
603  if (Cond)
604    EmitCheck(Cond);
605}
606
607
608CodeGenFunction::ComplexPairTy CodeGenFunction::
609EmitComplexPrePostIncDec(const UnaryOperator *E, LValue LV,
610                         bool isInc, bool isPre) {
611  ComplexPairTy InVal = LoadComplexFromAddr(LV.getAddress(),
612                                            LV.isVolatileQualified());
613
614  llvm::Value *NextVal;
615  if (isa<llvm::IntegerType>(InVal.first->getType())) {
616    uint64_t AmountVal = isInc ? 1 : -1;
617    NextVal = llvm::ConstantInt::get(InVal.first->getType(), AmountVal, true);
618
619    // Add the inc/dec to the real part.
620    NextVal = Builder.CreateAdd(InVal.first, NextVal, isInc ? "inc" : "dec");
621  } else {
622    QualType ElemTy = E->getType()->getAs<ComplexType>()->getElementType();
623    llvm::APFloat FVal(getContext().getFloatTypeSemantics(ElemTy), 1);
624    if (!isInc)
625      FVal.changeSign();
626    NextVal = llvm::ConstantFP::get(getLLVMContext(), FVal);
627
628    // Add the inc/dec to the real part.
629    NextVal = Builder.CreateFAdd(InVal.first, NextVal, isInc ? "inc" : "dec");
630  }
631
632  ComplexPairTy IncVal(NextVal, InVal.second);
633
634  // Store the updated result through the lvalue.
635  StoreComplexToAddr(IncVal, LV.getAddress(), LV.isVolatileQualified());
636
637  // If this is a postinc, return the value read from memory, otherwise use the
638  // updated value.
639  return isPre ? IncVal : InVal;
640}
641
642
643//===----------------------------------------------------------------------===//
644//                         LValue Expression Emission
645//===----------------------------------------------------------------------===//
646
647RValue CodeGenFunction::GetUndefRValue(QualType Ty) {
648  if (Ty->isVoidType())
649    return RValue::get(0);
650
651  if (const ComplexType *CTy = Ty->getAs<ComplexType>()) {
652    llvm::Type *EltTy = ConvertType(CTy->getElementType());
653    llvm::Value *U = llvm::UndefValue::get(EltTy);
654    return RValue::getComplex(std::make_pair(U, U));
655  }
656
657  // If this is a use of an undefined aggregate type, the aggregate must have an
658  // identifiable address.  Just because the contents of the value are undefined
659  // doesn't mean that the address can't be taken and compared.
660  if (hasAggregateLLVMType(Ty)) {
661    llvm::Value *DestPtr = CreateMemTemp(Ty, "undef.agg.tmp");
662    return RValue::getAggregate(DestPtr);
663  }
664
665  return RValue::get(llvm::UndefValue::get(ConvertType(Ty)));
666}
667
668RValue CodeGenFunction::EmitUnsupportedRValue(const Expr *E,
669                                              const char *Name) {
670  ErrorUnsupported(E, Name);
671  return GetUndefRValue(E->getType());
672}
673
674LValue CodeGenFunction::EmitUnsupportedLValue(const Expr *E,
675                                              const char *Name) {
676  ErrorUnsupported(E, Name);
677  llvm::Type *Ty = llvm::PointerType::getUnqual(ConvertType(E->getType()));
678  return MakeAddrLValue(llvm::UndefValue::get(Ty), E->getType());
679}
680
681LValue CodeGenFunction::EmitCheckedLValue(const Expr *E, TypeCheckKind TCK) {
682  LValue LV = EmitLValue(E);
683  if (!isa<DeclRefExpr>(E) && !LV.isBitField() && LV.isSimple())
684    EmitTypeCheck(TCK, LV.getAddress(), E->getType(), LV.getAlignment());
685  return LV;
686}
687
688/// EmitLValue - Emit code to compute a designator that specifies the location
689/// of the expression.
690///
691/// This can return one of two things: a simple address or a bitfield reference.
692/// In either case, the LLVM Value* in the LValue structure is guaranteed to be
693/// an LLVM pointer type.
694///
695/// If this returns a bitfield reference, nothing about the pointee type of the
696/// LLVM value is known: For example, it may not be a pointer to an integer.
697///
698/// If this returns a normal address, and if the lvalue's C type is fixed size,
699/// this method guarantees that the returned pointer type will point to an LLVM
700/// type of the same size of the lvalue's type.  If the lvalue has a variable
701/// length type, this is not possible.
702///
703LValue CodeGenFunction::EmitLValue(const Expr *E) {
704  switch (E->getStmtClass()) {
705  default: return EmitUnsupportedLValue(E, "l-value expression");
706
707  case Expr::ObjCPropertyRefExprClass:
708    llvm_unreachable("cannot emit a property reference directly");
709
710  case Expr::ObjCSelectorExprClass:
711  return EmitObjCSelectorLValue(cast<ObjCSelectorExpr>(E));
712  case Expr::ObjCIsaExprClass:
713    return EmitObjCIsaExpr(cast<ObjCIsaExpr>(E));
714  case Expr::BinaryOperatorClass:
715    return EmitBinaryOperatorLValue(cast<BinaryOperator>(E));
716  case Expr::CompoundAssignOperatorClass:
717    if (!E->getType()->isAnyComplexType())
718      return EmitCompoundAssignmentLValue(cast<CompoundAssignOperator>(E));
719    return EmitComplexCompoundAssignmentLValue(cast<CompoundAssignOperator>(E));
720  case Expr::CallExprClass:
721  case Expr::CXXMemberCallExprClass:
722  case Expr::CXXOperatorCallExprClass:
723  case Expr::UserDefinedLiteralClass:
724    return EmitCallExprLValue(cast<CallExpr>(E));
725  case Expr::VAArgExprClass:
726    return EmitVAArgExprLValue(cast<VAArgExpr>(E));
727  case Expr::DeclRefExprClass:
728    return EmitDeclRefLValue(cast<DeclRefExpr>(E));
729  case Expr::ParenExprClass:
730    return EmitLValue(cast<ParenExpr>(E)->getSubExpr());
731  case Expr::GenericSelectionExprClass:
732    return EmitLValue(cast<GenericSelectionExpr>(E)->getResultExpr());
733  case Expr::PredefinedExprClass:
734    return EmitPredefinedLValue(cast<PredefinedExpr>(E));
735  case Expr::StringLiteralClass:
736    return EmitStringLiteralLValue(cast<StringLiteral>(E));
737  case Expr::ObjCEncodeExprClass:
738    return EmitObjCEncodeExprLValue(cast<ObjCEncodeExpr>(E));
739  case Expr::PseudoObjectExprClass:
740    return EmitPseudoObjectLValue(cast<PseudoObjectExpr>(E));
741  case Expr::InitListExprClass:
742    return EmitInitListLValue(cast<InitListExpr>(E));
743  case Expr::CXXTemporaryObjectExprClass:
744  case Expr::CXXConstructExprClass:
745    return EmitCXXConstructLValue(cast<CXXConstructExpr>(E));
746  case Expr::CXXBindTemporaryExprClass:
747    return EmitCXXBindTemporaryLValue(cast<CXXBindTemporaryExpr>(E));
748  case Expr::LambdaExprClass:
749    return EmitLambdaLValue(cast<LambdaExpr>(E));
750
751  case Expr::ExprWithCleanupsClass: {
752    const ExprWithCleanups *cleanups = cast<ExprWithCleanups>(E);
753    enterFullExpression(cleanups);
754    RunCleanupsScope Scope(*this);
755    return EmitLValue(cleanups->getSubExpr());
756  }
757
758  case Expr::CXXScalarValueInitExprClass:
759    return EmitNullInitializationLValue(cast<CXXScalarValueInitExpr>(E));
760  case Expr::CXXDefaultArgExprClass:
761    return EmitLValue(cast<CXXDefaultArgExpr>(E)->getExpr());
762  case Expr::CXXTypeidExprClass:
763    return EmitCXXTypeidLValue(cast<CXXTypeidExpr>(E));
764
765  case Expr::ObjCMessageExprClass:
766    return EmitObjCMessageExprLValue(cast<ObjCMessageExpr>(E));
767  case Expr::ObjCIvarRefExprClass:
768    return EmitObjCIvarRefLValue(cast<ObjCIvarRefExpr>(E));
769  case Expr::StmtExprClass:
770    return EmitStmtExprLValue(cast<StmtExpr>(E));
771  case Expr::UnaryOperatorClass:
772    return EmitUnaryOpLValue(cast<UnaryOperator>(E));
773  case Expr::ArraySubscriptExprClass:
774    return EmitArraySubscriptExpr(cast<ArraySubscriptExpr>(E));
775  case Expr::ExtVectorElementExprClass:
776    return EmitExtVectorElementExpr(cast<ExtVectorElementExpr>(E));
777  case Expr::MemberExprClass:
778    return EmitMemberExpr(cast<MemberExpr>(E));
779  case Expr::CompoundLiteralExprClass:
780    return EmitCompoundLiteralLValue(cast<CompoundLiteralExpr>(E));
781  case Expr::ConditionalOperatorClass:
782    return EmitConditionalOperatorLValue(cast<ConditionalOperator>(E));
783  case Expr::BinaryConditionalOperatorClass:
784    return EmitConditionalOperatorLValue(cast<BinaryConditionalOperator>(E));
785  case Expr::ChooseExprClass:
786    return EmitLValue(cast<ChooseExpr>(E)->getChosenSubExpr(getContext()));
787  case Expr::OpaqueValueExprClass:
788    return EmitOpaqueValueLValue(cast<OpaqueValueExpr>(E));
789  case Expr::SubstNonTypeTemplateParmExprClass:
790    return EmitLValue(cast<SubstNonTypeTemplateParmExpr>(E)->getReplacement());
791  case Expr::ImplicitCastExprClass:
792  case Expr::CStyleCastExprClass:
793  case Expr::CXXFunctionalCastExprClass:
794  case Expr::CXXStaticCastExprClass:
795  case Expr::CXXDynamicCastExprClass:
796  case Expr::CXXReinterpretCastExprClass:
797  case Expr::CXXConstCastExprClass:
798  case Expr::ObjCBridgedCastExprClass:
799    return EmitCastLValue(cast<CastExpr>(E));
800
801  case Expr::MaterializeTemporaryExprClass:
802    return EmitMaterializeTemporaryExpr(cast<MaterializeTemporaryExpr>(E));
803  }
804}
805
806/// Given an object of the given canonical type, can we safely copy a
807/// value out of it based on its initializer?
808static bool isConstantEmittableObjectType(QualType type) {
809  assert(type.isCanonical());
810  assert(!type->isReferenceType());
811
812  // Must be const-qualified but non-volatile.
813  Qualifiers qs = type.getLocalQualifiers();
814  if (!qs.hasConst() || qs.hasVolatile()) return false;
815
816  // Otherwise, all object types satisfy this except C++ classes with
817  // mutable subobjects or non-trivial copy/destroy behavior.
818  if (const RecordType *RT = dyn_cast<RecordType>(type))
819    if (const CXXRecordDecl *RD = dyn_cast<CXXRecordDecl>(RT->getDecl()))
820      if (RD->hasMutableFields() || !RD->isTrivial())
821        return false;
822
823  return true;
824}
825
826/// Can we constant-emit a load of a reference to a variable of the
827/// given type?  This is different from predicates like
828/// Decl::isUsableInConstantExpressions because we do want it to apply
829/// in situations that don't necessarily satisfy the language's rules
830/// for this (e.g. C++'s ODR-use rules).  For example, we want to able
831/// to do this with const float variables even if those variables
832/// aren't marked 'constexpr'.
833enum ConstantEmissionKind {
834  CEK_None,
835  CEK_AsReferenceOnly,
836  CEK_AsValueOrReference,
837  CEK_AsValueOnly
838};
839static ConstantEmissionKind checkVarTypeForConstantEmission(QualType type) {
840  type = type.getCanonicalType();
841  if (const ReferenceType *ref = dyn_cast<ReferenceType>(type)) {
842    if (isConstantEmittableObjectType(ref->getPointeeType()))
843      return CEK_AsValueOrReference;
844    return CEK_AsReferenceOnly;
845  }
846  if (isConstantEmittableObjectType(type))
847    return CEK_AsValueOnly;
848  return CEK_None;
849}
850
851/// Try to emit a reference to the given value without producing it as
852/// an l-value.  This is actually more than an optimization: we can't
853/// produce an l-value for variables that we never actually captured
854/// in a block or lambda, which means const int variables or constexpr
855/// literals or similar.
856CodeGenFunction::ConstantEmission
857CodeGenFunction::tryEmitAsConstant(DeclRefExpr *refExpr) {
858  ValueDecl *value = refExpr->getDecl();
859
860  // The value needs to be an enum constant or a constant variable.
861  ConstantEmissionKind CEK;
862  if (isa<ParmVarDecl>(value)) {
863    CEK = CEK_None;
864  } else if (VarDecl *var = dyn_cast<VarDecl>(value)) {
865    CEK = checkVarTypeForConstantEmission(var->getType());
866  } else if (isa<EnumConstantDecl>(value)) {
867    CEK = CEK_AsValueOnly;
868  } else {
869    CEK = CEK_None;
870  }
871  if (CEK == CEK_None) return ConstantEmission();
872
873  Expr::EvalResult result;
874  bool resultIsReference;
875  QualType resultType;
876
877  // It's best to evaluate all the way as an r-value if that's permitted.
878  if (CEK != CEK_AsReferenceOnly &&
879      refExpr->EvaluateAsRValue(result, getContext())) {
880    resultIsReference = false;
881    resultType = refExpr->getType();
882
883  // Otherwise, try to evaluate as an l-value.
884  } else if (CEK != CEK_AsValueOnly &&
885             refExpr->EvaluateAsLValue(result, getContext())) {
886    resultIsReference = true;
887    resultType = value->getType();
888
889  // Failure.
890  } else {
891    return ConstantEmission();
892  }
893
894  // In any case, if the initializer has side-effects, abandon ship.
895  if (result.HasSideEffects)
896    return ConstantEmission();
897
898  // Emit as a constant.
899  llvm::Constant *C = CGM.EmitConstantValue(result.Val, resultType, this);
900
901  // Make sure we emit a debug reference to the global variable.
902  // This should probably fire even for
903  if (isa<VarDecl>(value)) {
904    if (!getContext().DeclMustBeEmitted(cast<VarDecl>(value)))
905      EmitDeclRefExprDbgValue(refExpr, C);
906  } else {
907    assert(isa<EnumConstantDecl>(value));
908    EmitDeclRefExprDbgValue(refExpr, C);
909  }
910
911  // If we emitted a reference constant, we need to dereference that.
912  if (resultIsReference)
913    return ConstantEmission::forReference(C);
914
915  return ConstantEmission::forValue(C);
916}
917
918llvm::Value *CodeGenFunction::EmitLoadOfScalar(LValue lvalue) {
919  return EmitLoadOfScalar(lvalue.getAddress(), lvalue.isVolatile(),
920                          lvalue.getAlignment().getQuantity(),
921                          lvalue.getType(), lvalue.getTBAAInfo());
922}
923
924static bool hasBooleanRepresentation(QualType Ty) {
925  if (Ty->isBooleanType())
926    return true;
927
928  if (const EnumType *ET = Ty->getAs<EnumType>())
929    return ET->getDecl()->getIntegerType()->isBooleanType();
930
931  if (const AtomicType *AT = Ty->getAs<AtomicType>())
932    return hasBooleanRepresentation(AT->getValueType());
933
934  return false;
935}
936
937llvm::MDNode *CodeGenFunction::getRangeForLoadFromType(QualType Ty) {
938  const EnumType *ET = Ty->getAs<EnumType>();
939  bool IsRegularCPlusPlusEnum = (getLangOpts().CPlusPlus && ET &&
940                                 CGM.getCodeGenOpts().StrictEnums &&
941                                 !ET->getDecl()->isFixed());
942  bool IsBool = hasBooleanRepresentation(Ty);
943  if (!IsBool && !IsRegularCPlusPlusEnum)
944    return NULL;
945
946  llvm::APInt Min;
947  llvm::APInt End;
948  if (IsBool) {
949    Min = llvm::APInt(8, 0);
950    End = llvm::APInt(8, 2);
951  } else {
952    const EnumDecl *ED = ET->getDecl();
953    llvm::Type *LTy = ConvertTypeForMem(ED->getIntegerType());
954    unsigned Bitwidth = LTy->getScalarSizeInBits();
955    unsigned NumNegativeBits = ED->getNumNegativeBits();
956    unsigned NumPositiveBits = ED->getNumPositiveBits();
957
958    if (NumNegativeBits) {
959      unsigned NumBits = std::max(NumNegativeBits, NumPositiveBits + 1);
960      assert(NumBits <= Bitwidth);
961      End = llvm::APInt(Bitwidth, 1) << (NumBits - 1);
962      Min = -End;
963    } else {
964      assert(NumPositiveBits <= Bitwidth);
965      End = llvm::APInt(Bitwidth, 1) << NumPositiveBits;
966      Min = llvm::APInt(Bitwidth, 0);
967    }
968  }
969
970  llvm::MDBuilder MDHelper(getLLVMContext());
971  return MDHelper.createRange(Min, End);
972}
973
974llvm::Value *CodeGenFunction::EmitLoadOfScalar(llvm::Value *Addr, bool Volatile,
975                                              unsigned Alignment, QualType Ty,
976                                              llvm::MDNode *TBAAInfo) {
977
978  // For better performance, handle vector loads differently.
979  if (Ty->isVectorType()) {
980    llvm::Value *V;
981    const llvm::Type *EltTy =
982    cast<llvm::PointerType>(Addr->getType())->getElementType();
983
984    const llvm::VectorType *VTy = cast<llvm::VectorType>(EltTy);
985
986    // Handle vectors of size 3, like size 4 for better performance.
987    if (VTy->getNumElements() == 3) {
988
989      // Bitcast to vec4 type.
990      llvm::VectorType *vec4Ty = llvm::VectorType::get(VTy->getElementType(),
991                                                         4);
992      llvm::PointerType *ptVec4Ty =
993      llvm::PointerType::get(vec4Ty,
994                             (cast<llvm::PointerType>(
995                                      Addr->getType()))->getAddressSpace());
996      llvm::Value *Cast = Builder.CreateBitCast(Addr, ptVec4Ty,
997                                                "castToVec4");
998      // Now load value.
999      llvm::Value *LoadVal = Builder.CreateLoad(Cast, Volatile, "loadVec4");
1000
1001      // Shuffle vector to get vec3.
1002      llvm::SmallVector<llvm::Constant*, 3> Mask;
1003      Mask.push_back(llvm::ConstantInt::get(
1004                                    llvm::Type::getInt32Ty(getLLVMContext()),
1005                                            0));
1006      Mask.push_back(llvm::ConstantInt::get(
1007                                    llvm::Type::getInt32Ty(getLLVMContext()),
1008                                            1));
1009      Mask.push_back(llvm::ConstantInt::get(
1010                                     llvm::Type::getInt32Ty(getLLVMContext()),
1011                                            2));
1012
1013      llvm::Value *MaskV = llvm::ConstantVector::get(Mask);
1014      V = Builder.CreateShuffleVector(LoadVal,
1015                                      llvm::UndefValue::get(vec4Ty),
1016                                      MaskV, "extractVec");
1017      return EmitFromMemory(V, Ty);
1018    }
1019  }
1020
1021  llvm::LoadInst *Load = Builder.CreateLoad(Addr);
1022  if (Volatile)
1023    Load->setVolatile(true);
1024  if (Alignment)
1025    Load->setAlignment(Alignment);
1026  if (TBAAInfo)
1027    CGM.DecorateInstruction(Load, TBAAInfo);
1028  // If this is an atomic type, all normal reads must be atomic
1029  if (Ty->isAtomicType())
1030    Load->setAtomic(llvm::SequentiallyConsistent);
1031
1032  if (CGM.getCodeGenOpts().OptimizationLevel > 0)
1033    if (llvm::MDNode *RangeInfo = getRangeForLoadFromType(Ty))
1034      Load->setMetadata(llvm::LLVMContext::MD_range, RangeInfo);
1035
1036  return EmitFromMemory(Load, Ty);
1037}
1038
1039llvm::Value *CodeGenFunction::EmitToMemory(llvm::Value *Value, QualType Ty) {
1040  // Bool has a different representation in memory than in registers.
1041  if (hasBooleanRepresentation(Ty)) {
1042    // This should really always be an i1, but sometimes it's already
1043    // an i8, and it's awkward to track those cases down.
1044    if (Value->getType()->isIntegerTy(1))
1045      return Builder.CreateZExt(Value, Builder.getInt8Ty(), "frombool");
1046    assert(Value->getType()->isIntegerTy(8) && "value rep of bool not i1/i8");
1047  }
1048
1049  return Value;
1050}
1051
1052llvm::Value *CodeGenFunction::EmitFromMemory(llvm::Value *Value, QualType Ty) {
1053  // Bool has a different representation in memory than in registers.
1054  if (hasBooleanRepresentation(Ty)) {
1055    assert(Value->getType()->isIntegerTy(8) && "memory rep of bool not i8");
1056    return Builder.CreateTrunc(Value, Builder.getInt1Ty(), "tobool");
1057  }
1058
1059  return Value;
1060}
1061
1062void CodeGenFunction::EmitStoreOfScalar(llvm::Value *Value, llvm::Value *Addr,
1063                                        bool Volatile, unsigned Alignment,
1064                                        QualType Ty,
1065                                        llvm::MDNode *TBAAInfo,
1066                                        bool isInit) {
1067
1068  // Handle vectors differently to get better performance.
1069  if (Ty->isVectorType()) {
1070    llvm::Type *SrcTy = Value->getType();
1071    llvm::VectorType *VecTy = cast<llvm::VectorType>(SrcTy);
1072    // Handle vec3 special.
1073    if (VecTy->getNumElements() == 3) {
1074      llvm::LLVMContext &VMContext = getLLVMContext();
1075
1076      // Our source is a vec3, do a shuffle vector to make it a vec4.
1077      llvm::SmallVector<llvm::Constant*, 4> Mask;
1078      Mask.push_back(llvm::ConstantInt::get(
1079                                            llvm::Type::getInt32Ty(VMContext),
1080                                            0));
1081      Mask.push_back(llvm::ConstantInt::get(
1082                                            llvm::Type::getInt32Ty(VMContext),
1083                                            1));
1084      Mask.push_back(llvm::ConstantInt::get(
1085                                            llvm::Type::getInt32Ty(VMContext),
1086                                            2));
1087      Mask.push_back(llvm::UndefValue::get(llvm::Type::getInt32Ty(VMContext)));
1088
1089      llvm::Value *MaskV = llvm::ConstantVector::get(Mask);
1090      Value = Builder.CreateShuffleVector(Value,
1091                                          llvm::UndefValue::get(VecTy),
1092                                          MaskV, "extractVec");
1093      SrcTy = llvm::VectorType::get(VecTy->getElementType(), 4);
1094    }
1095    llvm::PointerType *DstPtr = cast<llvm::PointerType>(Addr->getType());
1096    if (DstPtr->getElementType() != SrcTy) {
1097      llvm::Type *MemTy =
1098      llvm::PointerType::get(SrcTy, DstPtr->getAddressSpace());
1099      Addr = Builder.CreateBitCast(Addr, MemTy, "storetmp");
1100    }
1101  }
1102
1103  Value = EmitToMemory(Value, Ty);
1104
1105  llvm::StoreInst *Store = Builder.CreateStore(Value, Addr, Volatile);
1106  if (Alignment)
1107    Store->setAlignment(Alignment);
1108  if (TBAAInfo)
1109    CGM.DecorateInstruction(Store, TBAAInfo);
1110  if (!isInit && Ty->isAtomicType())
1111    Store->setAtomic(llvm::SequentiallyConsistent);
1112}
1113
1114void CodeGenFunction::EmitStoreOfScalar(llvm::Value *value, LValue lvalue,
1115    bool isInit) {
1116  EmitStoreOfScalar(value, lvalue.getAddress(), lvalue.isVolatile(),
1117                    lvalue.getAlignment().getQuantity(), lvalue.getType(),
1118                    lvalue.getTBAAInfo(), isInit);
1119}
1120
1121/// EmitLoadOfLValue - Given an expression that represents a value lvalue, this
1122/// method emits the address of the lvalue, then loads the result as an rvalue,
1123/// returning the rvalue.
1124RValue CodeGenFunction::EmitLoadOfLValue(LValue LV) {
1125  if (LV.isObjCWeak()) {
1126    // load of a __weak object.
1127    llvm::Value *AddrWeakObj = LV.getAddress();
1128    return RValue::get(CGM.getObjCRuntime().EmitObjCWeakRead(*this,
1129                                                             AddrWeakObj));
1130  }
1131  if (LV.getQuals().getObjCLifetime() == Qualifiers::OCL_Weak)
1132    return RValue::get(EmitARCLoadWeak(LV.getAddress()));
1133
1134  if (LV.isSimple()) {
1135    assert(!LV.getType()->isFunctionType());
1136
1137    // Everything needs a load.
1138    return RValue::get(EmitLoadOfScalar(LV));
1139  }
1140
1141  if (LV.isVectorElt()) {
1142    llvm::LoadInst *Load = Builder.CreateLoad(LV.getVectorAddr(),
1143                                              LV.isVolatileQualified());
1144    Load->setAlignment(LV.getAlignment().getQuantity());
1145    return RValue::get(Builder.CreateExtractElement(Load, LV.getVectorIdx(),
1146                                                    "vecext"));
1147  }
1148
1149  // If this is a reference to a subset of the elements of a vector, either
1150  // shuffle the input or extract/insert them as appropriate.
1151  if (LV.isExtVectorElt())
1152    return EmitLoadOfExtVectorElementLValue(LV);
1153
1154  assert(LV.isBitField() && "Unknown LValue type!");
1155  return EmitLoadOfBitfieldLValue(LV);
1156}
1157
1158RValue CodeGenFunction::EmitLoadOfBitfieldLValue(LValue LV) {
1159  const CGBitFieldInfo &Info = LV.getBitFieldInfo();
1160
1161  // Get the output type.
1162  llvm::Type *ResLTy = ConvertType(LV.getType());
1163  unsigned ResSizeInBits = CGM.getTargetData().getTypeSizeInBits(ResLTy);
1164
1165  // Compute the result as an OR of all of the individual component accesses.
1166  llvm::Value *Res = 0;
1167  for (unsigned i = 0, e = Info.getNumComponents(); i != e; ++i) {
1168    const CGBitFieldInfo::AccessInfo &AI = Info.getComponent(i);
1169    CharUnits AccessAlignment = AI.AccessAlignment;
1170    if (!LV.getAlignment().isZero())
1171      AccessAlignment = std::min(AccessAlignment, LV.getAlignment());
1172
1173    // Get the field pointer.
1174    llvm::Value *Ptr = LV.getBitFieldBaseAddr();
1175
1176    // Only offset by the field index if used, so that incoming values are not
1177    // required to be structures.
1178    if (AI.FieldIndex)
1179      Ptr = Builder.CreateStructGEP(Ptr, AI.FieldIndex, "bf.field");
1180
1181    // Offset by the byte offset, if used.
1182    if (!AI.FieldByteOffset.isZero()) {
1183      Ptr = EmitCastToVoidPtr(Ptr);
1184      Ptr = Builder.CreateConstGEP1_32(Ptr, AI.FieldByteOffset.getQuantity(),
1185                                       "bf.field.offs");
1186    }
1187
1188    // Cast to the access type.
1189    llvm::Type *PTy = llvm::Type::getIntNPtrTy(getLLVMContext(), AI.AccessWidth,
1190                       CGM.getContext().getTargetAddressSpace(LV.getType()));
1191    Ptr = Builder.CreateBitCast(Ptr, PTy);
1192
1193    // Perform the load.
1194    llvm::LoadInst *Load = Builder.CreateLoad(Ptr, LV.isVolatileQualified());
1195    Load->setAlignment(AccessAlignment.getQuantity());
1196
1197    // Shift out unused low bits and mask out unused high bits.
1198    llvm::Value *Val = Load;
1199    if (AI.FieldBitStart)
1200      Val = Builder.CreateLShr(Load, AI.FieldBitStart);
1201    Val = Builder.CreateAnd(Val, llvm::APInt::getLowBitsSet(AI.AccessWidth,
1202                                                            AI.TargetBitWidth),
1203                            "bf.clear");
1204
1205    // Extend or truncate to the target size.
1206    if (AI.AccessWidth < ResSizeInBits)
1207      Val = Builder.CreateZExt(Val, ResLTy);
1208    else if (AI.AccessWidth > ResSizeInBits)
1209      Val = Builder.CreateTrunc(Val, ResLTy);
1210
1211    // Shift into place, and OR into the result.
1212    if (AI.TargetBitOffset)
1213      Val = Builder.CreateShl(Val, AI.TargetBitOffset);
1214    Res = Res ? Builder.CreateOr(Res, Val) : Val;
1215  }
1216
1217  // If the bit-field is signed, perform the sign-extension.
1218  //
1219  // FIXME: This can easily be folded into the load of the high bits, which
1220  // could also eliminate the mask of high bits in some situations.
1221  if (Info.isSigned()) {
1222    unsigned ExtraBits = ResSizeInBits - Info.getSize();
1223    if (ExtraBits)
1224      Res = Builder.CreateAShr(Builder.CreateShl(Res, ExtraBits),
1225                               ExtraBits, "bf.val.sext");
1226  }
1227
1228  return RValue::get(Res);
1229}
1230
1231// If this is a reference to a subset of the elements of a vector, create an
1232// appropriate shufflevector.
1233RValue CodeGenFunction::EmitLoadOfExtVectorElementLValue(LValue LV) {
1234  llvm::LoadInst *Load = Builder.CreateLoad(LV.getExtVectorAddr(),
1235                                            LV.isVolatileQualified());
1236  Load->setAlignment(LV.getAlignment().getQuantity());
1237  llvm::Value *Vec = Load;
1238
1239  const llvm::Constant *Elts = LV.getExtVectorElts();
1240
1241  // If the result of the expression is a non-vector type, we must be extracting
1242  // a single element.  Just codegen as an extractelement.
1243  const VectorType *ExprVT = LV.getType()->getAs<VectorType>();
1244  if (!ExprVT) {
1245    unsigned InIdx = getAccessedFieldNo(0, Elts);
1246    llvm::Value *Elt = llvm::ConstantInt::get(Int32Ty, InIdx);
1247    return RValue::get(Builder.CreateExtractElement(Vec, Elt));
1248  }
1249
1250  // Always use shuffle vector to try to retain the original program structure
1251  unsigned NumResultElts = ExprVT->getNumElements();
1252
1253  SmallVector<llvm::Constant*, 4> Mask;
1254  for (unsigned i = 0; i != NumResultElts; ++i)
1255    Mask.push_back(Builder.getInt32(getAccessedFieldNo(i, Elts)));
1256
1257  llvm::Value *MaskV = llvm::ConstantVector::get(Mask);
1258  Vec = Builder.CreateShuffleVector(Vec, llvm::UndefValue::get(Vec->getType()),
1259                                    MaskV);
1260  return RValue::get(Vec);
1261}
1262
1263
1264
1265/// EmitStoreThroughLValue - Store the specified rvalue into the specified
1266/// lvalue, where both are guaranteed to the have the same type, and that type
1267/// is 'Ty'.
1268void CodeGenFunction::EmitStoreThroughLValue(RValue Src, LValue Dst, bool isInit) {
1269  if (!Dst.isSimple()) {
1270    if (Dst.isVectorElt()) {
1271      // Read/modify/write the vector, inserting the new element.
1272      llvm::LoadInst *Load = Builder.CreateLoad(Dst.getVectorAddr(),
1273                                                Dst.isVolatileQualified());
1274      Load->setAlignment(Dst.getAlignment().getQuantity());
1275      llvm::Value *Vec = Load;
1276      Vec = Builder.CreateInsertElement(Vec, Src.getScalarVal(),
1277                                        Dst.getVectorIdx(), "vecins");
1278      llvm::StoreInst *Store = Builder.CreateStore(Vec, Dst.getVectorAddr(),
1279                                                   Dst.isVolatileQualified());
1280      Store->setAlignment(Dst.getAlignment().getQuantity());
1281      return;
1282    }
1283
1284    // If this is an update of extended vector elements, insert them as
1285    // appropriate.
1286    if (Dst.isExtVectorElt())
1287      return EmitStoreThroughExtVectorComponentLValue(Src, Dst);
1288
1289    assert(Dst.isBitField() && "Unknown LValue type");
1290    return EmitStoreThroughBitfieldLValue(Src, Dst);
1291  }
1292
1293  // There's special magic for assigning into an ARC-qualified l-value.
1294  if (Qualifiers::ObjCLifetime Lifetime = Dst.getQuals().getObjCLifetime()) {
1295    switch (Lifetime) {
1296    case Qualifiers::OCL_None:
1297      llvm_unreachable("present but none");
1298
1299    case Qualifiers::OCL_ExplicitNone:
1300      // nothing special
1301      break;
1302
1303    case Qualifiers::OCL_Strong:
1304      EmitARCStoreStrong(Dst, Src.getScalarVal(), /*ignore*/ true);
1305      return;
1306
1307    case Qualifiers::OCL_Weak:
1308      EmitARCStoreWeak(Dst.getAddress(), Src.getScalarVal(), /*ignore*/ true);
1309      return;
1310
1311    case Qualifiers::OCL_Autoreleasing:
1312      Src = RValue::get(EmitObjCExtendObjectLifetime(Dst.getType(),
1313                                                     Src.getScalarVal()));
1314      // fall into the normal path
1315      break;
1316    }
1317  }
1318
1319  if (Dst.isObjCWeak() && !Dst.isNonGC()) {
1320    // load of a __weak object.
1321    llvm::Value *LvalueDst = Dst.getAddress();
1322    llvm::Value *src = Src.getScalarVal();
1323     CGM.getObjCRuntime().EmitObjCWeakAssign(*this, src, LvalueDst);
1324    return;
1325  }
1326
1327  if (Dst.isObjCStrong() && !Dst.isNonGC()) {
1328    // load of a __strong object.
1329    llvm::Value *LvalueDst = Dst.getAddress();
1330    llvm::Value *src = Src.getScalarVal();
1331    if (Dst.isObjCIvar()) {
1332      assert(Dst.getBaseIvarExp() && "BaseIvarExp is NULL");
1333      llvm::Type *ResultType = ConvertType(getContext().LongTy);
1334      llvm::Value *RHS = EmitScalarExpr(Dst.getBaseIvarExp());
1335      llvm::Value *dst = RHS;
1336      RHS = Builder.CreatePtrToInt(RHS, ResultType, "sub.ptr.rhs.cast");
1337      llvm::Value *LHS =
1338        Builder.CreatePtrToInt(LvalueDst, ResultType, "sub.ptr.lhs.cast");
1339      llvm::Value *BytesBetween = Builder.CreateSub(LHS, RHS, "ivar.offset");
1340      CGM.getObjCRuntime().EmitObjCIvarAssign(*this, src, dst,
1341                                              BytesBetween);
1342    } else if (Dst.isGlobalObjCRef()) {
1343      CGM.getObjCRuntime().EmitObjCGlobalAssign(*this, src, LvalueDst,
1344                                                Dst.isThreadLocalRef());
1345    }
1346    else
1347      CGM.getObjCRuntime().EmitObjCStrongCastAssign(*this, src, LvalueDst);
1348    return;
1349  }
1350
1351  assert(Src.isScalar() && "Can't emit an agg store with this method");
1352  EmitStoreOfScalar(Src.getScalarVal(), Dst, isInit);
1353}
1354
1355void CodeGenFunction::EmitStoreThroughBitfieldLValue(RValue Src, LValue Dst,
1356                                                     llvm::Value **Result) {
1357  const CGBitFieldInfo &Info = Dst.getBitFieldInfo();
1358
1359  // Get the output type.
1360  llvm::Type *ResLTy = ConvertTypeForMem(Dst.getType());
1361  unsigned ResSizeInBits = CGM.getTargetData().getTypeSizeInBits(ResLTy);
1362
1363  // Get the source value, truncated to the width of the bit-field.
1364  llvm::Value *SrcVal = Src.getScalarVal();
1365
1366  if (hasBooleanRepresentation(Dst.getType()))
1367    SrcVal = Builder.CreateIntCast(SrcVal, ResLTy, /*IsSigned=*/false);
1368
1369  SrcVal = Builder.CreateAnd(SrcVal, llvm::APInt::getLowBitsSet(ResSizeInBits,
1370                                                                Info.getSize()),
1371                             "bf.value");
1372
1373  // Return the new value of the bit-field, if requested.
1374  if (Result) {
1375    // Cast back to the proper type for result.
1376    llvm::Type *SrcTy = Src.getScalarVal()->getType();
1377    llvm::Value *ReloadVal = Builder.CreateIntCast(SrcVal, SrcTy, false,
1378                                                   "bf.reload.val");
1379
1380    // Sign extend if necessary.
1381    if (Info.isSigned()) {
1382      unsigned ExtraBits = ResSizeInBits - Info.getSize();
1383      if (ExtraBits)
1384        ReloadVal = Builder.CreateAShr(Builder.CreateShl(ReloadVal, ExtraBits),
1385                                       ExtraBits, "bf.reload.sext");
1386    }
1387
1388    *Result = ReloadVal;
1389  }
1390
1391  // Iterate over the components, writing each piece to memory.
1392  for (unsigned i = 0, e = Info.getNumComponents(); i != e; ++i) {
1393    const CGBitFieldInfo::AccessInfo &AI = Info.getComponent(i);
1394    CharUnits AccessAlignment = AI.AccessAlignment;
1395    if (!Dst.getAlignment().isZero())
1396      AccessAlignment = std::min(AccessAlignment, Dst.getAlignment());
1397
1398    // Get the field pointer.
1399    llvm::Value *Ptr = Dst.getBitFieldBaseAddr();
1400    unsigned addressSpace =
1401      cast<llvm::PointerType>(Ptr->getType())->getAddressSpace();
1402
1403    // Only offset by the field index if used, so that incoming values are not
1404    // required to be structures.
1405    if (AI.FieldIndex)
1406      Ptr = Builder.CreateStructGEP(Ptr, AI.FieldIndex, "bf.field");
1407
1408    // Offset by the byte offset, if used.
1409    if (!AI.FieldByteOffset.isZero()) {
1410      Ptr = EmitCastToVoidPtr(Ptr);
1411      Ptr = Builder.CreateConstGEP1_32(Ptr, AI.FieldByteOffset.getQuantity(),
1412                                       "bf.field.offs");
1413    }
1414
1415    // Cast to the access type.
1416    llvm::Type *AccessLTy =
1417      llvm::Type::getIntNTy(getLLVMContext(), AI.AccessWidth);
1418
1419    llvm::Type *PTy = AccessLTy->getPointerTo(addressSpace);
1420    Ptr = Builder.CreateBitCast(Ptr, PTy);
1421
1422    // Extract the piece of the bit-field value to write in this access, limited
1423    // to the values that are part of this access.
1424    llvm::Value *Val = SrcVal;
1425    if (AI.TargetBitOffset)
1426      Val = Builder.CreateLShr(Val, AI.TargetBitOffset);
1427    Val = Builder.CreateAnd(Val, llvm::APInt::getLowBitsSet(ResSizeInBits,
1428                                                            AI.TargetBitWidth));
1429
1430    // Extend or truncate to the access size.
1431    if (ResSizeInBits < AI.AccessWidth)
1432      Val = Builder.CreateZExt(Val, AccessLTy);
1433    else if (ResSizeInBits > AI.AccessWidth)
1434      Val = Builder.CreateTrunc(Val, AccessLTy);
1435
1436    // Shift into the position in memory.
1437    if (AI.FieldBitStart)
1438      Val = Builder.CreateShl(Val, AI.FieldBitStart);
1439
1440    // If necessary, load and OR in bits that are outside of the bit-field.
1441    if (AI.TargetBitWidth != AI.AccessWidth) {
1442      llvm::LoadInst *Load = Builder.CreateLoad(Ptr, Dst.isVolatileQualified());
1443      Load->setAlignment(AccessAlignment.getQuantity());
1444
1445      // Compute the mask for zeroing the bits that are part of the bit-field.
1446      llvm::APInt InvMask =
1447        ~llvm::APInt::getBitsSet(AI.AccessWidth, AI.FieldBitStart,
1448                                 AI.FieldBitStart + AI.TargetBitWidth);
1449
1450      // Apply the mask and OR in to the value to write.
1451      Val = Builder.CreateOr(Builder.CreateAnd(Load, InvMask), Val);
1452    }
1453
1454    // Write the value.
1455    llvm::StoreInst *Store = Builder.CreateStore(Val, Ptr,
1456                                                 Dst.isVolatileQualified());
1457    Store->setAlignment(AccessAlignment.getQuantity());
1458  }
1459}
1460
1461void CodeGenFunction::EmitStoreThroughExtVectorComponentLValue(RValue Src,
1462                                                               LValue Dst) {
1463  // This access turns into a read/modify/write of the vector.  Load the input
1464  // value now.
1465  llvm::LoadInst *Load = Builder.CreateLoad(Dst.getExtVectorAddr(),
1466                                            Dst.isVolatileQualified());
1467  Load->setAlignment(Dst.getAlignment().getQuantity());
1468  llvm::Value *Vec = Load;
1469  const llvm::Constant *Elts = Dst.getExtVectorElts();
1470
1471  llvm::Value *SrcVal = Src.getScalarVal();
1472
1473  if (const VectorType *VTy = Dst.getType()->getAs<VectorType>()) {
1474    unsigned NumSrcElts = VTy->getNumElements();
1475    unsigned NumDstElts =
1476       cast<llvm::VectorType>(Vec->getType())->getNumElements();
1477    if (NumDstElts == NumSrcElts) {
1478      // Use shuffle vector is the src and destination are the same number of
1479      // elements and restore the vector mask since it is on the side it will be
1480      // stored.
1481      SmallVector<llvm::Constant*, 4> Mask(NumDstElts);
1482      for (unsigned i = 0; i != NumSrcElts; ++i)
1483        Mask[getAccessedFieldNo(i, Elts)] = Builder.getInt32(i);
1484
1485      llvm::Value *MaskV = llvm::ConstantVector::get(Mask);
1486      Vec = Builder.CreateShuffleVector(SrcVal,
1487                                        llvm::UndefValue::get(Vec->getType()),
1488                                        MaskV);
1489    } else if (NumDstElts > NumSrcElts) {
1490      // Extended the source vector to the same length and then shuffle it
1491      // into the destination.
1492      // FIXME: since we're shuffling with undef, can we just use the indices
1493      //        into that?  This could be simpler.
1494      SmallVector<llvm::Constant*, 4> ExtMask;
1495      for (unsigned i = 0; i != NumSrcElts; ++i)
1496        ExtMask.push_back(Builder.getInt32(i));
1497      ExtMask.resize(NumDstElts, llvm::UndefValue::get(Int32Ty));
1498      llvm::Value *ExtMaskV = llvm::ConstantVector::get(ExtMask);
1499      llvm::Value *ExtSrcVal =
1500        Builder.CreateShuffleVector(SrcVal,
1501                                    llvm::UndefValue::get(SrcVal->getType()),
1502                                    ExtMaskV);
1503      // build identity
1504      SmallVector<llvm::Constant*, 4> Mask;
1505      for (unsigned i = 0; i != NumDstElts; ++i)
1506        Mask.push_back(Builder.getInt32(i));
1507
1508      // modify when what gets shuffled in
1509      for (unsigned i = 0; i != NumSrcElts; ++i)
1510        Mask[getAccessedFieldNo(i, Elts)] = Builder.getInt32(i+NumDstElts);
1511      llvm::Value *MaskV = llvm::ConstantVector::get(Mask);
1512      Vec = Builder.CreateShuffleVector(Vec, ExtSrcVal, MaskV);
1513    } else {
1514      // We should never shorten the vector
1515      llvm_unreachable("unexpected shorten vector length");
1516    }
1517  } else {
1518    // If the Src is a scalar (not a vector) it must be updating one element.
1519    unsigned InIdx = getAccessedFieldNo(0, Elts);
1520    llvm::Value *Elt = llvm::ConstantInt::get(Int32Ty, InIdx);
1521    Vec = Builder.CreateInsertElement(Vec, SrcVal, Elt);
1522  }
1523
1524  llvm::StoreInst *Store = Builder.CreateStore(Vec, Dst.getExtVectorAddr(),
1525                                               Dst.isVolatileQualified());
1526  Store->setAlignment(Dst.getAlignment().getQuantity());
1527}
1528
1529// setObjCGCLValueClass - sets class of he lvalue for the purpose of
1530// generating write-barries API. It is currently a global, ivar,
1531// or neither.
1532static void setObjCGCLValueClass(const ASTContext &Ctx, const Expr *E,
1533                                 LValue &LV,
1534                                 bool IsMemberAccess=false) {
1535  if (Ctx.getLangOpts().getGC() == LangOptions::NonGC)
1536    return;
1537
1538  if (isa<ObjCIvarRefExpr>(E)) {
1539    QualType ExpTy = E->getType();
1540    if (IsMemberAccess && ExpTy->isPointerType()) {
1541      // If ivar is a structure pointer, assigning to field of
1542      // this struct follows gcc's behavior and makes it a non-ivar
1543      // writer-barrier conservatively.
1544      ExpTy = ExpTy->getAs<PointerType>()->getPointeeType();
1545      if (ExpTy->isRecordType()) {
1546        LV.setObjCIvar(false);
1547        return;
1548      }
1549    }
1550    LV.setObjCIvar(true);
1551    ObjCIvarRefExpr *Exp = cast<ObjCIvarRefExpr>(const_cast<Expr*>(E));
1552    LV.setBaseIvarExp(Exp->getBase());
1553    LV.setObjCArray(E->getType()->isArrayType());
1554    return;
1555  }
1556
1557  if (const DeclRefExpr *Exp = dyn_cast<DeclRefExpr>(E)) {
1558    if (const VarDecl *VD = dyn_cast<VarDecl>(Exp->getDecl())) {
1559      if (VD->hasGlobalStorage()) {
1560        LV.setGlobalObjCRef(true);
1561        LV.setThreadLocalRef(VD->isThreadSpecified());
1562      }
1563    }
1564    LV.setObjCArray(E->getType()->isArrayType());
1565    return;
1566  }
1567
1568  if (const UnaryOperator *Exp = dyn_cast<UnaryOperator>(E)) {
1569    setObjCGCLValueClass(Ctx, Exp->getSubExpr(), LV, IsMemberAccess);
1570    return;
1571  }
1572
1573  if (const ParenExpr *Exp = dyn_cast<ParenExpr>(E)) {
1574    setObjCGCLValueClass(Ctx, Exp->getSubExpr(), LV, IsMemberAccess);
1575    if (LV.isObjCIvar()) {
1576      // If cast is to a structure pointer, follow gcc's behavior and make it
1577      // a non-ivar write-barrier.
1578      QualType ExpTy = E->getType();
1579      if (ExpTy->isPointerType())
1580        ExpTy = ExpTy->getAs<PointerType>()->getPointeeType();
1581      if (ExpTy->isRecordType())
1582        LV.setObjCIvar(false);
1583    }
1584    return;
1585  }
1586
1587  if (const GenericSelectionExpr *Exp = dyn_cast<GenericSelectionExpr>(E)) {
1588    setObjCGCLValueClass(Ctx, Exp->getResultExpr(), LV);
1589    return;
1590  }
1591
1592  if (const ImplicitCastExpr *Exp = dyn_cast<ImplicitCastExpr>(E)) {
1593    setObjCGCLValueClass(Ctx, Exp->getSubExpr(), LV, IsMemberAccess);
1594    return;
1595  }
1596
1597  if (const CStyleCastExpr *Exp = dyn_cast<CStyleCastExpr>(E)) {
1598    setObjCGCLValueClass(Ctx, Exp->getSubExpr(), LV, IsMemberAccess);
1599    return;
1600  }
1601
1602  if (const ObjCBridgedCastExpr *Exp = dyn_cast<ObjCBridgedCastExpr>(E)) {
1603    setObjCGCLValueClass(Ctx, Exp->getSubExpr(), LV, IsMemberAccess);
1604    return;
1605  }
1606
1607  if (const ArraySubscriptExpr *Exp = dyn_cast<ArraySubscriptExpr>(E)) {
1608    setObjCGCLValueClass(Ctx, Exp->getBase(), LV);
1609    if (LV.isObjCIvar() && !LV.isObjCArray())
1610      // Using array syntax to assigning to what an ivar points to is not
1611      // same as assigning to the ivar itself. {id *Names;} Names[i] = 0;
1612      LV.setObjCIvar(false);
1613    else if (LV.isGlobalObjCRef() && !LV.isObjCArray())
1614      // Using array syntax to assigning to what global points to is not
1615      // same as assigning to the global itself. {id *G;} G[i] = 0;
1616      LV.setGlobalObjCRef(false);
1617    return;
1618  }
1619
1620  if (const MemberExpr *Exp = dyn_cast<MemberExpr>(E)) {
1621    setObjCGCLValueClass(Ctx, Exp->getBase(), LV, true);
1622    // We don't know if member is an 'ivar', but this flag is looked at
1623    // only in the context of LV.isObjCIvar().
1624    LV.setObjCArray(E->getType()->isArrayType());
1625    return;
1626  }
1627}
1628
1629static llvm::Value *
1630EmitBitCastOfLValueToProperType(CodeGenFunction &CGF,
1631                                llvm::Value *V, llvm::Type *IRType,
1632                                StringRef Name = StringRef()) {
1633  unsigned AS = cast<llvm::PointerType>(V->getType())->getAddressSpace();
1634  return CGF.Builder.CreateBitCast(V, IRType->getPointerTo(AS), Name);
1635}
1636
1637static LValue EmitGlobalVarDeclLValue(CodeGenFunction &CGF,
1638                                      const Expr *E, const VarDecl *VD) {
1639  assert((VD->hasExternalStorage() || VD->isFileVarDecl()) &&
1640         "Var decl must have external storage or be a file var decl!");
1641
1642  llvm::Value *V = CGF.CGM.GetAddrOfGlobalVar(VD);
1643  llvm::Type *RealVarTy = CGF.getTypes().ConvertTypeForMem(VD->getType());
1644  V = EmitBitCastOfLValueToProperType(CGF, V, RealVarTy);
1645  CharUnits Alignment = CGF.getContext().getDeclAlign(VD);
1646  QualType T = E->getType();
1647  LValue LV;
1648  if (VD->getType()->isReferenceType()) {
1649    llvm::LoadInst *LI = CGF.Builder.CreateLoad(V);
1650    LI->setAlignment(Alignment.getQuantity());
1651    V = LI;
1652    LV = CGF.MakeNaturalAlignAddrLValue(V, T);
1653  } else {
1654    LV = CGF.MakeAddrLValue(V, E->getType(), Alignment);
1655  }
1656  setObjCGCLValueClass(CGF.getContext(), E, LV);
1657  return LV;
1658}
1659
1660static LValue EmitFunctionDeclLValue(CodeGenFunction &CGF,
1661                                     const Expr *E, const FunctionDecl *FD) {
1662  llvm::Value *V = CGF.CGM.GetAddrOfFunction(FD);
1663  if (!FD->hasPrototype()) {
1664    if (const FunctionProtoType *Proto =
1665            FD->getType()->getAs<FunctionProtoType>()) {
1666      // Ugly case: for a K&R-style definition, the type of the definition
1667      // isn't the same as the type of a use.  Correct for this with a
1668      // bitcast.
1669      QualType NoProtoType =
1670          CGF.getContext().getFunctionNoProtoType(Proto->getResultType());
1671      NoProtoType = CGF.getContext().getPointerType(NoProtoType);
1672      V = CGF.Builder.CreateBitCast(V, CGF.ConvertType(NoProtoType));
1673    }
1674  }
1675  CharUnits Alignment = CGF.getContext().getDeclAlign(FD);
1676  return CGF.MakeAddrLValue(V, E->getType(), Alignment);
1677}
1678
1679LValue CodeGenFunction::EmitDeclRefLValue(const DeclRefExpr *E) {
1680  const NamedDecl *ND = E->getDecl();
1681  CharUnits Alignment = getContext().getDeclAlign(ND);
1682  QualType T = E->getType();
1683
1684  // FIXME: We should be able to assert this for FunctionDecls as well!
1685  // FIXME: We should be able to assert this for all DeclRefExprs, not just
1686  // those with a valid source location.
1687  assert((ND->isUsed(false) || !isa<VarDecl>(ND) ||
1688          !E->getLocation().isValid()) &&
1689         "Should not use decl without marking it used!");
1690
1691  if (ND->hasAttr<WeakRefAttr>()) {
1692    const ValueDecl *VD = cast<ValueDecl>(ND);
1693    llvm::Constant *Aliasee = CGM.GetWeakRefReference(VD);
1694    return MakeAddrLValue(Aliasee, E->getType(), Alignment);
1695  }
1696
1697  if (const VarDecl *VD = dyn_cast<VarDecl>(ND)) {
1698    // Check if this is a global variable.
1699    if (VD->hasExternalStorage() || VD->isFileVarDecl())
1700      return EmitGlobalVarDeclLValue(*this, E, VD);
1701
1702    bool isBlockVariable = VD->hasAttr<BlocksAttr>();
1703
1704    bool NonGCable = VD->hasLocalStorage() &&
1705                     !VD->getType()->isReferenceType() &&
1706                     !isBlockVariable;
1707
1708    llvm::Value *V = LocalDeclMap[VD];
1709    if (!V && VD->isStaticLocal())
1710      V = CGM.getStaticLocalDeclAddress(VD);
1711
1712    // Use special handling for lambdas.
1713    if (!V) {
1714      if (FieldDecl *FD = LambdaCaptureFields.lookup(VD)) {
1715        QualType LambdaTagType = getContext().getTagDeclType(FD->getParent());
1716        LValue LambdaLV = MakeNaturalAlignAddrLValue(CXXABIThisValue,
1717                                                     LambdaTagType);
1718        return EmitLValueForField(LambdaLV, FD);
1719      }
1720
1721      assert(isa<BlockDecl>(CurCodeDecl) && E->refersToEnclosingLocal());
1722      CharUnits alignment = getContext().getDeclAlign(VD);
1723      return MakeAddrLValue(GetAddrOfBlockDecl(VD, isBlockVariable),
1724                            E->getType(), alignment);
1725    }
1726
1727    assert(V && "DeclRefExpr not entered in LocalDeclMap?");
1728
1729    if (isBlockVariable)
1730      V = BuildBlockByrefAddress(V, VD);
1731
1732    LValue LV;
1733    if (VD->getType()->isReferenceType()) {
1734      llvm::LoadInst *LI = Builder.CreateLoad(V);
1735      LI->setAlignment(Alignment.getQuantity());
1736      V = LI;
1737      LV = MakeNaturalAlignAddrLValue(V, T);
1738    } else {
1739      LV = MakeAddrLValue(V, T, Alignment);
1740    }
1741
1742    if (NonGCable) {
1743      LV.getQuals().removeObjCGCAttr();
1744      LV.setNonGC(true);
1745    }
1746    setObjCGCLValueClass(getContext(), E, LV);
1747    return LV;
1748  }
1749
1750  if (const FunctionDecl *fn = dyn_cast<FunctionDecl>(ND))
1751    return EmitFunctionDeclLValue(*this, E, fn);
1752
1753  llvm_unreachable("Unhandled DeclRefExpr");
1754}
1755
1756LValue CodeGenFunction::EmitUnaryOpLValue(const UnaryOperator *E) {
1757  // __extension__ doesn't affect lvalue-ness.
1758  if (E->getOpcode() == UO_Extension)
1759    return EmitLValue(E->getSubExpr());
1760
1761  QualType ExprTy = getContext().getCanonicalType(E->getSubExpr()->getType());
1762  switch (E->getOpcode()) {
1763  default: llvm_unreachable("Unknown unary operator lvalue!");
1764  case UO_Deref: {
1765    QualType T = E->getSubExpr()->getType()->getPointeeType();
1766    assert(!T.isNull() && "CodeGenFunction::EmitUnaryOpLValue: Illegal type");
1767
1768    LValue LV = MakeNaturalAlignAddrLValue(EmitScalarExpr(E->getSubExpr()), T);
1769    LV.getQuals().setAddressSpace(ExprTy.getAddressSpace());
1770
1771    // We should not generate __weak write barrier on indirect reference
1772    // of a pointer to object; as in void foo (__weak id *param); *param = 0;
1773    // But, we continue to generate __strong write barrier on indirect write
1774    // into a pointer to object.
1775    if (getContext().getLangOpts().ObjC1 &&
1776        getContext().getLangOpts().getGC() != LangOptions::NonGC &&
1777        LV.isObjCWeak())
1778      LV.setNonGC(!E->isOBJCGCCandidate(getContext()));
1779    return LV;
1780  }
1781  case UO_Real:
1782  case UO_Imag: {
1783    LValue LV = EmitLValue(E->getSubExpr());
1784    assert(LV.isSimple() && "real/imag on non-ordinary l-value");
1785    llvm::Value *Addr = LV.getAddress();
1786
1787    // __real is valid on scalars.  This is a faster way of testing that.
1788    // __imag can only produce an rvalue on scalars.
1789    if (E->getOpcode() == UO_Real &&
1790        !cast<llvm::PointerType>(Addr->getType())
1791           ->getElementType()->isStructTy()) {
1792      assert(E->getSubExpr()->getType()->isArithmeticType());
1793      return LV;
1794    }
1795
1796    assert(E->getSubExpr()->getType()->isAnyComplexType());
1797
1798    unsigned Idx = E->getOpcode() == UO_Imag;
1799    return MakeAddrLValue(Builder.CreateStructGEP(LV.getAddress(),
1800                                                  Idx, "idx"),
1801                          ExprTy);
1802  }
1803  case UO_PreInc:
1804  case UO_PreDec: {
1805    LValue LV = EmitLValue(E->getSubExpr());
1806    bool isInc = E->getOpcode() == UO_PreInc;
1807
1808    if (E->getType()->isAnyComplexType())
1809      EmitComplexPrePostIncDec(E, LV, isInc, true/*isPre*/);
1810    else
1811      EmitScalarPrePostIncDec(E, LV, isInc, true/*isPre*/);
1812    return LV;
1813  }
1814  }
1815}
1816
1817LValue CodeGenFunction::EmitStringLiteralLValue(const StringLiteral *E) {
1818  return MakeAddrLValue(CGM.GetAddrOfConstantStringFromLiteral(E),
1819                        E->getType());
1820}
1821
1822LValue CodeGenFunction::EmitObjCEncodeExprLValue(const ObjCEncodeExpr *E) {
1823  return MakeAddrLValue(CGM.GetAddrOfConstantStringFromObjCEncode(E),
1824                        E->getType());
1825}
1826
1827static llvm::Constant*
1828GetAddrOfConstantWideString(StringRef Str,
1829                            const char *GlobalName,
1830                            ASTContext &Context,
1831                            QualType Ty, SourceLocation Loc,
1832                            CodeGenModule &CGM) {
1833
1834  StringLiteral *SL = StringLiteral::Create(Context,
1835                                            Str,
1836                                            StringLiteral::Wide,
1837                                            /*Pascal = */false,
1838                                            Ty, Loc);
1839  llvm::Constant *C = CGM.GetConstantArrayFromStringLiteral(SL);
1840  llvm::GlobalVariable *GV =
1841    new llvm::GlobalVariable(CGM.getModule(), C->getType(),
1842                             !CGM.getLangOpts().WritableStrings,
1843                             llvm::GlobalValue::PrivateLinkage,
1844                             C, GlobalName);
1845  const unsigned WideAlignment =
1846    Context.getTypeAlignInChars(Ty).getQuantity();
1847  GV->setAlignment(WideAlignment);
1848  return GV;
1849}
1850
1851static void ConvertUTF8ToWideString(unsigned CharByteWidth, StringRef Source,
1852                                    SmallString<32>& Target) {
1853  Target.resize(CharByteWidth * (Source.size() + 1));
1854  char* ResultPtr = &Target[0];
1855  bool success = ConvertUTF8toWide(CharByteWidth, Source, ResultPtr);
1856  (void)success;
1857  assert(success);
1858  Target.resize(ResultPtr - &Target[0]);
1859}
1860
1861LValue CodeGenFunction::EmitPredefinedLValue(const PredefinedExpr *E) {
1862  switch (E->getIdentType()) {
1863  default:
1864    return EmitUnsupportedLValue(E, "predefined expression");
1865
1866  case PredefinedExpr::Func:
1867  case PredefinedExpr::Function:
1868  case PredefinedExpr::LFunction:
1869  case PredefinedExpr::PrettyFunction: {
1870    unsigned IdentType = E->getIdentType();
1871    std::string GlobalVarName;
1872
1873    switch (IdentType) {
1874    default: llvm_unreachable("Invalid type");
1875    case PredefinedExpr::Func:
1876      GlobalVarName = "__func__.";
1877      break;
1878    case PredefinedExpr::Function:
1879      GlobalVarName = "__FUNCTION__.";
1880      break;
1881    case PredefinedExpr::LFunction:
1882      GlobalVarName = "L__FUNCTION__.";
1883      break;
1884    case PredefinedExpr::PrettyFunction:
1885      GlobalVarName = "__PRETTY_FUNCTION__.";
1886      break;
1887    }
1888
1889    StringRef FnName = CurFn->getName();
1890    if (FnName.startswith("\01"))
1891      FnName = FnName.substr(1);
1892    GlobalVarName += FnName;
1893
1894    const Decl *CurDecl = CurCodeDecl;
1895    if (CurDecl == 0)
1896      CurDecl = getContext().getTranslationUnitDecl();
1897
1898    std::string FunctionName =
1899        (isa<BlockDecl>(CurDecl)
1900         ? FnName.str()
1901         : PredefinedExpr::ComputeName((PredefinedExpr::IdentType)IdentType,
1902                                       CurDecl));
1903
1904    const Type* ElemType = E->getType()->getArrayElementTypeNoTypeQual();
1905    llvm::Constant *C;
1906    if (ElemType->isWideCharType()) {
1907      SmallString<32> RawChars;
1908      ConvertUTF8ToWideString(
1909          getContext().getTypeSizeInChars(ElemType).getQuantity(),
1910          FunctionName, RawChars);
1911      C = GetAddrOfConstantWideString(RawChars,
1912                                      GlobalVarName.c_str(),
1913                                      getContext(),
1914                                      E->getType(),
1915                                      E->getLocation(),
1916                                      CGM);
1917    } else {
1918      C = CGM.GetAddrOfConstantCString(FunctionName,
1919                                       GlobalVarName.c_str(),
1920                                       1);
1921    }
1922    return MakeAddrLValue(C, E->getType());
1923  }
1924  }
1925}
1926
1927void CodeGenFunction::EmitCheck(llvm::Value *Checked) {
1928  const CodeGenOptions &GCO = CGM.getCodeGenOpts();
1929
1930  llvm::BasicBlock *Cont = createBasicBlock("cont");
1931
1932  // If we are not optimzing, don't collapse all calls to trap in the function
1933  // to the same call, that way, in the debugger they can see which operation
1934  // did in fact fail.  If we are optimizing, we collapse all calls to trap down
1935  // to just one per function to save on codesize.
1936  bool NeedNewTrapBB = !GCO.OptimizationLevel || !TrapBB;
1937
1938  if (NeedNewTrapBB)
1939    TrapBB = createBasicBlock("trap");
1940
1941  Builder.CreateCondBr(Checked, Cont, TrapBB);
1942
1943  if (NeedNewTrapBB) {
1944    EmitBlock(TrapBB);
1945
1946    llvm::Value *F = CGM.getIntrinsic(llvm::Intrinsic::trap);
1947    llvm::CallInst *TrapCall = Builder.CreateCall(F);
1948    TrapCall->setDoesNotReturn();
1949    TrapCall->setDoesNotThrow();
1950    Builder.CreateUnreachable();
1951  }
1952
1953  EmitBlock(Cont);
1954}
1955
1956/// isSimpleArrayDecayOperand - If the specified expr is a simple decay from an
1957/// array to pointer, return the array subexpression.
1958static const Expr *isSimpleArrayDecayOperand(const Expr *E) {
1959  // If this isn't just an array->pointer decay, bail out.
1960  const CastExpr *CE = dyn_cast<CastExpr>(E);
1961  if (CE == 0 || CE->getCastKind() != CK_ArrayToPointerDecay)
1962    return 0;
1963
1964  // If this is a decay from variable width array, bail out.
1965  const Expr *SubExpr = CE->getSubExpr();
1966  if (SubExpr->getType()->isVariableArrayType())
1967    return 0;
1968
1969  return SubExpr;
1970}
1971
1972LValue CodeGenFunction::EmitArraySubscriptExpr(const ArraySubscriptExpr *E) {
1973  // The index must always be an integer, which is not an aggregate.  Emit it.
1974  llvm::Value *Idx = EmitScalarExpr(E->getIdx());
1975  QualType IdxTy  = E->getIdx()->getType();
1976  bool IdxSigned = IdxTy->isSignedIntegerOrEnumerationType();
1977
1978  // If the base is a vector type, then we are forming a vector element lvalue
1979  // with this subscript.
1980  if (E->getBase()->getType()->isVectorType()) {
1981    // Emit the vector as an lvalue to get its address.
1982    LValue LHS = EmitLValue(E->getBase());
1983    assert(LHS.isSimple() && "Can only subscript lvalue vectors here!");
1984    Idx = Builder.CreateIntCast(Idx, Int32Ty, IdxSigned, "vidx");
1985    return LValue::MakeVectorElt(LHS.getAddress(), Idx,
1986                                 E->getBase()->getType(), LHS.getAlignment());
1987  }
1988
1989  // Extend or truncate the index type to 32 or 64-bits.
1990  if (Idx->getType() != IntPtrTy)
1991    Idx = Builder.CreateIntCast(Idx, IntPtrTy, IdxSigned, "idxprom");
1992
1993  // We know that the pointer points to a type of the correct size, unless the
1994  // size is a VLA or Objective-C interface.
1995  llvm::Value *Address = 0;
1996  CharUnits ArrayAlignment;
1997  if (const VariableArrayType *vla =
1998        getContext().getAsVariableArrayType(E->getType())) {
1999    // The base must be a pointer, which is not an aggregate.  Emit
2000    // it.  It needs to be emitted first in case it's what captures
2001    // the VLA bounds.
2002    Address = EmitScalarExpr(E->getBase());
2003
2004    // The element count here is the total number of non-VLA elements.
2005    llvm::Value *numElements = getVLASize(vla).first;
2006
2007    // Effectively, the multiply by the VLA size is part of the GEP.
2008    // GEP indexes are signed, and scaling an index isn't permitted to
2009    // signed-overflow, so we use the same semantics for our explicit
2010    // multiply.  We suppress this if overflow is not undefined behavior.
2011    if (getLangOpts().isSignedOverflowDefined()) {
2012      Idx = Builder.CreateMul(Idx, numElements);
2013      Address = Builder.CreateGEP(Address, Idx, "arrayidx");
2014    } else {
2015      Idx = Builder.CreateNSWMul(Idx, numElements);
2016      Address = Builder.CreateInBoundsGEP(Address, Idx, "arrayidx");
2017    }
2018  } else if (const ObjCObjectType *OIT = E->getType()->getAs<ObjCObjectType>()){
2019    // Indexing over an interface, as in "NSString *P; P[4];"
2020    llvm::Value *InterfaceSize =
2021      llvm::ConstantInt::get(Idx->getType(),
2022          getContext().getTypeSizeInChars(OIT).getQuantity());
2023
2024    Idx = Builder.CreateMul(Idx, InterfaceSize);
2025
2026    // The base must be a pointer, which is not an aggregate.  Emit it.
2027    llvm::Value *Base = EmitScalarExpr(E->getBase());
2028    Address = EmitCastToVoidPtr(Base);
2029    Address = Builder.CreateGEP(Address, Idx, "arrayidx");
2030    Address = Builder.CreateBitCast(Address, Base->getType());
2031  } else if (const Expr *Array = isSimpleArrayDecayOperand(E->getBase())) {
2032    // If this is A[i] where A is an array, the frontend will have decayed the
2033    // base to be a ArrayToPointerDecay implicit cast.  While correct, it is
2034    // inefficient at -O0 to emit a "gep A, 0, 0" when codegen'ing it, then a
2035    // "gep x, i" here.  Emit one "gep A, 0, i".
2036    assert(Array->getType()->isArrayType() &&
2037           "Array to pointer decay must have array source type!");
2038    LValue ArrayLV = EmitLValue(Array);
2039    llvm::Value *ArrayPtr = ArrayLV.getAddress();
2040    llvm::Value *Zero = llvm::ConstantInt::get(Int32Ty, 0);
2041    llvm::Value *Args[] = { Zero, Idx };
2042
2043    // Propagate the alignment from the array itself to the result.
2044    ArrayAlignment = ArrayLV.getAlignment();
2045
2046    if (getContext().getLangOpts().isSignedOverflowDefined())
2047      Address = Builder.CreateGEP(ArrayPtr, Args, "arrayidx");
2048    else
2049      Address = Builder.CreateInBoundsGEP(ArrayPtr, Args, "arrayidx");
2050  } else {
2051    // The base must be a pointer, which is not an aggregate.  Emit it.
2052    llvm::Value *Base = EmitScalarExpr(E->getBase());
2053    if (getContext().getLangOpts().isSignedOverflowDefined())
2054      Address = Builder.CreateGEP(Base, Idx, "arrayidx");
2055    else
2056      Address = Builder.CreateInBoundsGEP(Base, Idx, "arrayidx");
2057  }
2058
2059  QualType T = E->getBase()->getType()->getPointeeType();
2060  assert(!T.isNull() &&
2061         "CodeGenFunction::EmitArraySubscriptExpr(): Illegal base type");
2062
2063
2064  // Limit the alignment to that of the result type.
2065  LValue LV;
2066  if (!ArrayAlignment.isZero()) {
2067    CharUnits Align = getContext().getTypeAlignInChars(T);
2068    ArrayAlignment = std::min(Align, ArrayAlignment);
2069    LV = MakeAddrLValue(Address, T, ArrayAlignment);
2070  } else {
2071    LV = MakeNaturalAlignAddrLValue(Address, T);
2072  }
2073
2074  LV.getQuals().setAddressSpace(E->getBase()->getType().getAddressSpace());
2075
2076  if (getContext().getLangOpts().ObjC1 &&
2077      getContext().getLangOpts().getGC() != LangOptions::NonGC) {
2078    LV.setNonGC(!E->isOBJCGCCandidate(getContext()));
2079    setObjCGCLValueClass(getContext(), E, LV);
2080  }
2081  return LV;
2082}
2083
2084static
2085llvm::Constant *GenerateConstantVector(CGBuilderTy &Builder,
2086                                       SmallVector<unsigned, 4> &Elts) {
2087  SmallVector<llvm::Constant*, 4> CElts;
2088  for (unsigned i = 0, e = Elts.size(); i != e; ++i)
2089    CElts.push_back(Builder.getInt32(Elts[i]));
2090
2091  return llvm::ConstantVector::get(CElts);
2092}
2093
2094LValue CodeGenFunction::
2095EmitExtVectorElementExpr(const ExtVectorElementExpr *E) {
2096  // Emit the base vector as an l-value.
2097  LValue Base;
2098
2099  // ExtVectorElementExpr's base can either be a vector or pointer to vector.
2100  if (E->isArrow()) {
2101    // If it is a pointer to a vector, emit the address and form an lvalue with
2102    // it.
2103    llvm::Value *Ptr = EmitScalarExpr(E->getBase());
2104    const PointerType *PT = E->getBase()->getType()->getAs<PointerType>();
2105    Base = MakeAddrLValue(Ptr, PT->getPointeeType());
2106    Base.getQuals().removeObjCGCAttr();
2107  } else if (E->getBase()->isGLValue()) {
2108    // Otherwise, if the base is an lvalue ( as in the case of foo.x.x),
2109    // emit the base as an lvalue.
2110    assert(E->getBase()->getType()->isVectorType());
2111    Base = EmitLValue(E->getBase());
2112  } else {
2113    // Otherwise, the base is a normal rvalue (as in (V+V).x), emit it as such.
2114    assert(E->getBase()->getType()->isVectorType() &&
2115           "Result must be a vector");
2116    llvm::Value *Vec = EmitScalarExpr(E->getBase());
2117
2118    // Store the vector to memory (because LValue wants an address).
2119    llvm::Value *VecMem = CreateMemTemp(E->getBase()->getType());
2120    Builder.CreateStore(Vec, VecMem);
2121    Base = MakeAddrLValue(VecMem, E->getBase()->getType());
2122  }
2123
2124  QualType type =
2125    E->getType().withCVRQualifiers(Base.getQuals().getCVRQualifiers());
2126
2127  // Encode the element access list into a vector of unsigned indices.
2128  SmallVector<unsigned, 4> Indices;
2129  E->getEncodedElementAccess(Indices);
2130
2131  if (Base.isSimple()) {
2132    llvm::Constant *CV = GenerateConstantVector(Builder, Indices);
2133    return LValue::MakeExtVectorElt(Base.getAddress(), CV, type,
2134                                    Base.getAlignment());
2135  }
2136  assert(Base.isExtVectorElt() && "Can only subscript lvalue vec elts here!");
2137
2138  llvm::Constant *BaseElts = Base.getExtVectorElts();
2139  SmallVector<llvm::Constant *, 4> CElts;
2140
2141  for (unsigned i = 0, e = Indices.size(); i != e; ++i)
2142    CElts.push_back(BaseElts->getAggregateElement(Indices[i]));
2143  llvm::Constant *CV = llvm::ConstantVector::get(CElts);
2144  return LValue::MakeExtVectorElt(Base.getExtVectorAddr(), CV, type,
2145                                  Base.getAlignment());
2146}
2147
2148LValue CodeGenFunction::EmitMemberExpr(const MemberExpr *E) {
2149  Expr *BaseExpr = E->getBase();
2150
2151  // If this is s.x, emit s as an lvalue.  If it is s->x, emit s as a scalar.
2152  LValue BaseLV;
2153  if (E->isArrow()) {
2154    llvm::Value *Ptr = EmitScalarExpr(BaseExpr);
2155    QualType PtrTy = BaseExpr->getType()->getPointeeType();
2156    EmitTypeCheck(TCK_MemberAccess, Ptr, PtrTy);
2157    BaseLV = MakeNaturalAlignAddrLValue(Ptr, PtrTy);
2158  } else
2159    BaseLV = EmitCheckedLValue(BaseExpr, TCK_MemberAccess);
2160
2161  NamedDecl *ND = E->getMemberDecl();
2162  if (FieldDecl *Field = dyn_cast<FieldDecl>(ND)) {
2163    LValue LV = EmitLValueForField(BaseLV, Field);
2164    setObjCGCLValueClass(getContext(), E, LV);
2165    return LV;
2166  }
2167
2168  if (VarDecl *VD = dyn_cast<VarDecl>(ND))
2169    return EmitGlobalVarDeclLValue(*this, E, VD);
2170
2171  if (const FunctionDecl *FD = dyn_cast<FunctionDecl>(ND))
2172    return EmitFunctionDeclLValue(*this, E, FD);
2173
2174  llvm_unreachable("Unhandled member declaration!");
2175}
2176
2177LValue CodeGenFunction::EmitLValueForField(LValue base,
2178                                           const FieldDecl *field) {
2179  if (field->isBitField()) {
2180    const CGRecordLayout &RL =
2181      CGM.getTypes().getCGRecordLayout(field->getParent());
2182    const CGBitFieldInfo &Info = RL.getBitFieldInfo(field);
2183    QualType fieldType =
2184      field->getType().withCVRQualifiers(base.getVRQualifiers());
2185    return LValue::MakeBitfield(base.getAddress(), Info, fieldType,
2186                                base.getAlignment());
2187  }
2188
2189  const RecordDecl *rec = field->getParent();
2190  QualType type = field->getType();
2191  CharUnits alignment = getContext().getDeclAlign(field);
2192
2193  // FIXME: It should be impossible to have an LValue without alignment for a
2194  // complete type.
2195  if (!base.getAlignment().isZero())
2196    alignment = std::min(alignment, base.getAlignment());
2197
2198  bool mayAlias = rec->hasAttr<MayAliasAttr>();
2199
2200  llvm::Value *addr = base.getAddress();
2201  unsigned cvr = base.getVRQualifiers();
2202  if (rec->isUnion()) {
2203    // For unions, there is no pointer adjustment.
2204    assert(!type->isReferenceType() && "union has reference member");
2205  } else {
2206    // For structs, we GEP to the field that the record layout suggests.
2207    unsigned idx = CGM.getTypes().getCGRecordLayout(rec).getLLVMFieldNo(field);
2208    addr = Builder.CreateStructGEP(addr, idx, field->getName());
2209
2210    // If this is a reference field, load the reference right now.
2211    if (const ReferenceType *refType = type->getAs<ReferenceType>()) {
2212      llvm::LoadInst *load = Builder.CreateLoad(addr, "ref");
2213      if (cvr & Qualifiers::Volatile) load->setVolatile(true);
2214      load->setAlignment(alignment.getQuantity());
2215
2216      if (CGM.shouldUseTBAA()) {
2217        llvm::MDNode *tbaa;
2218        if (mayAlias)
2219          tbaa = CGM.getTBAAInfo(getContext().CharTy);
2220        else
2221          tbaa = CGM.getTBAAInfo(type);
2222        CGM.DecorateInstruction(load, tbaa);
2223      }
2224
2225      addr = load;
2226      mayAlias = false;
2227      type = refType->getPointeeType();
2228      if (type->isIncompleteType())
2229        alignment = CharUnits();
2230      else
2231        alignment = getContext().getTypeAlignInChars(type);
2232      cvr = 0; // qualifiers don't recursively apply to referencee
2233    }
2234  }
2235
2236  // Make sure that the address is pointing to the right type.  This is critical
2237  // for both unions and structs.  A union needs a bitcast, a struct element
2238  // will need a bitcast if the LLVM type laid out doesn't match the desired
2239  // type.
2240  addr = EmitBitCastOfLValueToProperType(*this, addr,
2241                                         CGM.getTypes().ConvertTypeForMem(type),
2242                                         field->getName());
2243
2244  if (field->hasAttr<AnnotateAttr>())
2245    addr = EmitFieldAnnotations(field, addr);
2246
2247  LValue LV = MakeAddrLValue(addr, type, alignment);
2248  LV.getQuals().addCVRQualifiers(cvr);
2249
2250  // __weak attribute on a field is ignored.
2251  if (LV.getQuals().getObjCGCAttr() == Qualifiers::Weak)
2252    LV.getQuals().removeObjCGCAttr();
2253
2254  // Fields of may_alias structs act like 'char' for TBAA purposes.
2255  // FIXME: this should get propagated down through anonymous structs
2256  // and unions.
2257  if (mayAlias && LV.getTBAAInfo())
2258    LV.setTBAAInfo(CGM.getTBAAInfo(getContext().CharTy));
2259
2260  return LV;
2261}
2262
2263LValue
2264CodeGenFunction::EmitLValueForFieldInitialization(LValue Base,
2265                                                  const FieldDecl *Field) {
2266  QualType FieldType = Field->getType();
2267
2268  if (!FieldType->isReferenceType())
2269    return EmitLValueForField(Base, Field);
2270
2271  const CGRecordLayout &RL =
2272    CGM.getTypes().getCGRecordLayout(Field->getParent());
2273  unsigned idx = RL.getLLVMFieldNo(Field);
2274  llvm::Value *V = Builder.CreateStructGEP(Base.getAddress(), idx);
2275  assert(!FieldType.getObjCGCAttr() && "fields cannot have GC attrs");
2276
2277  // Make sure that the address is pointing to the right type.  This is critical
2278  // for both unions and structs.  A union needs a bitcast, a struct element
2279  // will need a bitcast if the LLVM type laid out doesn't match the desired
2280  // type.
2281  llvm::Type *llvmType = ConvertTypeForMem(FieldType);
2282  V = EmitBitCastOfLValueToProperType(*this, V, llvmType, Field->getName());
2283
2284  CharUnits Alignment = getContext().getDeclAlign(Field);
2285
2286  // FIXME: It should be impossible to have an LValue without alignment for a
2287  // complete type.
2288  if (!Base.getAlignment().isZero())
2289    Alignment = std::min(Alignment, Base.getAlignment());
2290
2291  return MakeAddrLValue(V, FieldType, Alignment);
2292}
2293
2294LValue CodeGenFunction::EmitCompoundLiteralLValue(const CompoundLiteralExpr *E){
2295  if (E->isFileScope()) {
2296    llvm::Value *GlobalPtr = CGM.GetAddrOfConstantCompoundLiteral(E);
2297    return MakeAddrLValue(GlobalPtr, E->getType());
2298  }
2299  if (E->getType()->isVariablyModifiedType())
2300    // make sure to emit the VLA size.
2301    EmitVariablyModifiedType(E->getType());
2302
2303  llvm::Value *DeclPtr = CreateMemTemp(E->getType(), ".compoundliteral");
2304  const Expr *InitExpr = E->getInitializer();
2305  LValue Result = MakeAddrLValue(DeclPtr, E->getType());
2306
2307  EmitAnyExprToMem(InitExpr, DeclPtr, E->getType().getQualifiers(),
2308                   /*Init*/ true);
2309
2310  return Result;
2311}
2312
2313LValue CodeGenFunction::EmitInitListLValue(const InitListExpr *E) {
2314  if (!E->isGLValue())
2315    // Initializing an aggregate temporary in C++11: T{...}.
2316    return EmitAggExprToLValue(E);
2317
2318  // An lvalue initializer list must be initializing a reference.
2319  assert(E->getNumInits() == 1 && "reference init with multiple values");
2320  return EmitLValue(E->getInit(0));
2321}
2322
2323LValue CodeGenFunction::
2324EmitConditionalOperatorLValue(const AbstractConditionalOperator *expr) {
2325  if (!expr->isGLValue()) {
2326    // ?: here should be an aggregate.
2327    assert((hasAggregateLLVMType(expr->getType()) &&
2328            !expr->getType()->isAnyComplexType()) &&
2329           "Unexpected conditional operator!");
2330    return EmitAggExprToLValue(expr);
2331  }
2332
2333  OpaqueValueMapping binding(*this, expr);
2334
2335  const Expr *condExpr = expr->getCond();
2336  bool CondExprBool;
2337  if (ConstantFoldsToSimpleInteger(condExpr, CondExprBool)) {
2338    const Expr *live = expr->getTrueExpr(), *dead = expr->getFalseExpr();
2339    if (!CondExprBool) std::swap(live, dead);
2340
2341    if (!ContainsLabel(dead))
2342      return EmitLValue(live);
2343  }
2344
2345  llvm::BasicBlock *lhsBlock = createBasicBlock("cond.true");
2346  llvm::BasicBlock *rhsBlock = createBasicBlock("cond.false");
2347  llvm::BasicBlock *contBlock = createBasicBlock("cond.end");
2348
2349  ConditionalEvaluation eval(*this);
2350  EmitBranchOnBoolExpr(condExpr, lhsBlock, rhsBlock);
2351
2352  // Any temporaries created here are conditional.
2353  EmitBlock(lhsBlock);
2354  eval.begin(*this);
2355  LValue lhs = EmitLValue(expr->getTrueExpr());
2356  eval.end(*this);
2357
2358  if (!lhs.isSimple())
2359    return EmitUnsupportedLValue(expr, "conditional operator");
2360
2361  lhsBlock = Builder.GetInsertBlock();
2362  Builder.CreateBr(contBlock);
2363
2364  // Any temporaries created here are conditional.
2365  EmitBlock(rhsBlock);
2366  eval.begin(*this);
2367  LValue rhs = EmitLValue(expr->getFalseExpr());
2368  eval.end(*this);
2369  if (!rhs.isSimple())
2370    return EmitUnsupportedLValue(expr, "conditional operator");
2371  rhsBlock = Builder.GetInsertBlock();
2372
2373  EmitBlock(contBlock);
2374
2375  llvm::PHINode *phi = Builder.CreatePHI(lhs.getAddress()->getType(), 2,
2376                                         "cond-lvalue");
2377  phi->addIncoming(lhs.getAddress(), lhsBlock);
2378  phi->addIncoming(rhs.getAddress(), rhsBlock);
2379  return MakeAddrLValue(phi, expr->getType());
2380}
2381
2382/// EmitCastLValue - Casts are never lvalues unless that cast is to a reference
2383/// type. If the cast is to a reference, we can have the usual lvalue result,
2384/// otherwise if a cast is needed by the code generator in an lvalue context,
2385/// then it must mean that we need the address of an aggregate in order to
2386/// access one of its members.  This can happen for all the reasons that casts
2387/// are permitted with aggregate result, including noop aggregate casts, and
2388/// cast from scalar to union.
2389LValue CodeGenFunction::EmitCastLValue(const CastExpr *E) {
2390  switch (E->getCastKind()) {
2391  case CK_ToVoid:
2392    return EmitUnsupportedLValue(E, "unexpected cast lvalue");
2393
2394  case CK_Dependent:
2395    llvm_unreachable("dependent cast kind in IR gen!");
2396
2397  case CK_BuiltinFnToFnPtr:
2398    llvm_unreachable("builtin functions are handled elsewhere");
2399
2400  // These two casts are currently treated as no-ops, although they could
2401  // potentially be real operations depending on the target's ABI.
2402  case CK_NonAtomicToAtomic:
2403  case CK_AtomicToNonAtomic:
2404
2405  case CK_NoOp:
2406  case CK_LValueToRValue:
2407    if (!E->getSubExpr()->Classify(getContext()).isPRValue()
2408        || E->getType()->isRecordType())
2409      return EmitLValue(E->getSubExpr());
2410    // Fall through to synthesize a temporary.
2411
2412  case CK_BitCast:
2413  case CK_ArrayToPointerDecay:
2414  case CK_FunctionToPointerDecay:
2415  case CK_NullToMemberPointer:
2416  case CK_NullToPointer:
2417  case CK_IntegralToPointer:
2418  case CK_PointerToIntegral:
2419  case CK_PointerToBoolean:
2420  case CK_VectorSplat:
2421  case CK_IntegralCast:
2422  case CK_IntegralToBoolean:
2423  case CK_IntegralToFloating:
2424  case CK_FloatingToIntegral:
2425  case CK_FloatingToBoolean:
2426  case CK_FloatingCast:
2427  case CK_FloatingRealToComplex:
2428  case CK_FloatingComplexToReal:
2429  case CK_FloatingComplexToBoolean:
2430  case CK_FloatingComplexCast:
2431  case CK_FloatingComplexToIntegralComplex:
2432  case CK_IntegralRealToComplex:
2433  case CK_IntegralComplexToReal:
2434  case CK_IntegralComplexToBoolean:
2435  case CK_IntegralComplexCast:
2436  case CK_IntegralComplexToFloatingComplex:
2437  case CK_DerivedToBaseMemberPointer:
2438  case CK_BaseToDerivedMemberPointer:
2439  case CK_MemberPointerToBoolean:
2440  case CK_ReinterpretMemberPointer:
2441  case CK_AnyPointerToBlockPointerCast:
2442  case CK_ARCProduceObject:
2443  case CK_ARCConsumeObject:
2444  case CK_ARCReclaimReturnedObject:
2445  case CK_ARCExtendBlockObject:
2446  case CK_CopyAndAutoreleaseBlockObject: {
2447    // These casts only produce lvalues when we're binding a reference to a
2448    // temporary realized from a (converted) pure rvalue. Emit the expression
2449    // as a value, copy it into a temporary, and return an lvalue referring to
2450    // that temporary.
2451    llvm::Value *V = CreateMemTemp(E->getType(), "ref.temp");
2452    EmitAnyExprToMem(E, V, E->getType().getQualifiers(), false);
2453    return MakeAddrLValue(V, E->getType());
2454  }
2455
2456  case CK_Dynamic: {
2457    LValue LV = EmitLValue(E->getSubExpr());
2458    llvm::Value *V = LV.getAddress();
2459    const CXXDynamicCastExpr *DCE = cast<CXXDynamicCastExpr>(E);
2460    return MakeAddrLValue(EmitDynamicCast(V, DCE), E->getType());
2461  }
2462
2463  case CK_ConstructorConversion:
2464  case CK_UserDefinedConversion:
2465  case CK_CPointerToObjCPointerCast:
2466  case CK_BlockPointerToObjCPointerCast:
2467    return EmitLValue(E->getSubExpr());
2468
2469  case CK_UncheckedDerivedToBase:
2470  case CK_DerivedToBase: {
2471    const RecordType *DerivedClassTy =
2472      E->getSubExpr()->getType()->getAs<RecordType>();
2473    CXXRecordDecl *DerivedClassDecl =
2474      cast<CXXRecordDecl>(DerivedClassTy->getDecl());
2475
2476    LValue LV = EmitLValue(E->getSubExpr());
2477    llvm::Value *This = LV.getAddress();
2478
2479    // Perform the derived-to-base conversion
2480    llvm::Value *Base =
2481      GetAddressOfBaseClass(This, DerivedClassDecl,
2482                            E->path_begin(), E->path_end(),
2483                            /*NullCheckValue=*/false);
2484
2485    return MakeAddrLValue(Base, E->getType());
2486  }
2487  case CK_ToUnion:
2488    return EmitAggExprToLValue(E);
2489  case CK_BaseToDerived: {
2490    const RecordType *DerivedClassTy = E->getType()->getAs<RecordType>();
2491    CXXRecordDecl *DerivedClassDecl =
2492      cast<CXXRecordDecl>(DerivedClassTy->getDecl());
2493
2494    LValue LV = EmitLValue(E->getSubExpr());
2495
2496    // Perform the base-to-derived conversion
2497    llvm::Value *Derived =
2498      GetAddressOfDerivedClass(LV.getAddress(), DerivedClassDecl,
2499                               E->path_begin(), E->path_end(),
2500                               /*NullCheckValue=*/false);
2501
2502    return MakeAddrLValue(Derived, E->getType());
2503  }
2504  case CK_LValueBitCast: {
2505    // This must be a reinterpret_cast (or c-style equivalent).
2506    const ExplicitCastExpr *CE = cast<ExplicitCastExpr>(E);
2507
2508    LValue LV = EmitLValue(E->getSubExpr());
2509    llvm::Value *V = Builder.CreateBitCast(LV.getAddress(),
2510                                           ConvertType(CE->getTypeAsWritten()));
2511    return MakeAddrLValue(V, E->getType());
2512  }
2513  case CK_ObjCObjectLValueCast: {
2514    LValue LV = EmitLValue(E->getSubExpr());
2515    QualType ToType = getContext().getLValueReferenceType(E->getType());
2516    llvm::Value *V = Builder.CreateBitCast(LV.getAddress(),
2517                                           ConvertType(ToType));
2518    return MakeAddrLValue(V, E->getType());
2519  }
2520  }
2521
2522  llvm_unreachable("Unhandled lvalue cast kind?");
2523}
2524
2525LValue CodeGenFunction::EmitNullInitializationLValue(
2526                                              const CXXScalarValueInitExpr *E) {
2527  QualType Ty = E->getType();
2528  LValue LV = MakeAddrLValue(CreateMemTemp(Ty), Ty);
2529  EmitNullInitialization(LV.getAddress(), Ty);
2530  return LV;
2531}
2532
2533LValue CodeGenFunction::EmitOpaqueValueLValue(const OpaqueValueExpr *e) {
2534  assert(OpaqueValueMappingData::shouldBindAsLValue(e));
2535  return getOpaqueLValueMapping(e);
2536}
2537
2538LValue CodeGenFunction::EmitMaterializeTemporaryExpr(
2539                                           const MaterializeTemporaryExpr *E) {
2540  RValue RV = EmitReferenceBindingToExpr(E, /*InitializedDecl=*/0);
2541  return MakeAddrLValue(RV.getScalarVal(), E->getType());
2542}
2543
2544RValue CodeGenFunction::EmitRValueForField(LValue LV,
2545                                           const FieldDecl *FD) {
2546  QualType FT = FD->getType();
2547  LValue FieldLV = EmitLValueForField(LV, FD);
2548  if (FT->isAnyComplexType())
2549    return RValue::getComplex(
2550        LoadComplexFromAddr(FieldLV.getAddress(),
2551                            FieldLV.isVolatileQualified()));
2552  else if (CodeGenFunction::hasAggregateLLVMType(FT))
2553    return FieldLV.asAggregateRValue();
2554
2555  return EmitLoadOfLValue(FieldLV);
2556}
2557
2558//===--------------------------------------------------------------------===//
2559//                             Expression Emission
2560//===--------------------------------------------------------------------===//
2561
2562RValue CodeGenFunction::EmitCallExpr(const CallExpr *E,
2563                                     ReturnValueSlot ReturnValue) {
2564  if (CGDebugInfo *DI = getDebugInfo())
2565    DI->EmitLocation(Builder, E->getLocStart());
2566
2567  // Builtins never have block type.
2568  if (E->getCallee()->getType()->isBlockPointerType())
2569    return EmitBlockCallExpr(E, ReturnValue);
2570
2571  if (const CXXMemberCallExpr *CE = dyn_cast<CXXMemberCallExpr>(E))
2572    return EmitCXXMemberCallExpr(CE, ReturnValue);
2573
2574  if (const CUDAKernelCallExpr *CE = dyn_cast<CUDAKernelCallExpr>(E))
2575    return EmitCUDAKernelCallExpr(CE, ReturnValue);
2576
2577  const Decl *TargetDecl = E->getCalleeDecl();
2578  if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(TargetDecl)) {
2579    if (unsigned builtinID = FD->getBuiltinID())
2580      return EmitBuiltinExpr(FD, builtinID, E);
2581  }
2582
2583  if (const CXXOperatorCallExpr *CE = dyn_cast<CXXOperatorCallExpr>(E))
2584    if (const CXXMethodDecl *MD = dyn_cast_or_null<CXXMethodDecl>(TargetDecl))
2585      return EmitCXXOperatorMemberCallExpr(CE, MD, ReturnValue);
2586
2587  if (const CXXPseudoDestructorExpr *PseudoDtor
2588          = dyn_cast<CXXPseudoDestructorExpr>(E->getCallee()->IgnoreParens())) {
2589    QualType DestroyedType = PseudoDtor->getDestroyedType();
2590    if (getContext().getLangOpts().ObjCAutoRefCount &&
2591        DestroyedType->isObjCLifetimeType() &&
2592        (DestroyedType.getObjCLifetime() == Qualifiers::OCL_Strong ||
2593         DestroyedType.getObjCLifetime() == Qualifiers::OCL_Weak)) {
2594      // Automatic Reference Counting:
2595      //   If the pseudo-expression names a retainable object with weak or
2596      //   strong lifetime, the object shall be released.
2597      Expr *BaseExpr = PseudoDtor->getBase();
2598      llvm::Value *BaseValue = NULL;
2599      Qualifiers BaseQuals;
2600
2601      // If this is s.x, emit s as an lvalue. If it is s->x, emit s as a scalar.
2602      if (PseudoDtor->isArrow()) {
2603        BaseValue = EmitScalarExpr(BaseExpr);
2604        const PointerType *PTy = BaseExpr->getType()->getAs<PointerType>();
2605        BaseQuals = PTy->getPointeeType().getQualifiers();
2606      } else {
2607        LValue BaseLV = EmitLValue(BaseExpr);
2608        BaseValue = BaseLV.getAddress();
2609        QualType BaseTy = BaseExpr->getType();
2610        BaseQuals = BaseTy.getQualifiers();
2611      }
2612
2613      switch (PseudoDtor->getDestroyedType().getObjCLifetime()) {
2614      case Qualifiers::OCL_None:
2615      case Qualifiers::OCL_ExplicitNone:
2616      case Qualifiers::OCL_Autoreleasing:
2617        break;
2618
2619      case Qualifiers::OCL_Strong:
2620        EmitARCRelease(Builder.CreateLoad(BaseValue,
2621                          PseudoDtor->getDestroyedType().isVolatileQualified()),
2622                       /*precise*/ true);
2623        break;
2624
2625      case Qualifiers::OCL_Weak:
2626        EmitARCDestroyWeak(BaseValue);
2627        break;
2628      }
2629    } else {
2630      // C++ [expr.pseudo]p1:
2631      //   The result shall only be used as the operand for the function call
2632      //   operator (), and the result of such a call has type void. The only
2633      //   effect is the evaluation of the postfix-expression before the dot or
2634      //   arrow.
2635      EmitScalarExpr(E->getCallee());
2636    }
2637
2638    return RValue::get(0);
2639  }
2640
2641  llvm::Value *Callee = EmitScalarExpr(E->getCallee());
2642  return EmitCall(E->getCallee()->getType(), Callee, ReturnValue,
2643                  E->arg_begin(), E->arg_end(), TargetDecl);
2644}
2645
2646LValue CodeGenFunction::EmitBinaryOperatorLValue(const BinaryOperator *E) {
2647  // Comma expressions just emit their LHS then their RHS as an l-value.
2648  if (E->getOpcode() == BO_Comma) {
2649    EmitIgnoredExpr(E->getLHS());
2650    EnsureInsertPoint();
2651    return EmitLValue(E->getRHS());
2652  }
2653
2654  if (E->getOpcode() == BO_PtrMemD ||
2655      E->getOpcode() == BO_PtrMemI)
2656    return EmitPointerToDataMemberBinaryExpr(E);
2657
2658  assert(E->getOpcode() == BO_Assign && "unexpected binary l-value");
2659
2660  // Note that in all of these cases, __block variables need the RHS
2661  // evaluated first just in case the variable gets moved by the RHS.
2662
2663  if (!hasAggregateLLVMType(E->getType())) {
2664    switch (E->getLHS()->getType().getObjCLifetime()) {
2665    case Qualifiers::OCL_Strong:
2666      return EmitARCStoreStrong(E, /*ignored*/ false).first;
2667
2668    case Qualifiers::OCL_Autoreleasing:
2669      return EmitARCStoreAutoreleasing(E).first;
2670
2671    // No reason to do any of these differently.
2672    case Qualifiers::OCL_None:
2673    case Qualifiers::OCL_ExplicitNone:
2674    case Qualifiers::OCL_Weak:
2675      break;
2676    }
2677
2678    RValue RV = EmitAnyExpr(E->getRHS());
2679    LValue LV = EmitLValue(E->getLHS());
2680    EmitStoreThroughLValue(RV, LV);
2681    return LV;
2682  }
2683
2684  if (E->getType()->isAnyComplexType())
2685    return EmitComplexAssignmentLValue(E);
2686
2687  return EmitAggExprToLValue(E);
2688}
2689
2690LValue CodeGenFunction::EmitCallExprLValue(const CallExpr *E) {
2691  RValue RV = EmitCallExpr(E);
2692
2693  if (!RV.isScalar())
2694    return MakeAddrLValue(RV.getAggregateAddr(), E->getType());
2695
2696  assert(E->getCallReturnType()->isReferenceType() &&
2697         "Can't have a scalar return unless the return type is a "
2698         "reference type!");
2699
2700  return MakeAddrLValue(RV.getScalarVal(), E->getType());
2701}
2702
2703LValue CodeGenFunction::EmitVAArgExprLValue(const VAArgExpr *E) {
2704  // FIXME: This shouldn't require another copy.
2705  return EmitAggExprToLValue(E);
2706}
2707
2708LValue CodeGenFunction::EmitCXXConstructLValue(const CXXConstructExpr *E) {
2709  assert(E->getType()->getAsCXXRecordDecl()->hasTrivialDestructor()
2710         && "binding l-value to type which needs a temporary");
2711  AggValueSlot Slot = CreateAggTemp(E->getType());
2712  EmitCXXConstructExpr(E, Slot);
2713  return MakeAddrLValue(Slot.getAddr(), E->getType());
2714}
2715
2716LValue
2717CodeGenFunction::EmitCXXTypeidLValue(const CXXTypeidExpr *E) {
2718  return MakeAddrLValue(EmitCXXTypeidExpr(E), E->getType());
2719}
2720
2721LValue
2722CodeGenFunction::EmitCXXBindTemporaryLValue(const CXXBindTemporaryExpr *E) {
2723  AggValueSlot Slot = CreateAggTemp(E->getType(), "temp.lvalue");
2724  Slot.setExternallyDestructed();
2725  EmitAggExpr(E->getSubExpr(), Slot);
2726  EmitCXXTemporary(E->getTemporary(), E->getType(), Slot.getAddr());
2727  return MakeAddrLValue(Slot.getAddr(), E->getType());
2728}
2729
2730LValue
2731CodeGenFunction::EmitLambdaLValue(const LambdaExpr *E) {
2732  AggValueSlot Slot = CreateAggTemp(E->getType(), "temp.lvalue");
2733  EmitLambdaExpr(E, Slot);
2734  return MakeAddrLValue(Slot.getAddr(), E->getType());
2735}
2736
2737LValue CodeGenFunction::EmitObjCMessageExprLValue(const ObjCMessageExpr *E) {
2738  RValue RV = EmitObjCMessageExpr(E);
2739
2740  if (!RV.isScalar())
2741    return MakeAddrLValue(RV.getAggregateAddr(), E->getType());
2742
2743  assert(E->getMethodDecl()->getResultType()->isReferenceType() &&
2744         "Can't have a scalar return unless the return type is a "
2745         "reference type!");
2746
2747  return MakeAddrLValue(RV.getScalarVal(), E->getType());
2748}
2749
2750LValue CodeGenFunction::EmitObjCSelectorLValue(const ObjCSelectorExpr *E) {
2751  llvm::Value *V =
2752    CGM.getObjCRuntime().GetSelector(Builder, E->getSelector(), true);
2753  return MakeAddrLValue(V, E->getType());
2754}
2755
2756llvm::Value *CodeGenFunction::EmitIvarOffset(const ObjCInterfaceDecl *Interface,
2757                                             const ObjCIvarDecl *Ivar) {
2758  return CGM.getObjCRuntime().EmitIvarOffset(*this, Interface, Ivar);
2759}
2760
2761LValue CodeGenFunction::EmitLValueForIvar(QualType ObjectTy,
2762                                          llvm::Value *BaseValue,
2763                                          const ObjCIvarDecl *Ivar,
2764                                          unsigned CVRQualifiers) {
2765  return CGM.getObjCRuntime().EmitObjCValueForIvar(*this, ObjectTy, BaseValue,
2766                                                   Ivar, CVRQualifiers);
2767}
2768
2769LValue CodeGenFunction::EmitObjCIvarRefLValue(const ObjCIvarRefExpr *E) {
2770  // FIXME: A lot of the code below could be shared with EmitMemberExpr.
2771  llvm::Value *BaseValue = 0;
2772  const Expr *BaseExpr = E->getBase();
2773  Qualifiers BaseQuals;
2774  QualType ObjectTy;
2775  if (E->isArrow()) {
2776    BaseValue = EmitScalarExpr(BaseExpr);
2777    ObjectTy = BaseExpr->getType()->getPointeeType();
2778    BaseQuals = ObjectTy.getQualifiers();
2779  } else {
2780    LValue BaseLV = EmitLValue(BaseExpr);
2781    // FIXME: this isn't right for bitfields.
2782    BaseValue = BaseLV.getAddress();
2783    ObjectTy = BaseExpr->getType();
2784    BaseQuals = ObjectTy.getQualifiers();
2785  }
2786
2787  LValue LV =
2788    EmitLValueForIvar(ObjectTy, BaseValue, E->getDecl(),
2789                      BaseQuals.getCVRQualifiers());
2790  setObjCGCLValueClass(getContext(), E, LV);
2791  return LV;
2792}
2793
2794LValue CodeGenFunction::EmitStmtExprLValue(const StmtExpr *E) {
2795  // Can only get l-value for message expression returning aggregate type
2796  RValue RV = EmitAnyExprToTemp(E);
2797  return MakeAddrLValue(RV.getAggregateAddr(), E->getType());
2798}
2799
2800RValue CodeGenFunction::EmitCall(QualType CalleeType, llvm::Value *Callee,
2801                                 ReturnValueSlot ReturnValue,
2802                                 CallExpr::const_arg_iterator ArgBeg,
2803                                 CallExpr::const_arg_iterator ArgEnd,
2804                                 const Decl *TargetDecl) {
2805  // Get the actual function type. The callee type will always be a pointer to
2806  // function type or a block pointer type.
2807  assert(CalleeType->isFunctionPointerType() &&
2808         "Call must have function pointer type!");
2809
2810  CalleeType = getContext().getCanonicalType(CalleeType);
2811
2812  const FunctionType *FnType
2813    = cast<FunctionType>(cast<PointerType>(CalleeType)->getPointeeType());
2814
2815  CallArgList Args;
2816  EmitCallArgs(Args, dyn_cast<FunctionProtoType>(FnType), ArgBeg, ArgEnd);
2817
2818  const CGFunctionInfo &FnInfo =
2819    CGM.getTypes().arrangeFreeFunctionCall(Args, FnType);
2820
2821  // C99 6.5.2.2p6:
2822  //   If the expression that denotes the called function has a type
2823  //   that does not include a prototype, [the default argument
2824  //   promotions are performed]. If the number of arguments does not
2825  //   equal the number of parameters, the behavior is undefined. If
2826  //   the function is defined with a type that includes a prototype,
2827  //   and either the prototype ends with an ellipsis (, ...) or the
2828  //   types of the arguments after promotion are not compatible with
2829  //   the types of the parameters, the behavior is undefined. If the
2830  //   function is defined with a type that does not include a
2831  //   prototype, and the types of the arguments after promotion are
2832  //   not compatible with those of the parameters after promotion,
2833  //   the behavior is undefined [except in some trivial cases].
2834  // That is, in the general case, we should assume that a call
2835  // through an unprototyped function type works like a *non-variadic*
2836  // call.  The way we make this work is to cast to the exact type
2837  // of the promoted arguments.
2838  if (isa<FunctionNoProtoType>(FnType) && !FnInfo.isVariadic()) {
2839    llvm::Type *CalleeTy = getTypes().GetFunctionType(FnInfo);
2840    CalleeTy = CalleeTy->getPointerTo();
2841    Callee = Builder.CreateBitCast(Callee, CalleeTy, "callee.knr.cast");
2842  }
2843
2844  return EmitCall(FnInfo, Callee, ReturnValue, Args, TargetDecl);
2845}
2846
2847LValue CodeGenFunction::
2848EmitPointerToDataMemberBinaryExpr(const BinaryOperator *E) {
2849  llvm::Value *BaseV;
2850  if (E->getOpcode() == BO_PtrMemI)
2851    BaseV = EmitScalarExpr(E->getLHS());
2852  else
2853    BaseV = EmitLValue(E->getLHS()).getAddress();
2854
2855  llvm::Value *OffsetV = EmitScalarExpr(E->getRHS());
2856
2857  const MemberPointerType *MPT
2858    = E->getRHS()->getType()->getAs<MemberPointerType>();
2859
2860  llvm::Value *AddV =
2861    CGM.getCXXABI().EmitMemberDataPointerAddress(*this, BaseV, OffsetV, MPT);
2862
2863  return MakeAddrLValue(AddV, MPT->getPointeeType());
2864}
2865
2866static void
2867EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *E, llvm::Value *Dest,
2868             llvm::Value *Ptr, llvm::Value *Val1, llvm::Value *Val2,
2869             uint64_t Size, unsigned Align, llvm::AtomicOrdering Order) {
2870  llvm::AtomicRMWInst::BinOp Op = llvm::AtomicRMWInst::Add;
2871  llvm::Instruction::BinaryOps PostOp = (llvm::Instruction::BinaryOps)0;
2872
2873  switch (E->getOp()) {
2874  case AtomicExpr::AO__c11_atomic_init:
2875    llvm_unreachable("Already handled!");
2876
2877  case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
2878  case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
2879  case AtomicExpr::AO__atomic_compare_exchange:
2880  case AtomicExpr::AO__atomic_compare_exchange_n: {
2881    // Note that cmpxchg only supports specifying one ordering and
2882    // doesn't support weak cmpxchg, at least at the moment.
2883    llvm::LoadInst *LoadVal1 = CGF.Builder.CreateLoad(Val1);
2884    LoadVal1->setAlignment(Align);
2885    llvm::LoadInst *LoadVal2 = CGF.Builder.CreateLoad(Val2);
2886    LoadVal2->setAlignment(Align);
2887    llvm::AtomicCmpXchgInst *CXI =
2888        CGF.Builder.CreateAtomicCmpXchg(Ptr, LoadVal1, LoadVal2, Order);
2889    CXI->setVolatile(E->isVolatile());
2890    llvm::StoreInst *StoreVal1 = CGF.Builder.CreateStore(CXI, Val1);
2891    StoreVal1->setAlignment(Align);
2892    llvm::Value *Cmp = CGF.Builder.CreateICmpEQ(CXI, LoadVal1);
2893    CGF.EmitStoreOfScalar(Cmp, CGF.MakeAddrLValue(Dest, E->getType()));
2894    return;
2895  }
2896
2897  case AtomicExpr::AO__c11_atomic_load:
2898  case AtomicExpr::AO__atomic_load_n:
2899  case AtomicExpr::AO__atomic_load: {
2900    llvm::LoadInst *Load = CGF.Builder.CreateLoad(Ptr);
2901    Load->setAtomic(Order);
2902    Load->setAlignment(Size);
2903    Load->setVolatile(E->isVolatile());
2904    llvm::StoreInst *StoreDest = CGF.Builder.CreateStore(Load, Dest);
2905    StoreDest->setAlignment(Align);
2906    return;
2907  }
2908
2909  case AtomicExpr::AO__c11_atomic_store:
2910  case AtomicExpr::AO__atomic_store:
2911  case AtomicExpr::AO__atomic_store_n: {
2912    assert(!Dest && "Store does not return a value");
2913    llvm::LoadInst *LoadVal1 = CGF.Builder.CreateLoad(Val1);
2914    LoadVal1->setAlignment(Align);
2915    llvm::StoreInst *Store = CGF.Builder.CreateStore(LoadVal1, Ptr);
2916    Store->setAtomic(Order);
2917    Store->setAlignment(Size);
2918    Store->setVolatile(E->isVolatile());
2919    return;
2920  }
2921
2922  case AtomicExpr::AO__c11_atomic_exchange:
2923  case AtomicExpr::AO__atomic_exchange_n:
2924  case AtomicExpr::AO__atomic_exchange:
2925    Op = llvm::AtomicRMWInst::Xchg;
2926    break;
2927
2928  case AtomicExpr::AO__atomic_add_fetch:
2929    PostOp = llvm::Instruction::Add;
2930    // Fall through.
2931  case AtomicExpr::AO__c11_atomic_fetch_add:
2932  case AtomicExpr::AO__atomic_fetch_add:
2933    Op = llvm::AtomicRMWInst::Add;
2934    break;
2935
2936  case AtomicExpr::AO__atomic_sub_fetch:
2937    PostOp = llvm::Instruction::Sub;
2938    // Fall through.
2939  case AtomicExpr::AO__c11_atomic_fetch_sub:
2940  case AtomicExpr::AO__atomic_fetch_sub:
2941    Op = llvm::AtomicRMWInst::Sub;
2942    break;
2943
2944  case AtomicExpr::AO__atomic_and_fetch:
2945    PostOp = llvm::Instruction::And;
2946    // Fall through.
2947  case AtomicExpr::AO__c11_atomic_fetch_and:
2948  case AtomicExpr::AO__atomic_fetch_and:
2949    Op = llvm::AtomicRMWInst::And;
2950    break;
2951
2952  case AtomicExpr::AO__atomic_or_fetch:
2953    PostOp = llvm::Instruction::Or;
2954    // Fall through.
2955  case AtomicExpr::AO__c11_atomic_fetch_or:
2956  case AtomicExpr::AO__atomic_fetch_or:
2957    Op = llvm::AtomicRMWInst::Or;
2958    break;
2959
2960  case AtomicExpr::AO__atomic_xor_fetch:
2961    PostOp = llvm::Instruction::Xor;
2962    // Fall through.
2963  case AtomicExpr::AO__c11_atomic_fetch_xor:
2964  case AtomicExpr::AO__atomic_fetch_xor:
2965    Op = llvm::AtomicRMWInst::Xor;
2966    break;
2967
2968  case AtomicExpr::AO__atomic_nand_fetch:
2969    PostOp = llvm::Instruction::And;
2970    // Fall through.
2971  case AtomicExpr::AO__atomic_fetch_nand:
2972    Op = llvm::AtomicRMWInst::Nand;
2973    break;
2974  }
2975
2976  llvm::LoadInst *LoadVal1 = CGF.Builder.CreateLoad(Val1);
2977  LoadVal1->setAlignment(Align);
2978  llvm::AtomicRMWInst *RMWI =
2979      CGF.Builder.CreateAtomicRMW(Op, Ptr, LoadVal1, Order);
2980  RMWI->setVolatile(E->isVolatile());
2981
2982  // For __atomic_*_fetch operations, perform the operation again to
2983  // determine the value which was written.
2984  llvm::Value *Result = RMWI;
2985  if (PostOp)
2986    Result = CGF.Builder.CreateBinOp(PostOp, RMWI, LoadVal1);
2987  if (E->getOp() == AtomicExpr::AO__atomic_nand_fetch)
2988    Result = CGF.Builder.CreateNot(Result);
2989  llvm::StoreInst *StoreDest = CGF.Builder.CreateStore(Result, Dest);
2990  StoreDest->setAlignment(Align);
2991}
2992
2993// This function emits any expression (scalar, complex, or aggregate)
2994// into a temporary alloca.
2995static llvm::Value *
2996EmitValToTemp(CodeGenFunction &CGF, Expr *E) {
2997  llvm::Value *DeclPtr = CGF.CreateMemTemp(E->getType(), ".atomictmp");
2998  CGF.EmitAnyExprToMem(E, DeclPtr, E->getType().getQualifiers(),
2999                       /*Init*/ true);
3000  return DeclPtr;
3001}
3002
3003static RValue ConvertTempToRValue(CodeGenFunction &CGF, QualType Ty,
3004                                  llvm::Value *Dest) {
3005  if (Ty->isAnyComplexType())
3006    return RValue::getComplex(CGF.LoadComplexFromAddr(Dest, false));
3007  if (CGF.hasAggregateLLVMType(Ty))
3008    return RValue::getAggregate(Dest);
3009  return RValue::get(CGF.EmitLoadOfScalar(CGF.MakeAddrLValue(Dest, Ty)));
3010}
3011
3012RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E, llvm::Value *Dest) {
3013  QualType AtomicTy = E->getPtr()->getType()->getPointeeType();
3014  QualType MemTy = AtomicTy;
3015  if (const AtomicType *AT = AtomicTy->getAs<AtomicType>())
3016    MemTy = AT->getValueType();
3017  CharUnits sizeChars = getContext().getTypeSizeInChars(AtomicTy);
3018  uint64_t Size = sizeChars.getQuantity();
3019  CharUnits alignChars = getContext().getTypeAlignInChars(AtomicTy);
3020  unsigned Align = alignChars.getQuantity();
3021  unsigned MaxInlineWidth =
3022      getContext().getTargetInfo().getMaxAtomicInlineWidth();
3023  bool UseLibcall = (Size != Align || Size > MaxInlineWidth);
3024
3025
3026
3027  llvm::Value *Ptr, *Order, *OrderFail = 0, *Val1 = 0, *Val2 = 0;
3028  Ptr = EmitScalarExpr(E->getPtr());
3029
3030  if (E->getOp() == AtomicExpr::AO__c11_atomic_init) {
3031    assert(!Dest && "Init does not return a value");
3032    if (!hasAggregateLLVMType(E->getVal1()->getType())) {
3033      QualType PointeeType
3034        = E->getPtr()->getType()->getAs<PointerType>()->getPointeeType();
3035      EmitScalarInit(EmitScalarExpr(E->getVal1()),
3036                     LValue::MakeAddr(Ptr, PointeeType, alignChars,
3037                                      getContext()));
3038    } else if (E->getType()->isAnyComplexType()) {
3039      EmitComplexExprIntoAddr(E->getVal1(), Ptr, E->isVolatile());
3040    } else {
3041      AggValueSlot Slot = AggValueSlot::forAddr(Ptr, alignChars,
3042                                        AtomicTy.getQualifiers(),
3043                                        AggValueSlot::IsNotDestructed,
3044                                        AggValueSlot::DoesNotNeedGCBarriers,
3045                                        AggValueSlot::IsNotAliased);
3046      EmitAggExpr(E->getVal1(), Slot);
3047    }
3048    return RValue::get(0);
3049  }
3050
3051  Order = EmitScalarExpr(E->getOrder());
3052
3053  switch (E->getOp()) {
3054  case AtomicExpr::AO__c11_atomic_init:
3055    llvm_unreachable("Already handled!");
3056
3057  case AtomicExpr::AO__c11_atomic_load:
3058  case AtomicExpr::AO__atomic_load_n:
3059    break;
3060
3061  case AtomicExpr::AO__atomic_load:
3062    Dest = EmitScalarExpr(E->getVal1());
3063    break;
3064
3065  case AtomicExpr::AO__atomic_store:
3066    Val1 = EmitScalarExpr(E->getVal1());
3067    break;
3068
3069  case AtomicExpr::AO__atomic_exchange:
3070    Val1 = EmitScalarExpr(E->getVal1());
3071    Dest = EmitScalarExpr(E->getVal2());
3072    break;
3073
3074  case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
3075  case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
3076  case AtomicExpr::AO__atomic_compare_exchange_n:
3077  case AtomicExpr::AO__atomic_compare_exchange:
3078    Val1 = EmitScalarExpr(E->getVal1());
3079    if (E->getOp() == AtomicExpr::AO__atomic_compare_exchange)
3080      Val2 = EmitScalarExpr(E->getVal2());
3081    else
3082      Val2 = EmitValToTemp(*this, E->getVal2());
3083    OrderFail = EmitScalarExpr(E->getOrderFail());
3084    // Evaluate and discard the 'weak' argument.
3085    if (E->getNumSubExprs() == 6)
3086      EmitScalarExpr(E->getWeak());
3087    break;
3088
3089  case AtomicExpr::AO__c11_atomic_fetch_add:
3090  case AtomicExpr::AO__c11_atomic_fetch_sub:
3091    if (MemTy->isPointerType()) {
3092      // For pointer arithmetic, we're required to do a bit of math:
3093      // adding 1 to an int* is not the same as adding 1 to a uintptr_t.
3094      // ... but only for the C11 builtins. The GNU builtins expect the
3095      // user to multiply by sizeof(T).
3096      QualType Val1Ty = E->getVal1()->getType();
3097      llvm::Value *Val1Scalar = EmitScalarExpr(E->getVal1());
3098      CharUnits PointeeIncAmt =
3099          getContext().getTypeSizeInChars(MemTy->getPointeeType());
3100      Val1Scalar = Builder.CreateMul(Val1Scalar, CGM.getSize(PointeeIncAmt));
3101      Val1 = CreateMemTemp(Val1Ty, ".atomictmp");
3102      EmitStoreOfScalar(Val1Scalar, MakeAddrLValue(Val1, Val1Ty));
3103      break;
3104    }
3105    // Fall through.
3106  case AtomicExpr::AO__atomic_fetch_add:
3107  case AtomicExpr::AO__atomic_fetch_sub:
3108  case AtomicExpr::AO__atomic_add_fetch:
3109  case AtomicExpr::AO__atomic_sub_fetch:
3110  case AtomicExpr::AO__c11_atomic_store:
3111  case AtomicExpr::AO__c11_atomic_exchange:
3112  case AtomicExpr::AO__atomic_store_n:
3113  case AtomicExpr::AO__atomic_exchange_n:
3114  case AtomicExpr::AO__c11_atomic_fetch_and:
3115  case AtomicExpr::AO__c11_atomic_fetch_or:
3116  case AtomicExpr::AO__c11_atomic_fetch_xor:
3117  case AtomicExpr::AO__atomic_fetch_and:
3118  case AtomicExpr::AO__atomic_fetch_or:
3119  case AtomicExpr::AO__atomic_fetch_xor:
3120  case AtomicExpr::AO__atomic_fetch_nand:
3121  case AtomicExpr::AO__atomic_and_fetch:
3122  case AtomicExpr::AO__atomic_or_fetch:
3123  case AtomicExpr::AO__atomic_xor_fetch:
3124  case AtomicExpr::AO__atomic_nand_fetch:
3125    Val1 = EmitValToTemp(*this, E->getVal1());
3126    break;
3127  }
3128
3129  if (!E->getType()->isVoidType() && !Dest)
3130    Dest = CreateMemTemp(E->getType(), ".atomicdst");
3131
3132  // Use a library call.  See: http://gcc.gnu.org/wiki/Atomic/GCCMM/LIbrary .
3133  if (UseLibcall) {
3134
3135    llvm::SmallVector<QualType, 5> Params;
3136    CallArgList Args;
3137    // Size is always the first parameter
3138    Args.add(RValue::get(llvm::ConstantInt::get(SizeTy, Size)),
3139             getContext().getSizeType());
3140    // Atomic address is always the second parameter
3141    Args.add(RValue::get(EmitCastToVoidPtr(Ptr)),
3142             getContext().VoidPtrTy);
3143
3144    const char* LibCallName;
3145    QualType RetTy = getContext().VoidTy;
3146    switch (E->getOp()) {
3147    // There is only one libcall for compare an exchange, because there is no
3148    // optimisation benefit possible from a libcall version of a weak compare
3149    // and exchange.
3150    // bool __atomic_compare_exchange(size_t size, void *obj, void *expected,
3151    //                                void *desired, int success, int failure)
3152    case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
3153    case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
3154    case AtomicExpr::AO__atomic_compare_exchange:
3155    case AtomicExpr::AO__atomic_compare_exchange_n:
3156      LibCallName = "__atomic_compare_exchange";
3157      RetTy = getContext().BoolTy;
3158      Args.add(RValue::get(EmitCastToVoidPtr(Val1)),
3159               getContext().VoidPtrTy);
3160      Args.add(RValue::get(EmitCastToVoidPtr(Val2)),
3161               getContext().VoidPtrTy);
3162      Args.add(RValue::get(Order),
3163               getContext().IntTy);
3164      Order = OrderFail;
3165      break;
3166    // void __atomic_exchange(size_t size, void *mem, void *val, void *return,
3167    //                        int order)
3168    case AtomicExpr::AO__c11_atomic_exchange:
3169    case AtomicExpr::AO__atomic_exchange_n:
3170    case AtomicExpr::AO__atomic_exchange:
3171      LibCallName = "__atomic_exchange";
3172      Args.add(RValue::get(EmitCastToVoidPtr(Val1)),
3173               getContext().VoidPtrTy);
3174      Args.add(RValue::get(EmitCastToVoidPtr(Dest)),
3175               getContext().VoidPtrTy);
3176      break;
3177    // void __atomic_store(size_t size, void *mem, void *val, int order)
3178    case AtomicExpr::AO__c11_atomic_store:
3179    case AtomicExpr::AO__atomic_store:
3180    case AtomicExpr::AO__atomic_store_n:
3181      LibCallName = "__atomic_store";
3182      Args.add(RValue::get(EmitCastToVoidPtr(Val1)),
3183               getContext().VoidPtrTy);
3184      break;
3185    // void __atomic_load(size_t size, void *mem, void *return, int order)
3186    case AtomicExpr::AO__c11_atomic_load:
3187    case AtomicExpr::AO__atomic_load:
3188    case AtomicExpr::AO__atomic_load_n:
3189      LibCallName = "__atomic_load";
3190      Args.add(RValue::get(EmitCastToVoidPtr(Dest)),
3191               getContext().VoidPtrTy);
3192      break;
3193#if 0
3194    // These are only defined for 1-16 byte integers.  It is not clear what
3195    // their semantics would be on anything else...
3196    case AtomicExpr::Add:   LibCallName = "__atomic_fetch_add_generic"; break;
3197    case AtomicExpr::Sub:   LibCallName = "__atomic_fetch_sub_generic"; break;
3198    case AtomicExpr::And:   LibCallName = "__atomic_fetch_and_generic"; break;
3199    case AtomicExpr::Or:    LibCallName = "__atomic_fetch_or_generic"; break;
3200    case AtomicExpr::Xor:   LibCallName = "__atomic_fetch_xor_generic"; break;
3201#endif
3202    default: return EmitUnsupportedRValue(E, "atomic library call");
3203    }
3204    // order is always the last parameter
3205    Args.add(RValue::get(Order),
3206             getContext().IntTy);
3207
3208    const CGFunctionInfo &FuncInfo =
3209        CGM.getTypes().arrangeFreeFunctionCall(RetTy, Args,
3210            FunctionType::ExtInfo(), RequiredArgs::All);
3211    llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(FuncInfo);
3212    llvm::Constant *Func = CGM.CreateRuntimeFunction(FTy, LibCallName);
3213    RValue Res = EmitCall(FuncInfo, Func, ReturnValueSlot(), Args);
3214    if (E->isCmpXChg())
3215      return Res;
3216    if (E->getType()->isVoidType())
3217      return RValue::get(0);
3218    return ConvertTempToRValue(*this, E->getType(), Dest);
3219  }
3220
3221  llvm::Type *IPtrTy =
3222      llvm::IntegerType::get(getLLVMContext(), Size * 8)->getPointerTo();
3223  llvm::Value *OrigDest = Dest;
3224  Ptr = Builder.CreateBitCast(Ptr, IPtrTy);
3225  if (Val1) Val1 = Builder.CreateBitCast(Val1, IPtrTy);
3226  if (Val2) Val2 = Builder.CreateBitCast(Val2, IPtrTy);
3227  if (Dest && !E->isCmpXChg()) Dest = Builder.CreateBitCast(Dest, IPtrTy);
3228
3229  if (isa<llvm::ConstantInt>(Order)) {
3230    int ord = cast<llvm::ConstantInt>(Order)->getZExtValue();
3231    switch (ord) {
3232    case 0:  // memory_order_relaxed
3233      EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, Size, Align,
3234                   llvm::Monotonic);
3235      break;
3236    case 1:  // memory_order_consume
3237    case 2:  // memory_order_acquire
3238      EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, Size, Align,
3239                   llvm::Acquire);
3240      break;
3241    case 3:  // memory_order_release
3242      EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, Size, Align,
3243                   llvm::Release);
3244      break;
3245    case 4:  // memory_order_acq_rel
3246      EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, Size, Align,
3247                   llvm::AcquireRelease);
3248      break;
3249    case 5:  // memory_order_seq_cst
3250      EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, Size, Align,
3251                   llvm::SequentiallyConsistent);
3252      break;
3253    default: // invalid order
3254      // We should not ever get here normally, but it's hard to
3255      // enforce that in general.
3256      break;
3257    }
3258    if (E->getType()->isVoidType())
3259      return RValue::get(0);
3260    return ConvertTempToRValue(*this, E->getType(), OrigDest);
3261  }
3262
3263  // Long case, when Order isn't obviously constant.
3264
3265  bool IsStore = E->getOp() == AtomicExpr::AO__c11_atomic_store ||
3266                 E->getOp() == AtomicExpr::AO__atomic_store ||
3267                 E->getOp() == AtomicExpr::AO__atomic_store_n;
3268  bool IsLoad = E->getOp() == AtomicExpr::AO__c11_atomic_load ||
3269                E->getOp() == AtomicExpr::AO__atomic_load ||
3270                E->getOp() == AtomicExpr::AO__atomic_load_n;
3271
3272  // Create all the relevant BB's
3273  llvm::BasicBlock *MonotonicBB = 0, *AcquireBB = 0, *ReleaseBB = 0,
3274                   *AcqRelBB = 0, *SeqCstBB = 0;
3275  MonotonicBB = createBasicBlock("monotonic", CurFn);
3276  if (!IsStore)
3277    AcquireBB = createBasicBlock("acquire", CurFn);
3278  if (!IsLoad)
3279    ReleaseBB = createBasicBlock("release", CurFn);
3280  if (!IsLoad && !IsStore)
3281    AcqRelBB = createBasicBlock("acqrel", CurFn);
3282  SeqCstBB = createBasicBlock("seqcst", CurFn);
3283  llvm::BasicBlock *ContBB = createBasicBlock("atomic.continue", CurFn);
3284
3285  // Create the switch for the split
3286  // MonotonicBB is arbitrarily chosen as the default case; in practice, this
3287  // doesn't matter unless someone is crazy enough to use something that
3288  // doesn't fold to a constant for the ordering.
3289  Order = Builder.CreateIntCast(Order, Builder.getInt32Ty(), false);
3290  llvm::SwitchInst *SI = Builder.CreateSwitch(Order, MonotonicBB);
3291
3292  // Emit all the different atomics
3293  Builder.SetInsertPoint(MonotonicBB);
3294  EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, Size, Align,
3295               llvm::Monotonic);
3296  Builder.CreateBr(ContBB);
3297  if (!IsStore) {
3298    Builder.SetInsertPoint(AcquireBB);
3299    EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, Size, Align,
3300                 llvm::Acquire);
3301    Builder.CreateBr(ContBB);
3302    SI->addCase(Builder.getInt32(1), AcquireBB);
3303    SI->addCase(Builder.getInt32(2), AcquireBB);
3304  }
3305  if (!IsLoad) {
3306    Builder.SetInsertPoint(ReleaseBB);
3307    EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, Size, Align,
3308                 llvm::Release);
3309    Builder.CreateBr(ContBB);
3310    SI->addCase(Builder.getInt32(3), ReleaseBB);
3311  }
3312  if (!IsLoad && !IsStore) {
3313    Builder.SetInsertPoint(AcqRelBB);
3314    EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, Size, Align,
3315                 llvm::AcquireRelease);
3316    Builder.CreateBr(ContBB);
3317    SI->addCase(Builder.getInt32(4), AcqRelBB);
3318  }
3319  Builder.SetInsertPoint(SeqCstBB);
3320  EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, Size, Align,
3321               llvm::SequentiallyConsistent);
3322  Builder.CreateBr(ContBB);
3323  SI->addCase(Builder.getInt32(5), SeqCstBB);
3324
3325  // Cleanup and return
3326  Builder.SetInsertPoint(ContBB);
3327  if (E->getType()->isVoidType())
3328    return RValue::get(0);
3329  return ConvertTempToRValue(*this, E->getType(), OrigDest);
3330}
3331
3332void CodeGenFunction::SetFPAccuracy(llvm::Value *Val, float Accuracy) {
3333  assert(Val->getType()->isFPOrFPVectorTy());
3334  if (Accuracy == 0.0 || !isa<llvm::Instruction>(Val))
3335    return;
3336
3337  llvm::MDBuilder MDHelper(getLLVMContext());
3338  llvm::MDNode *Node = MDHelper.createFPMath(Accuracy);
3339
3340  cast<llvm::Instruction>(Val)->setMetadata(llvm::LLVMContext::MD_fpmath, Node);
3341}
3342
3343namespace {
3344  struct LValueOrRValue {
3345    LValue LV;
3346    RValue RV;
3347  };
3348}
3349
3350static LValueOrRValue emitPseudoObjectExpr(CodeGenFunction &CGF,
3351                                           const PseudoObjectExpr *E,
3352                                           bool forLValue,
3353                                           AggValueSlot slot) {
3354  llvm::SmallVector<CodeGenFunction::OpaqueValueMappingData, 4> opaques;
3355
3356  // Find the result expression, if any.
3357  const Expr *resultExpr = E->getResultExpr();
3358  LValueOrRValue result;
3359
3360  for (PseudoObjectExpr::const_semantics_iterator
3361         i = E->semantics_begin(), e = E->semantics_end(); i != e; ++i) {
3362    const Expr *semantic = *i;
3363
3364    // If this semantic expression is an opaque value, bind it
3365    // to the result of its source expression.
3366    if (const OpaqueValueExpr *ov = dyn_cast<OpaqueValueExpr>(semantic)) {
3367
3368      // If this is the result expression, we may need to evaluate
3369      // directly into the slot.
3370      typedef CodeGenFunction::OpaqueValueMappingData OVMA;
3371      OVMA opaqueData;
3372      if (ov == resultExpr && ov->isRValue() && !forLValue &&
3373          CodeGenFunction::hasAggregateLLVMType(ov->getType()) &&
3374          !ov->getType()->isAnyComplexType()) {
3375        CGF.EmitAggExpr(ov->getSourceExpr(), slot);
3376
3377        LValue LV = CGF.MakeAddrLValue(slot.getAddr(), ov->getType());
3378        opaqueData = OVMA::bind(CGF, ov, LV);
3379        result.RV = slot.asRValue();
3380
3381      // Otherwise, emit as normal.
3382      } else {
3383        opaqueData = OVMA::bind(CGF, ov, ov->getSourceExpr());
3384
3385        // If this is the result, also evaluate the result now.
3386        if (ov == resultExpr) {
3387          if (forLValue)
3388            result.LV = CGF.EmitLValue(ov);
3389          else
3390            result.RV = CGF.EmitAnyExpr(ov, slot);
3391        }
3392      }
3393
3394      opaques.push_back(opaqueData);
3395
3396    // Otherwise, if the expression is the result, evaluate it
3397    // and remember the result.
3398    } else if (semantic == resultExpr) {
3399      if (forLValue)
3400        result.LV = CGF.EmitLValue(semantic);
3401      else
3402        result.RV = CGF.EmitAnyExpr(semantic, slot);
3403
3404    // Otherwise, evaluate the expression in an ignored context.
3405    } else {
3406      CGF.EmitIgnoredExpr(semantic);
3407    }
3408  }
3409
3410  // Unbind all the opaques now.
3411  for (unsigned i = 0, e = opaques.size(); i != e; ++i)
3412    opaques[i].unbind(CGF);
3413
3414  return result;
3415}
3416
3417RValue CodeGenFunction::EmitPseudoObjectRValue(const PseudoObjectExpr *E,
3418                                               AggValueSlot slot) {
3419  return emitPseudoObjectExpr(*this, E, false, slot).RV;
3420}
3421
3422LValue CodeGenFunction::EmitPseudoObjectLValue(const PseudoObjectExpr *E) {
3423  return emitPseudoObjectExpr(*this, E, true, AggValueSlot::ignored()).LV;
3424}
3425