CGExpr.cpp revision cb5620c9b213f4bd323912159fdddda35e258a14 
1//===--- CGExpr.cpp - Emit LLVM Code from Expressions ---------------------===//
2//
3//                     The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10// This contains code to emit Expr nodes as LLVM code.
11//
12//===----------------------------------------------------------------------===//
13
14#include "CodeGenFunction.h"
15#include "CGCXXABI.h"
16#include "CGCall.h"
17#include "CGDebugInfo.h"
18#include "CGObjCRuntime.h"
19#include "CGRecordLayout.h"
20#include "CodeGenModule.h"
21#include "TargetInfo.h"
22#include "clang/AST/ASTContext.h"
23#include "clang/AST/DeclObjC.h"
24#include "clang/Frontend/CodeGenOptions.h"
25#include "llvm/ADT/Hashing.h"
26#include "llvm/IR/DataLayout.h"
27#include "llvm/IR/Intrinsics.h"
28#include "llvm/IR/LLVMContext.h"
29#include "llvm/IR/MDBuilder.h"
30#include "llvm/Support/ConvertUTF.h"
31
32using namespace clang;
33using namespace CodeGen;
34
35//===--------------------------------------------------------------------===//
36//                        Miscellaneous Helper Methods
37//===--------------------------------------------------------------------===//
38
39llvm::Value *CodeGenFunction::EmitCastToVoidPtr(llvm::Value *value) {
40  unsigned addressSpace =
41    cast<llvm::PointerType>(value->getType())->getAddressSpace();
42
43  llvm::PointerType *destType = Int8PtrTy;
44  if (addressSpace)
45    destType = llvm::Type::getInt8PtrTy(getLLVMContext(), addressSpace);
46
47  if (value->getType() == destType) return value;
48  return Builder.CreateBitCast(value, destType);
49}
50
51/// CreateTempAlloca - This creates a alloca and inserts it into the entry
52/// block.
53llvm::AllocaInst *CodeGenFunction::CreateTempAlloca(llvm::Type *Ty,
54                                                    const Twine &Name) {
55  if (!Builder.isNamePreserving())
56    return new llvm::AllocaInst(Ty, 0, "", AllocaInsertPt);
57  return new llvm::AllocaInst(Ty, 0, Name, AllocaInsertPt);
58}
59
60void CodeGenFunction::InitTempAlloca(llvm::AllocaInst *Var,
61                                     llvm::Value *Init) {
62  llvm::StoreInst *Store = new llvm::StoreInst(Init, Var);
63  llvm::BasicBlock *Block = AllocaInsertPt->getParent();
64  Block->getInstList().insertAfter(&*AllocaInsertPt, Store);
65}
66
67llvm::AllocaInst *CodeGenFunction::CreateIRTemp(QualType Ty,
68                                                const Twine &Name) {
69  llvm::AllocaInst *Alloc = CreateTempAlloca(ConvertType(Ty), Name);
70  // FIXME: Should we prefer the preferred type alignment here?
71  CharUnits Align = getContext().getTypeAlignInChars(Ty);
72  Alloc->setAlignment(Align.getQuantity());
73  return Alloc;
74}
75
76llvm::AllocaInst *CodeGenFunction::CreateMemTemp(QualType Ty,
77                                                 const Twine &Name) {
78  llvm::AllocaInst *Alloc = CreateTempAlloca(ConvertTypeForMem(Ty), Name);
79  // FIXME: Should we prefer the preferred type alignment here?
80  CharUnits Align = getContext().getTypeAlignInChars(Ty);
81  Alloc->setAlignment(Align.getQuantity());
82  return Alloc;
83}
84
85/// EvaluateExprAsBool - Perform the usual unary conversions on the specified
86/// expression and compare the result against zero, returning an Int1Ty value.
87llvm::Value *CodeGenFunction::EvaluateExprAsBool(const Expr *E) {
88  if (const MemberPointerType *MPT = E->getType()->getAs<MemberPointerType>()) {
89    llvm::Value *MemPtr = EmitScalarExpr(E);
90    return CGM.getCXXABI().EmitMemberPointerIsNotNull(*this, MemPtr, MPT);
91  }
92
93  QualType BoolTy = getContext().BoolTy;
94  if (!E->getType()->isAnyComplexType())
95    return EmitScalarConversion(EmitScalarExpr(E), E->getType(), BoolTy);
96
97  return EmitComplexToScalarConversion(EmitComplexExpr(E), E->getType(),BoolTy);
98}
99
100/// EmitIgnoredExpr - Emit code to compute the specified expression,
101/// ignoring the result.
102void CodeGenFunction::EmitIgnoredExpr(const Expr *E) {
103  if (E->isRValue())
104    return (void) EmitAnyExpr(E, AggValueSlot::ignored(), true);
105
106  // Just emit it as an l-value and drop the result.
107  EmitLValue(E);
108}
109
110/// EmitAnyExpr - Emit code to compute the specified expression which
111/// can have any type.  The result is returned as an RValue struct.
112/// If this is an aggregate expression, AggSlot indicates where the
113/// result should be returned.
114RValue CodeGenFunction::EmitAnyExpr(const Expr *E,
115                                    AggValueSlot aggSlot,
116                                    bool ignoreResult) {
117  if (!hasAggregateLLVMType(E->getType()))
118    return RValue::get(EmitScalarExpr(E, ignoreResult));
119  else if (E->getType()->isAnyComplexType())
120    return RValue::getComplex(EmitComplexExpr(E, ignoreResult, ignoreResult));
121
122  if (!ignoreResult && aggSlot.isIgnored())
123    aggSlot = CreateAggTemp(E->getType(), "agg-temp");
124  EmitAggExpr(E, aggSlot);
125  return aggSlot.asRValue();
126}
127
128/// EmitAnyExprToTemp - Similary to EmitAnyExpr(), however, the result will
129/// always be accessible even if no aggregate location is provided.
130RValue CodeGenFunction::EmitAnyExprToTemp(const Expr *E) {
131  AggValueSlot AggSlot = AggValueSlot::ignored();
132
133  if (hasAggregateLLVMType(E->getType()) &&
134      !E->getType()->isAnyComplexType())
135    AggSlot = CreateAggTemp(E->getType(), "agg.tmp");
136  return EmitAnyExpr(E, AggSlot);
137}
138
139/// EmitAnyExprToMem - Evaluate an expression into a given memory
140/// location.
141void CodeGenFunction::EmitAnyExprToMem(const Expr *E,
142                                       llvm::Value *Location,
143                                       Qualifiers Quals,
144                                       bool IsInit) {
145  // FIXME: This function should take an LValue as an argument.
146  if (E->getType()->isAnyComplexType()) {
147    EmitComplexExprIntoAddr(E, Location, Quals.hasVolatile());
148  } else if (hasAggregateLLVMType(E->getType())) {
149    CharUnits Alignment = getContext().getTypeAlignInChars(E->getType());
150    EmitAggExpr(E, AggValueSlot::forAddr(Location, Alignment, Quals,
151                                         AggValueSlot::IsDestructed_t(IsInit),
152                                         AggValueSlot::DoesNotNeedGCBarriers,
153                                         AggValueSlot::IsAliased_t(!IsInit)));
154  } else {
155    RValue RV = RValue::get(EmitScalarExpr(E, /*Ignore*/ false));
156    LValue LV = MakeAddrLValue(Location, E->getType());
157    EmitStoreThroughLValue(RV, LV);
158  }
159}
160
161static llvm::Value *
162CreateReferenceTemporary(CodeGenFunction &CGF, QualType Type,
163                         const NamedDecl *InitializedDecl) {
164  if (const VarDecl *VD = dyn_cast_or_null<VarDecl>(InitializedDecl)) {
165    if (VD->hasGlobalStorage()) {
166      SmallString<256> Name;
167      llvm::raw_svector_ostream Out(Name);
168      CGF.CGM.getCXXABI().getMangleContext().mangleReferenceTemporary(VD, Out);
169      Out.flush();
170
171      llvm::Type *RefTempTy = CGF.ConvertTypeForMem(Type);
172
173      // Create the reference temporary.
174      llvm::GlobalValue *RefTemp =
175        new llvm::GlobalVariable(CGF.CGM.getModule(),
176                                 RefTempTy, /*isConstant=*/false,
177                                 llvm::GlobalValue::InternalLinkage,
178                                 llvm::Constant::getNullValue(RefTempTy),
179                                 Name.str());
180      return RefTemp;
181    }
182  }
183
184  return CGF.CreateMemTemp(Type, "ref.tmp");
185}
186
187static llvm::Value *
188EmitExprForReferenceBinding(CodeGenFunction &CGF, const Expr *E,
189                            llvm::Value *&ReferenceTemporary,
190                            const CXXDestructorDecl *&ReferenceTemporaryDtor,
191                            QualType &ObjCARCReferenceLifetimeType,
192                            const NamedDecl *InitializedDecl) {
193  const MaterializeTemporaryExpr *M = NULL;
194  E = E->findMaterializedTemporary(M);
195  // Objective-C++ ARC:
196  //   If we are binding a reference to a temporary that has ownership, we
197  //   need to perform retain/release operations on the temporary.
198  if (M && CGF.getLangOpts().ObjCAutoRefCount &&
199      M->getType()->isObjCLifetimeType() &&
200      (M->getType().getObjCLifetime() == Qualifiers::OCL_Strong ||
201       M->getType().getObjCLifetime() == Qualifiers::OCL_Weak ||
202       M->getType().getObjCLifetime() == Qualifiers::OCL_Autoreleasing))
203    ObjCARCReferenceLifetimeType = M->getType();
204
205  if (const ExprWithCleanups *EWC = dyn_cast<ExprWithCleanups>(E)) {
206    CGF.enterFullExpression(EWC);
207    CodeGenFunction::RunCleanupsScope Scope(CGF);
208
209    return EmitExprForReferenceBinding(CGF, EWC->getSubExpr(),
210                                       ReferenceTemporary,
211                                       ReferenceTemporaryDtor,
212                                       ObjCARCReferenceLifetimeType,
213                                       InitializedDecl);
214  }
215
216  RValue RV;
217  if (E->isGLValue()) {
218    // Emit the expression as an lvalue.
219    LValue LV = CGF.EmitLValue(E);
220
221    if (LV.isSimple())
222      return LV.getAddress();
223
224    // We have to load the lvalue.
225    RV = CGF.EmitLoadOfLValue(LV);
226  } else {
227    if (!ObjCARCReferenceLifetimeType.isNull()) {
228      ReferenceTemporary = CreateReferenceTemporary(CGF,
229                                                  ObjCARCReferenceLifetimeType,
230                                                    InitializedDecl);
231
232
233      LValue RefTempDst = CGF.MakeAddrLValue(ReferenceTemporary,
234                                             ObjCARCReferenceLifetimeType);
235
236      CGF.EmitScalarInit(E, dyn_cast_or_null<ValueDecl>(InitializedDecl),
237                         RefTempDst, false);
238
239      bool ExtendsLifeOfTemporary = false;
240      if (const VarDecl *Var = dyn_cast_or_null<VarDecl>(InitializedDecl)) {
241        if (Var->extendsLifetimeOfTemporary())
242          ExtendsLifeOfTemporary = true;
243      } else if (InitializedDecl && isa<FieldDecl>(InitializedDecl)) {
244        ExtendsLifeOfTemporary = true;
245      }
246
247      if (!ExtendsLifeOfTemporary) {
248        // Since the lifetime of this temporary isn't going to be extended,
249        // we need to clean it up ourselves at the end of the full expression.
250        switch (ObjCARCReferenceLifetimeType.getObjCLifetime()) {
251        case Qualifiers::OCL_None:
252        case Qualifiers::OCL_ExplicitNone:
253        case Qualifiers::OCL_Autoreleasing:
254          break;
255
256        case Qualifiers::OCL_Strong: {
257          assert(!ObjCARCReferenceLifetimeType->isArrayType());
258          CleanupKind cleanupKind = CGF.getARCCleanupKind();
259          CGF.pushDestroy(cleanupKind,
260                          ReferenceTemporary,
261                          ObjCARCReferenceLifetimeType,
262                          CodeGenFunction::destroyARCStrongImprecise,
263                          cleanupKind & EHCleanup);
264          break;
265        }
266
267        case Qualifiers::OCL_Weak:
268          assert(!ObjCARCReferenceLifetimeType->isArrayType());
269          CGF.pushDestroy(NormalAndEHCleanup,
270                          ReferenceTemporary,
271                          ObjCARCReferenceLifetimeType,
272                          CodeGenFunction::destroyARCWeak,
273                          /*useEHCleanupForArray*/ true);
274          break;
275        }
276
277        ObjCARCReferenceLifetimeType = QualType();
278      }
279
280      return ReferenceTemporary;
281    }
282
283    SmallVector<SubobjectAdjustment, 2> Adjustments;
284    E = E->skipRValueSubobjectAdjustments(Adjustments);
285    if (const OpaqueValueExpr *opaque = dyn_cast<OpaqueValueExpr>(E))
286      if (opaque->getType()->isRecordType())
287        return CGF.EmitOpaqueValueLValue(opaque).getAddress();
288
289    // Create a reference temporary if necessary.
290    AggValueSlot AggSlot = AggValueSlot::ignored();
291    if (CGF.hasAggregateLLVMType(E->getType()) &&
292        !E->getType()->isAnyComplexType()) {
293      ReferenceTemporary = CreateReferenceTemporary(CGF, E->getType(),
294                                                    InitializedDecl);
295      CharUnits Alignment = CGF.getContext().getTypeAlignInChars(E->getType());
296      AggValueSlot::IsDestructed_t isDestructed
297        = AggValueSlot::IsDestructed_t(InitializedDecl != 0);
298      AggSlot = AggValueSlot::forAddr(ReferenceTemporary, Alignment,
299                                      Qualifiers(), isDestructed,
300                                      AggValueSlot::DoesNotNeedGCBarriers,
301                                      AggValueSlot::IsNotAliased);
302    }
303
304    if (InitializedDecl) {
305      // Get the destructor for the reference temporary.
306      if (const RecordType *RT = E->getType()->getAs<RecordType>()) {
307        CXXRecordDecl *ClassDecl = cast<CXXRecordDecl>(RT->getDecl());
308        if (!ClassDecl->hasTrivialDestructor())
309          ReferenceTemporaryDtor = ClassDecl->getDestructor();
310      }
311    }
312
313    RV = CGF.EmitAnyExpr(E, AggSlot);
314
315    // Check if need to perform derived-to-base casts and/or field accesses, to
316    // get from the temporary object we created (and, potentially, for which we
317    // extended the lifetime) to the subobject we're binding the reference to.
318    if (!Adjustments.empty()) {
319      llvm::Value *Object = RV.getAggregateAddr();
320      for (unsigned I = Adjustments.size(); I != 0; --I) {
321        SubobjectAdjustment &Adjustment = Adjustments[I-1];
322        switch (Adjustment.Kind) {
323        case SubobjectAdjustment::DerivedToBaseAdjustment:
324          Object =
325              CGF.GetAddressOfBaseClass(Object,
326                                        Adjustment.DerivedToBase.DerivedClass,
327                              Adjustment.DerivedToBase.BasePath->path_begin(),
328                              Adjustment.DerivedToBase.BasePath->path_end(),
329                                        /*NullCheckValue=*/false);
330          break;
331
332        case SubobjectAdjustment::FieldAdjustment: {
333          LValue LV = CGF.MakeAddrLValue(Object, E->getType());
334          LV = CGF.EmitLValueForField(LV, Adjustment.Field);
335          if (LV.isSimple()) {
336            Object = LV.getAddress();
337            break;
338          }
339
340          // For non-simple lvalues, we actually have to create a copy of
341          // the object we're binding to.
342          QualType T = Adjustment.Field->getType().getNonReferenceType()
343                                                  .getUnqualifiedType();
344          Object = CreateReferenceTemporary(CGF, T, InitializedDecl);
345          LValue TempLV = CGF.MakeAddrLValue(Object,
346                                             Adjustment.Field->getType());
347          CGF.EmitStoreThroughLValue(CGF.EmitLoadOfLValue(LV), TempLV);
348          break;
349        }
350
351        case SubobjectAdjustment::MemberPointerAdjustment: {
352          llvm::Value *Ptr = CGF.EmitScalarExpr(Adjustment.Ptr.RHS);
353          Object = CGF.CGM.getCXXABI().EmitMemberDataPointerAddress(
354                        CGF, Object, Ptr, Adjustment.Ptr.MPT);
355          break;
356        }
357        }
358      }
359
360      return Object;
361    }
362  }
363
364  if (RV.isAggregate())
365    return RV.getAggregateAddr();
366
367  // Create a temporary variable that we can bind the reference to.
368  ReferenceTemporary = CreateReferenceTemporary(CGF, E->getType(),
369                                                InitializedDecl);
370
371
372  unsigned Alignment =
373    CGF.getContext().getTypeAlignInChars(E->getType()).getQuantity();
374  if (RV.isScalar())
375    CGF.EmitStoreOfScalar(RV.getScalarVal(), ReferenceTemporary,
376                          /*Volatile=*/false, Alignment, E->getType());
377  else
378    CGF.StoreComplexToAddr(RV.getComplexVal(), ReferenceTemporary,
379                           /*Volatile=*/false);
380  return ReferenceTemporary;
381}
382
383RValue
384CodeGenFunction::EmitReferenceBindingToExpr(const Expr *E,
385                                            const NamedDecl *InitializedDecl) {
386  llvm::Value *ReferenceTemporary = 0;
387  const CXXDestructorDecl *ReferenceTemporaryDtor = 0;
388  QualType ObjCARCReferenceLifetimeType;
389  llvm::Value *Value = EmitExprForReferenceBinding(*this, E, ReferenceTemporary,
390                                                   ReferenceTemporaryDtor,
391                                                   ObjCARCReferenceLifetimeType,
392                                                   InitializedDecl);
393  if (SanitizePerformTypeCheck && !E->getType()->isFunctionType()) {
394    // C++11 [dcl.ref]p5 (as amended by core issue 453):
395    //   If a glvalue to which a reference is directly bound designates neither
396    //   an existing object or function of an appropriate type nor a region of
397    //   storage of suitable size and alignment to contain an object of the
398    //   reference's type, the behavior is undefined.
399    QualType Ty = E->getType();
400    EmitTypeCheck(TCK_ReferenceBinding, E->getExprLoc(), Value, Ty);
401  }
402  if (!ReferenceTemporaryDtor && ObjCARCReferenceLifetimeType.isNull())
403    return RValue::get(Value);
404
405  // Make sure to call the destructor for the reference temporary.
406  const VarDecl *VD = dyn_cast_or_null<VarDecl>(InitializedDecl);
407  if (VD && VD->hasGlobalStorage()) {
408    if (ReferenceTemporaryDtor) {
409      llvm::Constant *DtorFn =
410        CGM.GetAddrOfCXXDestructor(ReferenceTemporaryDtor, Dtor_Complete);
411      CGM.getCXXABI().registerGlobalDtor(*this, DtorFn,
412                                    cast<llvm::Constant>(ReferenceTemporary));
413    } else {
414      assert(!ObjCARCReferenceLifetimeType.isNull());
415      // Note: We intentionally do not register a global "destructor" to
416      // release the object.
417    }
418
419    return RValue::get(Value);
420  }
421
422  if (ReferenceTemporaryDtor)
423    PushDestructorCleanup(ReferenceTemporaryDtor, ReferenceTemporary);
424  else {
425    switch (ObjCARCReferenceLifetimeType.getObjCLifetime()) {
426    case Qualifiers::OCL_None:
427      llvm_unreachable(
428                      "Not a reference temporary that needs to be deallocated");
429    case Qualifiers::OCL_ExplicitNone:
430    case Qualifiers::OCL_Autoreleasing:
431      // Nothing to do.
432      break;
433
434    case Qualifiers::OCL_Strong: {
435      bool precise = VD && VD->hasAttr<ObjCPreciseLifetimeAttr>();
436      CleanupKind cleanupKind = getARCCleanupKind();
437      pushDestroy(cleanupKind, ReferenceTemporary, ObjCARCReferenceLifetimeType,
438                  precise ? destroyARCStrongPrecise : destroyARCStrongImprecise,
439                  cleanupKind & EHCleanup);
440      break;
441    }
442
443    case Qualifiers::OCL_Weak: {
444      // __weak objects always get EH cleanups; otherwise, exceptions
445      // could cause really nasty crashes instead of mere leaks.
446      pushDestroy(NormalAndEHCleanup, ReferenceTemporary,
447                  ObjCARCReferenceLifetimeType, destroyARCWeak, true);
448      break;
449    }
450    }
451  }
452
453  return RValue::get(Value);
454}
455
456
457/// getAccessedFieldNo - Given an encoded value and a result number, return the
458/// input field number being accessed.
459unsigned CodeGenFunction::getAccessedFieldNo(unsigned Idx,
460                                             const llvm::Constant *Elts) {
461  return cast<llvm::ConstantInt>(Elts->getAggregateElement(Idx))
462      ->getZExtValue();
463}
464
465/// Emit the hash_16_bytes function from include/llvm/ADT/Hashing.h.
466static llvm::Value *emitHash16Bytes(CGBuilderTy &Builder, llvm::Value *Low,
467                                    llvm::Value *High) {
468  llvm::Value *KMul = Builder.getInt64(0x9ddfea08eb382d69ULL);
469  llvm::Value *K47 = Builder.getInt64(47);
470  llvm::Value *A0 = Builder.CreateMul(Builder.CreateXor(Low, High), KMul);
471  llvm::Value *A1 = Builder.CreateXor(Builder.CreateLShr(A0, K47), A0);
472  llvm::Value *B0 = Builder.CreateMul(Builder.CreateXor(High, A1), KMul);
473  llvm::Value *B1 = Builder.CreateXor(Builder.CreateLShr(B0, K47), B0);
474  return Builder.CreateMul(B1, KMul);
475}
476
477void CodeGenFunction::EmitTypeCheck(TypeCheckKind TCK, SourceLocation Loc,
478                                    llvm::Value *Address,
479                                    QualType Ty, CharUnits Alignment) {
480  if (!SanitizePerformTypeCheck)
481    return;
482
483  // Don't check pointers outside the default address space. The null check
484  // isn't correct, the object-size check isn't supported by LLVM, and we can't
485  // communicate the addresses to the runtime handler for the vptr check.
486  if (Address->getType()->getPointerAddressSpace())
487    return;
488
489  llvm::Value *Cond = 0;
490
491  if (SanOpts->Null) {
492    // The glvalue must not be an empty glvalue.
493    Cond = Builder.CreateICmpNE(
494        Address, llvm::Constant::getNullValue(Address->getType()));
495  }
496
497  if (SanOpts->ObjectSize && !Ty->isIncompleteType()) {
498    uint64_t Size = getContext().getTypeSizeInChars(Ty).getQuantity();
499
500    // The glvalue must refer to a large enough storage region.
501    // FIXME: If Address Sanitizer is enabled, insert dynamic instrumentation
502    //        to check this.
503    llvm::Value *F = CGM.getIntrinsic(llvm::Intrinsic::objectsize, IntPtrTy);
504    llvm::Value *Min = Builder.getFalse();
505    llvm::Value *CastAddr = Builder.CreateBitCast(Address, Int8PtrTy);
506    llvm::Value *LargeEnough =
507        Builder.CreateICmpUGE(Builder.CreateCall2(F, CastAddr, Min),
508                              llvm::ConstantInt::get(IntPtrTy, Size));
509    Cond = Cond ? Builder.CreateAnd(Cond, LargeEnough) : LargeEnough;
510  }
511
512  uint64_t AlignVal = 0;
513
514  if (SanOpts->Alignment) {
515    AlignVal = Alignment.getQuantity();
516    if (!Ty->isIncompleteType() && !AlignVal)
517      AlignVal = getContext().getTypeAlignInChars(Ty).getQuantity();
518
519    // The glvalue must be suitably aligned.
520    if (AlignVal) {
521      llvm::Value *Align =
522          Builder.CreateAnd(Builder.CreatePtrToInt(Address, IntPtrTy),
523                            llvm::ConstantInt::get(IntPtrTy, AlignVal - 1));
524      llvm::Value *Aligned =
525        Builder.CreateICmpEQ(Align, llvm::ConstantInt::get(IntPtrTy, 0));
526      Cond = Cond ? Builder.CreateAnd(Cond, Aligned) : Aligned;
527    }
528  }
529
530  if (Cond) {
531    llvm::Constant *StaticData[] = {
532      EmitCheckSourceLocation(Loc),
533      EmitCheckTypeDescriptor(Ty),
534      llvm::ConstantInt::get(SizeTy, AlignVal),
535      llvm::ConstantInt::get(Int8Ty, TCK)
536    };
537    EmitCheck(Cond, "type_mismatch", StaticData, Address, CRK_Recoverable);
538  }
539
540  // If possible, check that the vptr indicates that there is a subobject of
541  // type Ty at offset zero within this object.
542  //
543  // C++11 [basic.life]p5,6:
544  //   [For storage which does not refer to an object within its lifetime]
545  //   The program has undefined behavior if:
546  //    -- the [pointer or glvalue] is used to access a non-static data member
547  //       or call a non-static member function
548  CXXRecordDecl *RD = Ty->getAsCXXRecordDecl();
549  if (SanOpts->Vptr &&
550      (TCK == TCK_MemberAccess || TCK == TCK_MemberCall) &&
551      RD && RD->hasDefinition() && RD->isDynamicClass()) {
552    // Compute a hash of the mangled name of the type.
553    //
554    // FIXME: This is not guaranteed to be deterministic! Move to a
555    //        fingerprinting mechanism once LLVM provides one. For the time
556    //        being the implementation happens to be deterministic.
557    SmallString<64> MangledName;
558    llvm::raw_svector_ostream Out(MangledName);
559    CGM.getCXXABI().getMangleContext().mangleCXXRTTI(Ty.getUnqualifiedType(),
560                                                     Out);
561    llvm::hash_code TypeHash = hash_value(Out.str());
562
563    // Load the vptr, and compute hash_16_bytes(TypeHash, vptr).
564    llvm::Value *Low = llvm::ConstantInt::get(Int64Ty, TypeHash);
565    llvm::Type *VPtrTy = llvm::PointerType::get(IntPtrTy, 0);
566    llvm::Value *VPtrAddr = Builder.CreateBitCast(Address, VPtrTy);
567    llvm::Value *VPtrVal = Builder.CreateLoad(VPtrAddr);
568    llvm::Value *High = Builder.CreateZExt(VPtrVal, Int64Ty);
569
570    llvm::Value *Hash = emitHash16Bytes(Builder, Low, High);
571    Hash = Builder.CreateTrunc(Hash, IntPtrTy);
572
573    // Look the hash up in our cache.
574    const int CacheSize = 128;
575    llvm::Type *HashTable = llvm::ArrayType::get(IntPtrTy, CacheSize);
576    llvm::Value *Cache = CGM.CreateRuntimeVariable(HashTable,
577                                                   "__ubsan_vptr_type_cache");
578    llvm::Value *Slot = Builder.CreateAnd(Hash,
579                                          llvm::ConstantInt::get(IntPtrTy,
580                                                                 CacheSize-1));
581    llvm::Value *Indices[] = { Builder.getInt32(0), Slot };
582    llvm::Value *CacheVal =
583      Builder.CreateLoad(Builder.CreateInBoundsGEP(Cache, Indices));
584
585    // If the hash isn't in the cache, call a runtime handler to perform the
586    // hard work of checking whether the vptr is for an object of the right
587    // type. This will either fill in the cache and return, or produce a
588    // diagnostic.
589    llvm::Constant *StaticData[] = {
590      EmitCheckSourceLocation(Loc),
591      EmitCheckTypeDescriptor(Ty),
592      CGM.GetAddrOfRTTIDescriptor(Ty.getUnqualifiedType()),
593      llvm::ConstantInt::get(Int8Ty, TCK)
594    };
595    llvm::Value *DynamicData[] = { Address, Hash };
596    EmitCheck(Builder.CreateICmpEQ(CacheVal, Hash),
597              "dynamic_type_cache_miss", StaticData, DynamicData,
598              CRK_AlwaysRecoverable);
599  }
600}
601
602
603CodeGenFunction::ComplexPairTy CodeGenFunction::
604EmitComplexPrePostIncDec(const UnaryOperator *E, LValue LV,
605                         bool isInc, bool isPre) {
606  ComplexPairTy InVal = LoadComplexFromAddr(LV.getAddress(),
607                                            LV.isVolatileQualified());
608
609  llvm::Value *NextVal;
610  if (isa<llvm::IntegerType>(InVal.first->getType())) {
611    uint64_t AmountVal = isInc ? 1 : -1;
612    NextVal = llvm::ConstantInt::get(InVal.first->getType(), AmountVal, true);
613
614    // Add the inc/dec to the real part.
615    NextVal = Builder.CreateAdd(InVal.first, NextVal, isInc ? "inc" : "dec");
616  } else {
617    QualType ElemTy = E->getType()->getAs<ComplexType>()->getElementType();
618    llvm::APFloat FVal(getContext().getFloatTypeSemantics(ElemTy), 1);
619    if (!isInc)
620      FVal.changeSign();
621    NextVal = llvm::ConstantFP::get(getLLVMContext(), FVal);
622
623    // Add the inc/dec to the real part.
624    NextVal = Builder.CreateFAdd(InVal.first, NextVal, isInc ? "inc" : "dec");
625  }
626
627  ComplexPairTy IncVal(NextVal, InVal.second);
628
629  // Store the updated result through the lvalue.
630  StoreComplexToAddr(IncVal, LV.getAddress(), LV.isVolatileQualified());
631
632  // If this is a postinc, return the value read from memory, otherwise use the
633  // updated value.
634  return isPre ? IncVal : InVal;
635}
636
637
638//===----------------------------------------------------------------------===//
639//                         LValue Expression Emission
640//===----------------------------------------------------------------------===//
641
642RValue CodeGenFunction::GetUndefRValue(QualType Ty) {
643  if (Ty->isVoidType())
644    return RValue::get(0);
645
646  if (const ComplexType *CTy = Ty->getAs<ComplexType>()) {
647    llvm::Type *EltTy = ConvertType(CTy->getElementType());
648    llvm::Value *U = llvm::UndefValue::get(EltTy);
649    return RValue::getComplex(std::make_pair(U, U));
650  }
651
652  // If this is a use of an undefined aggregate type, the aggregate must have an
653  // identifiable address.  Just because the contents of the value are undefined
654  // doesn't mean that the address can't be taken and compared.
655  if (hasAggregateLLVMType(Ty)) {
656    llvm::Value *DestPtr = CreateMemTemp(Ty, "undef.agg.tmp");
657    return RValue::getAggregate(DestPtr);
658  }
659
660  return RValue::get(llvm::UndefValue::get(ConvertType(Ty)));
661}
662
663RValue CodeGenFunction::EmitUnsupportedRValue(const Expr *E,
664                                              const char *Name) {
665  ErrorUnsupported(E, Name);
666  return GetUndefRValue(E->getType());
667}
668
669LValue CodeGenFunction::EmitUnsupportedLValue(const Expr *E,
670                                              const char *Name) {
671  ErrorUnsupported(E, Name);
672  llvm::Type *Ty = llvm::PointerType::getUnqual(ConvertType(E->getType()));
673  return MakeAddrLValue(llvm::UndefValue::get(Ty), E->getType());
674}
675
676LValue CodeGenFunction::EmitCheckedLValue(const Expr *E, TypeCheckKind TCK) {
677  LValue LV = EmitLValue(E);
678  if (!isa<DeclRefExpr>(E) && !LV.isBitField() && LV.isSimple())
679    EmitTypeCheck(TCK, E->getExprLoc(), LV.getAddress(),
680                  E->getType(), LV.getAlignment());
681  return LV;
682}
683
684/// EmitLValue - Emit code to compute a designator that specifies the location
685/// of the expression.
686///
687/// This can return one of two things: a simple address or a bitfield reference.
688/// In either case, the LLVM Value* in the LValue structure is guaranteed to be
689/// an LLVM pointer type.
690///
691/// If this returns a bitfield reference, nothing about the pointee type of the
692/// LLVM value is known: For example, it may not be a pointer to an integer.
693///
694/// If this returns a normal address, and if the lvalue's C type is fixed size,
695/// this method guarantees that the returned pointer type will point to an LLVM
696/// type of the same size of the lvalue's type.  If the lvalue has a variable
697/// length type, this is not possible.
698///
699LValue CodeGenFunction::EmitLValue(const Expr *E) {
700  switch (E->getStmtClass()) {
701  default: return EmitUnsupportedLValue(E, "l-value expression");
702
703  case Expr::ObjCPropertyRefExprClass:
704    llvm_unreachable("cannot emit a property reference directly");
705
706  case Expr::ObjCSelectorExprClass:
707    return EmitObjCSelectorLValue(cast<ObjCSelectorExpr>(E));
708  case Expr::ObjCIsaExprClass:
709    return EmitObjCIsaExpr(cast<ObjCIsaExpr>(E));
710  case Expr::BinaryOperatorClass:
711    return EmitBinaryOperatorLValue(cast<BinaryOperator>(E));
712  case Expr::CompoundAssignOperatorClass:
713    if (!E->getType()->isAnyComplexType())
714      return EmitCompoundAssignmentLValue(cast<CompoundAssignOperator>(E));
715    return EmitComplexCompoundAssignmentLValue(cast<CompoundAssignOperator>(E));
716  case Expr::CallExprClass:
717  case Expr::CXXMemberCallExprClass:
718  case Expr::CXXOperatorCallExprClass:
719  case Expr::UserDefinedLiteralClass:
720    return EmitCallExprLValue(cast<CallExpr>(E));
721  case Expr::VAArgExprClass:
722    return EmitVAArgExprLValue(cast<VAArgExpr>(E));
723  case Expr::DeclRefExprClass:
724    return EmitDeclRefLValue(cast<DeclRefExpr>(E));
725  case Expr::ParenExprClass:
726    return EmitLValue(cast<ParenExpr>(E)->getSubExpr());
727  case Expr::GenericSelectionExprClass:
728    return EmitLValue(cast<GenericSelectionExpr>(E)->getResultExpr());
729  case Expr::PredefinedExprClass:
730    return EmitPredefinedLValue(cast<PredefinedExpr>(E));
731  case Expr::StringLiteralClass:
732    return EmitStringLiteralLValue(cast<StringLiteral>(E));
733  case Expr::ObjCEncodeExprClass:
734    return EmitObjCEncodeExprLValue(cast<ObjCEncodeExpr>(E));
735  case Expr::PseudoObjectExprClass:
736    return EmitPseudoObjectLValue(cast<PseudoObjectExpr>(E));
737  case Expr::InitListExprClass:
738    return EmitInitListLValue(cast<InitListExpr>(E));
739  case Expr::CXXTemporaryObjectExprClass:
740  case Expr::CXXConstructExprClass:
741    return EmitCXXConstructLValue(cast<CXXConstructExpr>(E));
742  case Expr::CXXBindTemporaryExprClass:
743    return EmitCXXBindTemporaryLValue(cast<CXXBindTemporaryExpr>(E));
744  case Expr::CXXUuidofExprClass:
745    return EmitCXXUuidofLValue(cast<CXXUuidofExpr>(E));
746  case Expr::LambdaExprClass:
747    return EmitLambdaLValue(cast<LambdaExpr>(E));
748
749  case Expr::ExprWithCleanupsClass: {
750    const ExprWithCleanups *cleanups = cast<ExprWithCleanups>(E);
751    enterFullExpression(cleanups);
752    RunCleanupsScope Scope(*this);
753    return EmitLValue(cleanups->getSubExpr());
754  }
755
756  case Expr::CXXScalarValueInitExprClass:
757    return EmitNullInitializationLValue(cast<CXXScalarValueInitExpr>(E));
758  case Expr::CXXDefaultArgExprClass:
759    return EmitLValue(cast<CXXDefaultArgExpr>(E)->getExpr());
760  case Expr::CXXTypeidExprClass:
761    return EmitCXXTypeidLValue(cast<CXXTypeidExpr>(E));
762
763  case Expr::ObjCMessageExprClass:
764    return EmitObjCMessageExprLValue(cast<ObjCMessageExpr>(E));
765  case Expr::ObjCIvarRefExprClass:
766    return EmitObjCIvarRefLValue(cast<ObjCIvarRefExpr>(E));
767  case Expr::StmtExprClass:
768    return EmitStmtExprLValue(cast<StmtExpr>(E));
769  case Expr::UnaryOperatorClass:
770    return EmitUnaryOpLValue(cast<UnaryOperator>(E));
771  case Expr::ArraySubscriptExprClass:
772    return EmitArraySubscriptExpr(cast<ArraySubscriptExpr>(E));
773  case Expr::ExtVectorElementExprClass:
774    return EmitExtVectorElementExpr(cast<ExtVectorElementExpr>(E));
775  case Expr::MemberExprClass:
776    return EmitMemberExpr(cast<MemberExpr>(E));
777  case Expr::CompoundLiteralExprClass:
778    return EmitCompoundLiteralLValue(cast<CompoundLiteralExpr>(E));
779  case Expr::ConditionalOperatorClass:
780    return EmitConditionalOperatorLValue(cast<ConditionalOperator>(E));
781  case Expr::BinaryConditionalOperatorClass:
782    return EmitConditionalOperatorLValue(cast<BinaryConditionalOperator>(E));
783  case Expr::ChooseExprClass:
784    return EmitLValue(cast<ChooseExpr>(E)->getChosenSubExpr(getContext()));
785  case Expr::OpaqueValueExprClass:
786    return EmitOpaqueValueLValue(cast<OpaqueValueExpr>(E));
787  case Expr::SubstNonTypeTemplateParmExprClass:
788    return EmitLValue(cast<SubstNonTypeTemplateParmExpr>(E)->getReplacement());
789  case Expr::ImplicitCastExprClass:
790  case Expr::CStyleCastExprClass:
791  case Expr::CXXFunctionalCastExprClass:
792  case Expr::CXXStaticCastExprClass:
793  case Expr::CXXDynamicCastExprClass:
794  case Expr::CXXReinterpretCastExprClass:
795  case Expr::CXXConstCastExprClass:
796  case Expr::ObjCBridgedCastExprClass:
797    return EmitCastLValue(cast<CastExpr>(E));
798
799  case Expr::MaterializeTemporaryExprClass:
800    return EmitMaterializeTemporaryExpr(cast<MaterializeTemporaryExpr>(E));
801  }
802}
803
804/// Given an object of the given canonical type, can we safely copy a
805/// value out of it based on its initializer?
806static bool isConstantEmittableObjectType(QualType type) {
807  assert(type.isCanonical());
808  assert(!type->isReferenceType());
809
810  // Must be const-qualified but non-volatile.
811  Qualifiers qs = type.getLocalQualifiers();
812  if (!qs.hasConst() || qs.hasVolatile()) return false;
813
814  // Otherwise, all object types satisfy this except C++ classes with
815  // mutable subobjects or non-trivial copy/destroy behavior.
816  if (const RecordType *RT = dyn_cast<RecordType>(type))
817    if (const CXXRecordDecl *RD = dyn_cast<CXXRecordDecl>(RT->getDecl()))
818      if (RD->hasMutableFields() || !RD->isTrivial())
819        return false;
820
821  return true;
822}
823
824/// Can we constant-emit a load of a reference to a variable of the
825/// given type?  This is different from predicates like
826/// Decl::isUsableInConstantExpressions because we do want it to apply
827/// in situations that don't necessarily satisfy the language's rules
828/// for this (e.g. C++'s ODR-use rules).  For example, we want to able
829/// to do this with const float variables even if those variables
830/// aren't marked 'constexpr'.
831enum ConstantEmissionKind {
832  CEK_None,
833  CEK_AsReferenceOnly,
834  CEK_AsValueOrReference,
835  CEK_AsValueOnly
836};
837static ConstantEmissionKind checkVarTypeForConstantEmission(QualType type) {
838  type = type.getCanonicalType();
839  if (const ReferenceType *ref = dyn_cast<ReferenceType>(type)) {
840    if (isConstantEmittableObjectType(ref->getPointeeType()))
841      return CEK_AsValueOrReference;
842    return CEK_AsReferenceOnly;
843  }
844  if (isConstantEmittableObjectType(type))
845    return CEK_AsValueOnly;
846  return CEK_None;
847}
848
849/// Try to emit a reference to the given value without producing it as
850/// an l-value.  This is actually more than an optimization: we can't
851/// produce an l-value for variables that we never actually captured
852/// in a block or lambda, which means const int variables or constexpr
853/// literals or similar.
854CodeGenFunction::ConstantEmission
855CodeGenFunction::tryEmitAsConstant(DeclRefExpr *refExpr) {
856  ValueDecl *value = refExpr->getDecl();
857
858  // The value needs to be an enum constant or a constant variable.
859  ConstantEmissionKind CEK;
860  if (isa<ParmVarDecl>(value)) {
861    CEK = CEK_None;
862  } else if (VarDecl *var = dyn_cast<VarDecl>(value)) {
863    CEK = checkVarTypeForConstantEmission(var->getType());
864  } else if (isa<EnumConstantDecl>(value)) {
865    CEK = CEK_AsValueOnly;
866  } else {
867    CEK = CEK_None;
868  }
869  if (CEK == CEK_None) return ConstantEmission();
870
871  Expr::EvalResult result;
872  bool resultIsReference;
873  QualType resultType;
874
875  // It's best to evaluate all the way as an r-value if that's permitted.
876  if (CEK != CEK_AsReferenceOnly &&
877      refExpr->EvaluateAsRValue(result, getContext())) {
878    resultIsReference = false;
879    resultType = refExpr->getType();
880
881  // Otherwise, try to evaluate as an l-value.
882  } else if (CEK != CEK_AsValueOnly &&
883             refExpr->EvaluateAsLValue(result, getContext())) {
884    resultIsReference = true;
885    resultType = value->getType();
886
887  // Failure.
888  } else {
889    return ConstantEmission();
890  }
891
892  // In any case, if the initializer has side-effects, abandon ship.
893  if (result.HasSideEffects)
894    return ConstantEmission();
895
896  // Emit as a constant.
897  llvm::Constant *C = CGM.EmitConstantValue(result.Val, resultType, this);
898
899  // Make sure we emit a debug reference to the global variable.
900  // This should probably fire even for
901  if (isa<VarDecl>(value)) {
902    if (!getContext().DeclMustBeEmitted(cast<VarDecl>(value)))
903      EmitDeclRefExprDbgValue(refExpr, C);
904  } else {
905    assert(isa<EnumConstantDecl>(value));
906    EmitDeclRefExprDbgValue(refExpr, C);
907  }
908
909  // If we emitted a reference constant, we need to dereference that.
910  if (resultIsReference)
911    return ConstantEmission::forReference(C);
912
913  return ConstantEmission::forValue(C);
914}
915
916llvm::Value *CodeGenFunction::EmitLoadOfScalar(LValue lvalue) {
917  return EmitLoadOfScalar(lvalue.getAddress(), lvalue.isVolatile(),
918                          lvalue.getAlignment().getQuantity(),
919                          lvalue.getType(), lvalue.getTBAAInfo());
920}
921
922static bool hasBooleanRepresentation(QualType Ty) {
923  if (Ty->isBooleanType())
924    return true;
925
926  if (const EnumType *ET = Ty->getAs<EnumType>())
927    return ET->getDecl()->getIntegerType()->isBooleanType();
928
929  if (const AtomicType *AT = Ty->getAs<AtomicType>())
930    return hasBooleanRepresentation(AT->getValueType());
931
932  return false;
933}
934
935static bool getRangeForType(CodeGenFunction &CGF, QualType Ty,
936                            llvm::APInt &Min, llvm::APInt &End,
937                            bool StrictEnums) {
938  const EnumType *ET = Ty->getAs<EnumType>();
939  bool IsRegularCPlusPlusEnum = CGF.getLangOpts().CPlusPlus && StrictEnums &&
940                                ET && !ET->getDecl()->isFixed();
941  bool IsBool = hasBooleanRepresentation(Ty);
942  if (!IsBool && !IsRegularCPlusPlusEnum)
943    return false;
944
945  if (IsBool) {
946    Min = llvm::APInt(CGF.getContext().getTypeSize(Ty), 0);
947    End = llvm::APInt(CGF.getContext().getTypeSize(Ty), 2);
948  } else {
949    const EnumDecl *ED = ET->getDecl();
950    llvm::Type *LTy = CGF.ConvertTypeForMem(ED->getIntegerType());
951    unsigned Bitwidth = LTy->getScalarSizeInBits();
952    unsigned NumNegativeBits = ED->getNumNegativeBits();
953    unsigned NumPositiveBits = ED->getNumPositiveBits();
954
955    if (NumNegativeBits) {
956      unsigned NumBits = std::max(NumNegativeBits, NumPositiveBits + 1);
957      assert(NumBits <= Bitwidth);
958      End = llvm::APInt(Bitwidth, 1) << (NumBits - 1);
959      Min = -End;
960    } else {
961      assert(NumPositiveBits <= Bitwidth);
962      End = llvm::APInt(Bitwidth, 1) << NumPositiveBits;
963      Min = llvm::APInt(Bitwidth, 0);
964    }
965  }
966  return true;
967}
968
969llvm::MDNode *CodeGenFunction::getRangeForLoadFromType(QualType Ty) {
970  llvm::APInt Min, End;
971  if (!getRangeForType(*this, Ty, Min, End,
972                       CGM.getCodeGenOpts().StrictEnums))
973    return 0;
974
975  llvm::MDBuilder MDHelper(getLLVMContext());
976  return MDHelper.createRange(Min, End);
977}
978
979llvm::Value *CodeGenFunction::EmitLoadOfScalar(llvm::Value *Addr, bool Volatile,
980                                              unsigned Alignment, QualType Ty,
981                                              llvm::MDNode *TBAAInfo) {
982
983  // For better performance, handle vector loads differently.
984  if (Ty->isVectorType()) {
985    llvm::Value *V;
986    const llvm::Type *EltTy =
987    cast<llvm::PointerType>(Addr->getType())->getElementType();
988
989    const llvm::VectorType *VTy = cast<llvm::VectorType>(EltTy);
990
991    // Handle vectors of size 3, like size 4 for better performance.
992    if (VTy->getNumElements() == 3) {
993
994      // Bitcast to vec4 type.
995      llvm::VectorType *vec4Ty = llvm::VectorType::get(VTy->getElementType(),
996                                                         4);
997      llvm::PointerType *ptVec4Ty =
998      llvm::PointerType::get(vec4Ty,
999                             (cast<llvm::PointerType>(
1000                                      Addr->getType()))->getAddressSpace());
1001      llvm::Value *Cast = Builder.CreateBitCast(Addr, ptVec4Ty,
1002                                                "castToVec4");
1003      // Now load value.
1004      llvm::Value *LoadVal = Builder.CreateLoad(Cast, Volatile, "loadVec4");
1005
1006      // Shuffle vector to get vec3.
1007      llvm::Constant *Mask[] = {
1008        llvm::ConstantInt::get(llvm::Type::getInt32Ty(getLLVMContext()), 0),
1009        llvm::ConstantInt::get(llvm::Type::getInt32Ty(getLLVMContext()), 1),
1010        llvm::ConstantInt::get(llvm::Type::getInt32Ty(getLLVMContext()), 2)
1011      };
1012
1013      llvm::Value *MaskV = llvm::ConstantVector::get(Mask);
1014      V = Builder.CreateShuffleVector(LoadVal,
1015                                      llvm::UndefValue::get(vec4Ty),
1016                                      MaskV, "extractVec");
1017      return EmitFromMemory(V, Ty);
1018    }
1019  }
1020
1021  llvm::LoadInst *Load = Builder.CreateLoad(Addr);
1022  if (Volatile)
1023    Load->setVolatile(true);
1024  if (Alignment)
1025    Load->setAlignment(Alignment);
1026  if (TBAAInfo)
1027    CGM.DecorateInstruction(Load, TBAAInfo);
1028  // If this is an atomic type, all normal reads must be atomic
1029  if (Ty->isAtomicType())
1030    Load->setAtomic(llvm::SequentiallyConsistent);
1031
1032  if ((SanOpts->Bool && hasBooleanRepresentation(Ty)) ||
1033      (SanOpts->Enum && Ty->getAs<EnumType>())) {
1034    llvm::APInt Min, End;
1035    if (getRangeForType(*this, Ty, Min, End, true)) {
1036      --End;
1037      llvm::Value *Check;
1038      if (!Min)
1039        Check = Builder.CreateICmpULE(
1040          Load, llvm::ConstantInt::get(getLLVMContext(), End));
1041      else {
1042        llvm::Value *Upper = Builder.CreateICmpSLE(
1043          Load, llvm::ConstantInt::get(getLLVMContext(), End));
1044        llvm::Value *Lower = Builder.CreateICmpSGE(
1045          Load, llvm::ConstantInt::get(getLLVMContext(), Min));
1046        Check = Builder.CreateAnd(Upper, Lower);
1047      }
1048      // FIXME: Provide a SourceLocation.
1049      EmitCheck(Check, "load_invalid_value", EmitCheckTypeDescriptor(Ty),
1050                EmitCheckValue(Load), CRK_Recoverable);
1051    }
1052  } else if (CGM.getCodeGenOpts().OptimizationLevel > 0)
1053    if (llvm::MDNode *RangeInfo = getRangeForLoadFromType(Ty))
1054      Load->setMetadata(llvm::LLVMContext::MD_range, RangeInfo);
1055
1056  return EmitFromMemory(Load, Ty);
1057}
1058
1059llvm::Value *CodeGenFunction::EmitToMemory(llvm::Value *Value, QualType Ty) {
1060  // Bool has a different representation in memory than in registers.
1061  if (hasBooleanRepresentation(Ty)) {
1062    // This should really always be an i1, but sometimes it's already
1063    // an i8, and it's awkward to track those cases down.
1064    if (Value->getType()->isIntegerTy(1))
1065      return Builder.CreateZExt(Value, ConvertTypeForMem(Ty), "frombool");
1066    assert(Value->getType()->isIntegerTy(getContext().getTypeSize(Ty)) &&
1067           "wrong value rep of bool");
1068  }
1069
1070  return Value;
1071}
1072
1073llvm::Value *CodeGenFunction::EmitFromMemory(llvm::Value *Value, QualType Ty) {
1074  // Bool has a different representation in memory than in registers.
1075  if (hasBooleanRepresentation(Ty)) {
1076    assert(Value->getType()->isIntegerTy(getContext().getTypeSize(Ty)) &&
1077           "wrong value rep of bool");
1078    return Builder.CreateTrunc(Value, Builder.getInt1Ty(), "tobool");
1079  }
1080
1081  return Value;
1082}
1083
1084void CodeGenFunction::EmitStoreOfScalar(llvm::Value *Value, llvm::Value *Addr,
1085                                        bool Volatile, unsigned Alignment,
1086                                        QualType Ty,
1087                                        llvm::MDNode *TBAAInfo,
1088                                        bool isInit) {
1089
1090  // Handle vectors differently to get better performance.
1091  if (Ty->isVectorType()) {
1092    llvm::Type *SrcTy = Value->getType();
1093    llvm::VectorType *VecTy = cast<llvm::VectorType>(SrcTy);
1094    // Handle vec3 special.
1095    if (VecTy->getNumElements() == 3) {
1096      llvm::LLVMContext &VMContext = getLLVMContext();
1097
1098      // Our source is a vec3, do a shuffle vector to make it a vec4.
1099      SmallVector<llvm::Constant*, 4> Mask;
1100      Mask.push_back(llvm::ConstantInt::get(
1101                                            llvm::Type::getInt32Ty(VMContext),
1102                                            0));
1103      Mask.push_back(llvm::ConstantInt::get(
1104                                            llvm::Type::getInt32Ty(VMContext),
1105                                            1));
1106      Mask.push_back(llvm::ConstantInt::get(
1107                                            llvm::Type::getInt32Ty(VMContext),
1108                                            2));
1109      Mask.push_back(llvm::UndefValue::get(llvm::Type::getInt32Ty(VMContext)));
1110
1111      llvm::Value *MaskV = llvm::ConstantVector::get(Mask);
1112      Value = Builder.CreateShuffleVector(Value,
1113                                          llvm::UndefValue::get(VecTy),
1114                                          MaskV, "extractVec");
1115      SrcTy = llvm::VectorType::get(VecTy->getElementType(), 4);
1116    }
1117    llvm::PointerType *DstPtr = cast<llvm::PointerType>(Addr->getType());
1118    if (DstPtr->getElementType() != SrcTy) {
1119      llvm::Type *MemTy =
1120      llvm::PointerType::get(SrcTy, DstPtr->getAddressSpace());
1121      Addr = Builder.CreateBitCast(Addr, MemTy, "storetmp");
1122    }
1123  }
1124
1125  Value = EmitToMemory(Value, Ty);
1126
1127  llvm::StoreInst *Store = Builder.CreateStore(Value, Addr, Volatile);
1128  if (Alignment)
1129    Store->setAlignment(Alignment);
1130  if (TBAAInfo)
1131    CGM.DecorateInstruction(Store, TBAAInfo);
1132  if (!isInit && Ty->isAtomicType())
1133    Store->setAtomic(llvm::SequentiallyConsistent);
1134}
1135
1136void CodeGenFunction::EmitStoreOfScalar(llvm::Value *value, LValue lvalue,
1137    bool isInit) {
1138  EmitStoreOfScalar(value, lvalue.getAddress(), lvalue.isVolatile(),
1139                    lvalue.getAlignment().getQuantity(), lvalue.getType(),
1140                    lvalue.getTBAAInfo(), isInit);
1141}
1142
1143/// EmitLoadOfLValue - Given an expression that represents a value lvalue, this
1144/// method emits the address of the lvalue, then loads the result as an rvalue,
1145/// returning the rvalue.
1146RValue CodeGenFunction::EmitLoadOfLValue(LValue LV) {
1147  if (LV.isObjCWeak()) {
1148    // load of a __weak object.
1149    llvm::Value *AddrWeakObj = LV.getAddress();
1150    return RValue::get(CGM.getObjCRuntime().EmitObjCWeakRead(*this,
1151                                                             AddrWeakObj));
1152  }
1153  if (LV.getQuals().getObjCLifetime() == Qualifiers::OCL_Weak) {
1154    llvm::Value *Object = EmitARCLoadWeakRetained(LV.getAddress());
1155    Object = EmitObjCConsumeObject(LV.getType(), Object);
1156    return RValue::get(Object);
1157  }
1158
1159  if (LV.isSimple()) {
1160    assert(!LV.getType()->isFunctionType());
1161
1162    // Everything needs a load.
1163    return RValue::get(EmitLoadOfScalar(LV));
1164  }
1165
1166  if (LV.isVectorElt()) {
1167    llvm::LoadInst *Load = Builder.CreateLoad(LV.getVectorAddr(),
1168                                              LV.isVolatileQualified());
1169    Load->setAlignment(LV.getAlignment().getQuantity());
1170    return RValue::get(Builder.CreateExtractElement(Load, LV.getVectorIdx(),
1171                                                    "vecext"));
1172  }
1173
1174  // If this is a reference to a subset of the elements of a vector, either
1175  // shuffle the input or extract/insert them as appropriate.
1176  if (LV.isExtVectorElt())
1177    return EmitLoadOfExtVectorElementLValue(LV);
1178
1179  assert(LV.isBitField() && "Unknown LValue type!");
1180  return EmitLoadOfBitfieldLValue(LV);
1181}
1182
1183RValue CodeGenFunction::EmitLoadOfBitfieldLValue(LValue LV) {
1184  const CGBitFieldInfo &Info = LV.getBitFieldInfo();
1185
1186  // Get the output type.
1187  llvm::Type *ResLTy = ConvertType(LV.getType());
1188
1189  llvm::Value *Ptr = LV.getBitFieldAddr();
1190  llvm::Value *Val = Builder.CreateLoad(Ptr, LV.isVolatileQualified(),
1191                                        "bf.load");
1192  cast<llvm::LoadInst>(Val)->setAlignment(Info.StorageAlignment);
1193
1194  if (Info.IsSigned) {
1195    assert(static_cast<unsigned>(Info.Offset + Info.Size) <= Info.StorageSize);
1196    unsigned HighBits = Info.StorageSize - Info.Offset - Info.Size;
1197    if (HighBits)
1198      Val = Builder.CreateShl(Val, HighBits, "bf.shl");
1199    if (Info.Offset + HighBits)
1200      Val = Builder.CreateAShr(Val, Info.Offset + HighBits, "bf.ashr");
1201  } else {
1202    if (Info.Offset)
1203      Val = Builder.CreateLShr(Val, Info.Offset, "bf.lshr");
1204    if (static_cast<unsigned>(Info.Offset) + Info.Size < Info.StorageSize)
1205      Val = Builder.CreateAnd(Val, llvm::APInt::getLowBitsSet(Info.StorageSize,
1206                                                              Info.Size),
1207                              "bf.clear");
1208  }
1209  Val = Builder.CreateIntCast(Val, ResLTy, Info.IsSigned, "bf.cast");
1210
1211  return RValue::get(Val);
1212}
1213
1214// If this is a reference to a subset of the elements of a vector, create an
1215// appropriate shufflevector.
1216RValue CodeGenFunction::EmitLoadOfExtVectorElementLValue(LValue LV) {
1217  llvm::LoadInst *Load = Builder.CreateLoad(LV.getExtVectorAddr(),
1218                                            LV.isVolatileQualified());
1219  Load->setAlignment(LV.getAlignment().getQuantity());
1220  llvm::Value *Vec = Load;
1221
1222  const llvm::Constant *Elts = LV.getExtVectorElts();
1223
1224  // If the result of the expression is a non-vector type, we must be extracting
1225  // a single element.  Just codegen as an extractelement.
1226  const VectorType *ExprVT = LV.getType()->getAs<VectorType>();
1227  if (!ExprVT) {
1228    unsigned InIdx = getAccessedFieldNo(0, Elts);
1229    llvm::Value *Elt = llvm::ConstantInt::get(Int32Ty, InIdx);
1230    return RValue::get(Builder.CreateExtractElement(Vec, Elt));
1231  }
1232
1233  // Always use shuffle vector to try to retain the original program structure
1234  unsigned NumResultElts = ExprVT->getNumElements();
1235
1236  SmallVector<llvm::Constant*, 4> Mask;
1237  for (unsigned i = 0; i != NumResultElts; ++i)
1238    Mask.push_back(Builder.getInt32(getAccessedFieldNo(i, Elts)));
1239
1240  llvm::Value *MaskV = llvm::ConstantVector::get(Mask);
1241  Vec = Builder.CreateShuffleVector(Vec, llvm::UndefValue::get(Vec->getType()),
1242                                    MaskV);
1243  return RValue::get(Vec);
1244}
1245
1246
1247
1248/// EmitStoreThroughLValue - Store the specified rvalue into the specified
1249/// lvalue, where both are guaranteed to the have the same type, and that type
1250/// is 'Ty'.
1251void CodeGenFunction::EmitStoreThroughLValue(RValue Src, LValue Dst, bool isInit) {
1252  if (!Dst.isSimple()) {
1253    if (Dst.isVectorElt()) {
1254      // Read/modify/write the vector, inserting the new element.
1255      llvm::LoadInst *Load = Builder.CreateLoad(Dst.getVectorAddr(),
1256                                                Dst.isVolatileQualified());
1257      Load->setAlignment(Dst.getAlignment().getQuantity());
1258      llvm::Value *Vec = Load;
1259      Vec = Builder.CreateInsertElement(Vec, Src.getScalarVal(),
1260                                        Dst.getVectorIdx(), "vecins");
1261      llvm::StoreInst *Store = Builder.CreateStore(Vec, Dst.getVectorAddr(),
1262                                                   Dst.isVolatileQualified());
1263      Store->setAlignment(Dst.getAlignment().getQuantity());
1264      return;
1265    }
1266
1267    // If this is an update of extended vector elements, insert them as
1268    // appropriate.
1269    if (Dst.isExtVectorElt())
1270      return EmitStoreThroughExtVectorComponentLValue(Src, Dst);
1271
1272    assert(Dst.isBitField() && "Unknown LValue type");
1273    return EmitStoreThroughBitfieldLValue(Src, Dst);
1274  }
1275
1276  // There's special magic for assigning into an ARC-qualified l-value.
1277  if (Qualifiers::ObjCLifetime Lifetime = Dst.getQuals().getObjCLifetime()) {
1278    switch (Lifetime) {
1279    case Qualifiers::OCL_None:
1280      llvm_unreachable("present but none");
1281
1282    case Qualifiers::OCL_ExplicitNone:
1283      // nothing special
1284      break;
1285
1286    case Qualifiers::OCL_Strong:
1287      EmitARCStoreStrong(Dst, Src.getScalarVal(), /*ignore*/ true);
1288      return;
1289
1290    case Qualifiers::OCL_Weak:
1291      EmitARCStoreWeak(Dst.getAddress(), Src.getScalarVal(), /*ignore*/ true);
1292      return;
1293
1294    case Qualifiers::OCL_Autoreleasing:
1295      Src = RValue::get(EmitObjCExtendObjectLifetime(Dst.getType(),
1296                                                     Src.getScalarVal()));
1297      // fall into the normal path
1298      break;
1299    }
1300  }
1301
1302  if (Dst.isObjCWeak() && !Dst.isNonGC()) {
1303    // load of a __weak object.
1304    llvm::Value *LvalueDst = Dst.getAddress();
1305    llvm::Value *src = Src.getScalarVal();
1306     CGM.getObjCRuntime().EmitObjCWeakAssign(*this, src, LvalueDst);
1307    return;
1308  }
1309
1310  if (Dst.isObjCStrong() && !Dst.isNonGC()) {
1311    // load of a __strong object.
1312    llvm::Value *LvalueDst = Dst.getAddress();
1313    llvm::Value *src = Src.getScalarVal();
1314    if (Dst.isObjCIvar()) {
1315      assert(Dst.getBaseIvarExp() && "BaseIvarExp is NULL");
1316      llvm::Type *ResultType = ConvertType(getContext().LongTy);
1317      llvm::Value *RHS = EmitScalarExpr(Dst.getBaseIvarExp());
1318      llvm::Value *dst = RHS;
1319      RHS = Builder.CreatePtrToInt(RHS, ResultType, "sub.ptr.rhs.cast");
1320      llvm::Value *LHS =
1321        Builder.CreatePtrToInt(LvalueDst, ResultType, "sub.ptr.lhs.cast");
1322      llvm::Value *BytesBetween = Builder.CreateSub(LHS, RHS, "ivar.offset");
1323      CGM.getObjCRuntime().EmitObjCIvarAssign(*this, src, dst,
1324                                              BytesBetween);
1325    } else if (Dst.isGlobalObjCRef()) {
1326      CGM.getObjCRuntime().EmitObjCGlobalAssign(*this, src, LvalueDst,
1327                                                Dst.isThreadLocalRef());
1328    }
1329    else
1330      CGM.getObjCRuntime().EmitObjCStrongCastAssign(*this, src, LvalueDst);
1331    return;
1332  }
1333
1334  assert(Src.isScalar() && "Can't emit an agg store with this method");
1335  EmitStoreOfScalar(Src.getScalarVal(), Dst, isInit);
1336}
1337
1338void CodeGenFunction::EmitStoreThroughBitfieldLValue(RValue Src, LValue Dst,
1339                                                     llvm::Value **Result) {
1340  const CGBitFieldInfo &Info = Dst.getBitFieldInfo();
1341  llvm::Type *ResLTy = ConvertTypeForMem(Dst.getType());
1342  llvm::Value *Ptr = Dst.getBitFieldAddr();
1343
1344  // Get the source value, truncated to the width of the bit-field.
1345  llvm::Value *SrcVal = Src.getScalarVal();
1346
1347  // Cast the source to the storage type and shift it into place.
1348  SrcVal = Builder.CreateIntCast(SrcVal,
1349                                 Ptr->getType()->getPointerElementType(),
1350                                 /*IsSigned=*/false);
1351  llvm::Value *MaskedVal = SrcVal;
1352
1353  // See if there are other bits in the bitfield's storage we'll need to load
1354  // and mask together with source before storing.
1355  if (Info.StorageSize != Info.Size) {
1356    assert(Info.StorageSize > Info.Size && "Invalid bitfield size.");
1357    llvm::Value *Val = Builder.CreateLoad(Ptr, Dst.isVolatileQualified(),
1358                                          "bf.load");
1359    cast<llvm::LoadInst>(Val)->setAlignment(Info.StorageAlignment);
1360
1361    // Mask the source value as needed.
1362    if (!hasBooleanRepresentation(Dst.getType()))
1363      SrcVal = Builder.CreateAnd(SrcVal,
1364                                 llvm::APInt::getLowBitsSet(Info.StorageSize,
1365                                                            Info.Size),
1366                                 "bf.value");
1367    MaskedVal = SrcVal;
1368    if (Info.Offset)
1369      SrcVal = Builder.CreateShl(SrcVal, Info.Offset, "bf.shl");
1370
1371    // Mask out the original value.
1372    Val = Builder.CreateAnd(Val,
1373                            ~llvm::APInt::getBitsSet(Info.StorageSize,
1374                                                     Info.Offset,
1375                                                     Info.Offset + Info.Size),
1376                            "bf.clear");
1377
1378    // Or together the unchanged values and the source value.
1379    SrcVal = Builder.CreateOr(Val, SrcVal, "bf.set");
1380  } else {
1381    assert(Info.Offset == 0);
1382  }
1383
1384  // Write the new value back out.
1385  llvm::StoreInst *Store = Builder.CreateStore(SrcVal, Ptr,
1386                                               Dst.isVolatileQualified());
1387  Store->setAlignment(Info.StorageAlignment);
1388
1389  // Return the new value of the bit-field, if requested.
1390  if (Result) {
1391    llvm::Value *ResultVal = MaskedVal;
1392
1393    // Sign extend the value if needed.
1394    if (Info.IsSigned) {
1395      assert(Info.Size <= Info.StorageSize);
1396      unsigned HighBits = Info.StorageSize - Info.Size;
1397      if (HighBits) {
1398        ResultVal = Builder.CreateShl(ResultVal, HighBits, "bf.result.shl");
1399        ResultVal = Builder.CreateAShr(ResultVal, HighBits, "bf.result.ashr");
1400      }
1401    }
1402
1403    ResultVal = Builder.CreateIntCast(ResultVal, ResLTy, Info.IsSigned,
1404                                      "bf.result.cast");
1405    *Result = EmitFromMemory(ResultVal, Dst.getType());
1406  }
1407}
1408
1409void CodeGenFunction::EmitStoreThroughExtVectorComponentLValue(RValue Src,
1410                                                               LValue Dst) {
1411  // This access turns into a read/modify/write of the vector.  Load the input
1412  // value now.
1413  llvm::LoadInst *Load = Builder.CreateLoad(Dst.getExtVectorAddr(),
1414                                            Dst.isVolatileQualified());
1415  Load->setAlignment(Dst.getAlignment().getQuantity());
1416  llvm::Value *Vec = Load;
1417  const llvm::Constant *Elts = Dst.getExtVectorElts();
1418
1419  llvm::Value *SrcVal = Src.getScalarVal();
1420
1421  if (const VectorType *VTy = Dst.getType()->getAs<VectorType>()) {
1422    unsigned NumSrcElts = VTy->getNumElements();
1423    unsigned NumDstElts =
1424       cast<llvm::VectorType>(Vec->getType())->getNumElements();
1425    if (NumDstElts == NumSrcElts) {
1426      // Use shuffle vector is the src and destination are the same number of
1427      // elements and restore the vector mask since it is on the side it will be
1428      // stored.
1429      SmallVector<llvm::Constant*, 4> Mask(NumDstElts);
1430      for (unsigned i = 0; i != NumSrcElts; ++i)
1431        Mask[getAccessedFieldNo(i, Elts)] = Builder.getInt32(i);
1432
1433      llvm::Value *MaskV = llvm::ConstantVector::get(Mask);
1434      Vec = Builder.CreateShuffleVector(SrcVal,
1435                                        llvm::UndefValue::get(Vec->getType()),
1436                                        MaskV);
1437    } else if (NumDstElts > NumSrcElts) {
1438      // Extended the source vector to the same length and then shuffle it
1439      // into the destination.
1440      // FIXME: since we're shuffling with undef, can we just use the indices
1441      //        into that?  This could be simpler.
1442      SmallVector<llvm::Constant*, 4> ExtMask;
1443      for (unsigned i = 0; i != NumSrcElts; ++i)
1444        ExtMask.push_back(Builder.getInt32(i));
1445      ExtMask.resize(NumDstElts, llvm::UndefValue::get(Int32Ty));
1446      llvm::Value *ExtMaskV = llvm::ConstantVector::get(ExtMask);
1447      llvm::Value *ExtSrcVal =
1448        Builder.CreateShuffleVector(SrcVal,
1449                                    llvm::UndefValue::get(SrcVal->getType()),
1450                                    ExtMaskV);
1451      // build identity
1452      SmallVector<llvm::Constant*, 4> Mask;
1453      for (unsigned i = 0; i != NumDstElts; ++i)
1454        Mask.push_back(Builder.getInt32(i));
1455
1456      // modify when what gets shuffled in
1457      for (unsigned i = 0; i != NumSrcElts; ++i)
1458        Mask[getAccessedFieldNo(i, Elts)] = Builder.getInt32(i+NumDstElts);
1459      llvm::Value *MaskV = llvm::ConstantVector::get(Mask);
1460      Vec = Builder.CreateShuffleVector(Vec, ExtSrcVal, MaskV);
1461    } else {
1462      // We should never shorten the vector
1463      llvm_unreachable("unexpected shorten vector length");
1464    }
1465  } else {
1466    // If the Src is a scalar (not a vector) it must be updating one element.
1467    unsigned InIdx = getAccessedFieldNo(0, Elts);
1468    llvm::Value *Elt = llvm::ConstantInt::get(Int32Ty, InIdx);
1469    Vec = Builder.CreateInsertElement(Vec, SrcVal, Elt);
1470  }
1471
1472  llvm::StoreInst *Store = Builder.CreateStore(Vec, Dst.getExtVectorAddr(),
1473                                               Dst.isVolatileQualified());
1474  Store->setAlignment(Dst.getAlignment().getQuantity());
1475}
1476
1477// setObjCGCLValueClass - sets class of he lvalue for the purpose of
1478// generating write-barries API. It is currently a global, ivar,
1479// or neither.
1480static void setObjCGCLValueClass(const ASTContext &Ctx, const Expr *E,
1481                                 LValue &LV,
1482                                 bool IsMemberAccess=false) {
1483  if (Ctx.getLangOpts().getGC() == LangOptions::NonGC)
1484    return;
1485
1486  if (isa<ObjCIvarRefExpr>(E)) {
1487    QualType ExpTy = E->getType();
1488    if (IsMemberAccess && ExpTy->isPointerType()) {
1489      // If ivar is a structure pointer, assigning to field of
1490      // this struct follows gcc's behavior and makes it a non-ivar
1491      // writer-barrier conservatively.
1492      ExpTy = ExpTy->getAs<PointerType>()->getPointeeType();
1493      if (ExpTy->isRecordType()) {
1494        LV.setObjCIvar(false);
1495        return;
1496      }
1497    }
1498    LV.setObjCIvar(true);
1499    ObjCIvarRefExpr *Exp = cast<ObjCIvarRefExpr>(const_cast<Expr*>(E));
1500    LV.setBaseIvarExp(Exp->getBase());
1501    LV.setObjCArray(E->getType()->isArrayType());
1502    return;
1503  }
1504
1505  if (const DeclRefExpr *Exp = dyn_cast<DeclRefExpr>(E)) {
1506    if (const VarDecl *VD = dyn_cast<VarDecl>(Exp->getDecl())) {
1507      if (VD->hasGlobalStorage()) {
1508        LV.setGlobalObjCRef(true);
1509        LV.setThreadLocalRef(VD->isThreadSpecified());
1510      }
1511    }
1512    LV.setObjCArray(E->getType()->isArrayType());
1513    return;
1514  }
1515
1516  if (const UnaryOperator *Exp = dyn_cast<UnaryOperator>(E)) {
1517    setObjCGCLValueClass(Ctx, Exp->getSubExpr(), LV, IsMemberAccess);
1518    return;
1519  }
1520
1521  if (const ParenExpr *Exp = dyn_cast<ParenExpr>(E)) {
1522    setObjCGCLValueClass(Ctx, Exp->getSubExpr(), LV, IsMemberAccess);
1523    if (LV.isObjCIvar()) {
1524      // If cast is to a structure pointer, follow gcc's behavior and make it
1525      // a non-ivar write-barrier.
1526      QualType ExpTy = E->getType();
1527      if (ExpTy->isPointerType())
1528        ExpTy = ExpTy->getAs<PointerType>()->getPointeeType();
1529      if (ExpTy->isRecordType())
1530        LV.setObjCIvar(false);
1531    }
1532    return;
1533  }
1534
1535  if (const GenericSelectionExpr *Exp = dyn_cast<GenericSelectionExpr>(E)) {
1536    setObjCGCLValueClass(Ctx, Exp->getResultExpr(), LV);
1537    return;
1538  }
1539
1540  if (const ImplicitCastExpr *Exp = dyn_cast<ImplicitCastExpr>(E)) {
1541    setObjCGCLValueClass(Ctx, Exp->getSubExpr(), LV, IsMemberAccess);
1542    return;
1543  }
1544
1545  if (const CStyleCastExpr *Exp = dyn_cast<CStyleCastExpr>(E)) {
1546    setObjCGCLValueClass(Ctx, Exp->getSubExpr(), LV, IsMemberAccess);
1547    return;
1548  }
1549
1550  if (const ObjCBridgedCastExpr *Exp = dyn_cast<ObjCBridgedCastExpr>(E)) {
1551    setObjCGCLValueClass(Ctx, Exp->getSubExpr(), LV, IsMemberAccess);
1552    return;
1553  }
1554
1555  if (const ArraySubscriptExpr *Exp = dyn_cast<ArraySubscriptExpr>(E)) {
1556    setObjCGCLValueClass(Ctx, Exp->getBase(), LV);
1557    if (LV.isObjCIvar() && !LV.isObjCArray())
1558      // Using array syntax to assigning to what an ivar points to is not
1559      // same as assigning to the ivar itself. {id *Names;} Names[i] = 0;
1560      LV.setObjCIvar(false);
1561    else if (LV.isGlobalObjCRef() && !LV.isObjCArray())
1562      // Using array syntax to assigning to what global points to is not
1563      // same as assigning to the global itself. {id *G;} G[i] = 0;
1564      LV.setGlobalObjCRef(false);
1565    return;
1566  }
1567
1568  if (const MemberExpr *Exp = dyn_cast<MemberExpr>(E)) {
1569    setObjCGCLValueClass(Ctx, Exp->getBase(), LV, true);
1570    // We don't know if member is an 'ivar', but this flag is looked at
1571    // only in the context of LV.isObjCIvar().
1572    LV.setObjCArray(E->getType()->isArrayType());
1573    return;
1574  }
1575}
1576
1577static llvm::Value *
1578EmitBitCastOfLValueToProperType(CodeGenFunction &CGF,
1579                                llvm::Value *V, llvm::Type *IRType,
1580                                StringRef Name = StringRef()) {
1581  unsigned AS = cast<llvm::PointerType>(V->getType())->getAddressSpace();
1582  return CGF.Builder.CreateBitCast(V, IRType->getPointerTo(AS), Name);
1583}
1584
1585static LValue EmitGlobalVarDeclLValue(CodeGenFunction &CGF,
1586                                      const Expr *E, const VarDecl *VD) {
1587  llvm::Value *V = CGF.CGM.GetAddrOfGlobalVar(VD);
1588  llvm::Type *RealVarTy = CGF.getTypes().ConvertTypeForMem(VD->getType());
1589  V = EmitBitCastOfLValueToProperType(CGF, V, RealVarTy);
1590  CharUnits Alignment = CGF.getContext().getDeclAlign(VD);
1591  QualType T = E->getType();
1592  LValue LV;
1593  if (VD->getType()->isReferenceType()) {
1594    llvm::LoadInst *LI = CGF.Builder.CreateLoad(V);
1595    LI->setAlignment(Alignment.getQuantity());
1596    V = LI;
1597    LV = CGF.MakeNaturalAlignAddrLValue(V, T);
1598  } else {
1599    LV = CGF.MakeAddrLValue(V, E->getType(), Alignment);
1600  }
1601  setObjCGCLValueClass(CGF.getContext(), E, LV);
1602  return LV;
1603}
1604
1605static LValue EmitFunctionDeclLValue(CodeGenFunction &CGF,
1606                                     const Expr *E, const FunctionDecl *FD) {
1607  llvm::Value *V = CGF.CGM.GetAddrOfFunction(FD);
1608  if (!FD->hasPrototype()) {
1609    if (const FunctionProtoType *Proto =
1610            FD->getType()->getAs<FunctionProtoType>()) {
1611      // Ugly case: for a K&R-style definition, the type of the definition
1612      // isn't the same as the type of a use.  Correct for this with a
1613      // bitcast.
1614      QualType NoProtoType =
1615          CGF.getContext().getFunctionNoProtoType(Proto->getResultType());
1616      NoProtoType = CGF.getContext().getPointerType(NoProtoType);
1617      V = CGF.Builder.CreateBitCast(V, CGF.ConvertType(NoProtoType));
1618    }
1619  }
1620  CharUnits Alignment = CGF.getContext().getDeclAlign(FD);
1621  return CGF.MakeAddrLValue(V, E->getType(), Alignment);
1622}
1623
1624LValue CodeGenFunction::EmitDeclRefLValue(const DeclRefExpr *E) {
1625  const NamedDecl *ND = E->getDecl();
1626  CharUnits Alignment = getContext().getDeclAlign(ND);
1627  QualType T = E->getType();
1628
1629  // A DeclRefExpr for a reference initialized by a constant expression can
1630  // appear without being odr-used. Directly emit the constant initializer.
1631  if (const VarDecl *VD = dyn_cast<VarDecl>(ND)) {
1632    const Expr *Init = VD->getAnyInitializer(VD);
1633    if (Init && !isa<ParmVarDecl>(VD) && VD->getType()->isReferenceType() &&
1634        VD->isUsableInConstantExpressions(getContext()) &&
1635        VD->checkInitIsICE()) {
1636      llvm::Constant *Val =
1637        CGM.EmitConstantValue(*VD->evaluateValue(), VD->getType(), this);
1638      assert(Val && "failed to emit reference constant expression");
1639      // FIXME: Eventually we will want to emit vector element references.
1640      return MakeAddrLValue(Val, T, Alignment);
1641    }
1642  }
1643
1644  // FIXME: We should be able to assert this for FunctionDecls as well!
1645  // FIXME: We should be able to assert this for all DeclRefExprs, not just
1646  // those with a valid source location.
1647  assert((ND->isUsed(false) || !isa<VarDecl>(ND) ||
1648          !E->getLocation().isValid()) &&
1649         "Should not use decl without marking it used!");
1650
1651  if (ND->hasAttr<WeakRefAttr>()) {
1652    const ValueDecl *VD = cast<ValueDecl>(ND);
1653    llvm::Constant *Aliasee = CGM.GetWeakRefReference(VD);
1654    return MakeAddrLValue(Aliasee, T, Alignment);
1655  }
1656
1657  if (const VarDecl *VD = dyn_cast<VarDecl>(ND)) {
1658    // Check if this is a global variable.
1659    if (VD->hasLinkage() || VD->isStaticDataMember())
1660      return EmitGlobalVarDeclLValue(*this, E, VD);
1661
1662    bool isBlockVariable = VD->hasAttr<BlocksAttr>();
1663
1664    bool NonGCable = VD->hasLocalStorage() &&
1665                     !VD->getType()->isReferenceType() &&
1666                     !isBlockVariable;
1667
1668    llvm::Value *V = LocalDeclMap.lookup(VD);
1669    if (!V && VD->isStaticLocal())
1670      V = CGM.getStaticLocalDeclAddress(VD);
1671
1672    // Use special handling for lambdas.
1673    if (!V) {
1674      if (FieldDecl *FD = LambdaCaptureFields.lookup(VD)) {
1675        QualType LambdaTagType = getContext().getTagDeclType(FD->getParent());
1676        LValue LambdaLV = MakeNaturalAlignAddrLValue(CXXABIThisValue,
1677                                                     LambdaTagType);
1678        return EmitLValueForField(LambdaLV, FD);
1679      }
1680
1681      assert(isa<BlockDecl>(CurCodeDecl) && E->refersToEnclosingLocal());
1682      return MakeAddrLValue(GetAddrOfBlockDecl(VD, isBlockVariable),
1683                            T, Alignment);
1684    }
1685
1686    assert(V && "DeclRefExpr not entered in LocalDeclMap?");
1687
1688    if (isBlockVariable)
1689      V = BuildBlockByrefAddress(V, VD);
1690
1691    LValue LV;
1692    if (VD->getType()->isReferenceType()) {
1693      llvm::LoadInst *LI = Builder.CreateLoad(V);
1694      LI->setAlignment(Alignment.getQuantity());
1695      V = LI;
1696      LV = MakeNaturalAlignAddrLValue(V, T);
1697    } else {
1698      LV = MakeAddrLValue(V, T, Alignment);
1699    }
1700
1701    if (NonGCable) {
1702      LV.getQuals().removeObjCGCAttr();
1703      LV.setNonGC(true);
1704    }
1705    setObjCGCLValueClass(getContext(), E, LV);
1706    return LV;
1707  }
1708
1709  if (const FunctionDecl *fn = dyn_cast<FunctionDecl>(ND))
1710    return EmitFunctionDeclLValue(*this, E, fn);
1711
1712  llvm_unreachable("Unhandled DeclRefExpr");
1713}
1714
1715LValue CodeGenFunction::EmitUnaryOpLValue(const UnaryOperator *E) {
1716  // __extension__ doesn't affect lvalue-ness.
1717  if (E->getOpcode() == UO_Extension)
1718    return EmitLValue(E->getSubExpr());
1719
1720  QualType ExprTy = getContext().getCanonicalType(E->getSubExpr()->getType());
1721  switch (E->getOpcode()) {
1722  default: llvm_unreachable("Unknown unary operator lvalue!");
1723  case UO_Deref: {
1724    QualType T = E->getSubExpr()->getType()->getPointeeType();
1725    assert(!T.isNull() && "CodeGenFunction::EmitUnaryOpLValue: Illegal type");
1726
1727    LValue LV = MakeNaturalAlignAddrLValue(EmitScalarExpr(E->getSubExpr()), T);
1728    LV.getQuals().setAddressSpace(ExprTy.getAddressSpace());
1729
1730    // We should not generate __weak write barrier on indirect reference
1731    // of a pointer to object; as in void foo (__weak id *param); *param = 0;
1732    // But, we continue to generate __strong write barrier on indirect write
1733    // into a pointer to object.
1734    if (getLangOpts().ObjC1 &&
1735        getLangOpts().getGC() != LangOptions::NonGC &&
1736        LV.isObjCWeak())
1737      LV.setNonGC(!E->isOBJCGCCandidate(getContext()));
1738    return LV;
1739  }
1740  case UO_Real:
1741  case UO_Imag: {
1742    LValue LV = EmitLValue(E->getSubExpr());
1743    assert(LV.isSimple() && "real/imag on non-ordinary l-value");
1744    llvm::Value *Addr = LV.getAddress();
1745
1746    // __real is valid on scalars.  This is a faster way of testing that.
1747    // __imag can only produce an rvalue on scalars.
1748    if (E->getOpcode() == UO_Real &&
1749        !cast<llvm::PointerType>(Addr->getType())
1750           ->getElementType()->isStructTy()) {
1751      assert(E->getSubExpr()->getType()->isArithmeticType());
1752      return LV;
1753    }
1754
1755    assert(E->getSubExpr()->getType()->isAnyComplexType());
1756
1757    unsigned Idx = E->getOpcode() == UO_Imag;
1758    return MakeAddrLValue(Builder.CreateStructGEP(LV.getAddress(),
1759                                                  Idx, "idx"),
1760                          ExprTy);
1761  }
1762  case UO_PreInc:
1763  case UO_PreDec: {
1764    LValue LV = EmitLValue(E->getSubExpr());
1765    bool isInc = E->getOpcode() == UO_PreInc;
1766
1767    if (E->getType()->isAnyComplexType())
1768      EmitComplexPrePostIncDec(E, LV, isInc, true/*isPre*/);
1769    else
1770      EmitScalarPrePostIncDec(E, LV, isInc, true/*isPre*/);
1771    return LV;
1772  }
1773  }
1774}
1775
1776LValue CodeGenFunction::EmitStringLiteralLValue(const StringLiteral *E) {
1777  return MakeAddrLValue(CGM.GetAddrOfConstantStringFromLiteral(E),
1778                        E->getType());
1779}
1780
1781LValue CodeGenFunction::EmitObjCEncodeExprLValue(const ObjCEncodeExpr *E) {
1782  return MakeAddrLValue(CGM.GetAddrOfConstantStringFromObjCEncode(E),
1783                        E->getType());
1784}
1785
1786static llvm::Constant*
1787GetAddrOfConstantWideString(StringRef Str,
1788                            const char *GlobalName,
1789                            ASTContext &Context,
1790                            QualType Ty, SourceLocation Loc,
1791                            CodeGenModule &CGM) {
1792
1793  StringLiteral *SL = StringLiteral::Create(Context,
1794                                            Str,
1795                                            StringLiteral::Wide,
1796                                            /*Pascal = */false,
1797                                            Ty, Loc);
1798  llvm::Constant *C = CGM.GetConstantArrayFromStringLiteral(SL);
1799  llvm::GlobalVariable *GV =
1800    new llvm::GlobalVariable(CGM.getModule(), C->getType(),
1801                             !CGM.getLangOpts().WritableStrings,
1802                             llvm::GlobalValue::PrivateLinkage,
1803                             C, GlobalName);
1804  const unsigned WideAlignment =
1805    Context.getTypeAlignInChars(Ty).getQuantity();
1806  GV->setAlignment(WideAlignment);
1807  return GV;
1808}
1809
1810static void ConvertUTF8ToWideString(unsigned CharByteWidth, StringRef Source,
1811                                    SmallString<32>& Target) {
1812  Target.resize(CharByteWidth * (Source.size() + 1));
1813  char *ResultPtr = &Target[0];
1814  const UTF8 *ErrorPtr;
1815  bool success = ConvertUTF8toWide(CharByteWidth, Source, ResultPtr, ErrorPtr);
1816  (void)success;
1817  assert(success);
1818  Target.resize(ResultPtr - &Target[0]);
1819}
1820
1821LValue CodeGenFunction::EmitPredefinedLValue(const PredefinedExpr *E) {
1822  switch (E->getIdentType()) {
1823  default:
1824    return EmitUnsupportedLValue(E, "predefined expression");
1825
1826  case PredefinedExpr::Func:
1827  case PredefinedExpr::Function:
1828  case PredefinedExpr::LFunction:
1829  case PredefinedExpr::PrettyFunction: {
1830    unsigned IdentType = E->getIdentType();
1831    std::string GlobalVarName;
1832
1833    switch (IdentType) {
1834    default: llvm_unreachable("Invalid type");
1835    case PredefinedExpr::Func:
1836      GlobalVarName = "__func__.";
1837      break;
1838    case PredefinedExpr::Function:
1839      GlobalVarName = "__FUNCTION__.";
1840      break;
1841    case PredefinedExpr::LFunction:
1842      GlobalVarName = "L__FUNCTION__.";
1843      break;
1844    case PredefinedExpr::PrettyFunction:
1845      GlobalVarName = "__PRETTY_FUNCTION__.";
1846      break;
1847    }
1848
1849    StringRef FnName = CurFn->getName();
1850    if (FnName.startswith("\01"))
1851      FnName = FnName.substr(1);
1852    GlobalVarName += FnName;
1853
1854    const Decl *CurDecl = CurCodeDecl;
1855    if (CurDecl == 0)
1856      CurDecl = getContext().getTranslationUnitDecl();
1857
1858    std::string FunctionName =
1859        (isa<BlockDecl>(CurDecl)
1860         ? FnName.str()
1861         : PredefinedExpr::ComputeName((PredefinedExpr::IdentType)IdentType,
1862                                       CurDecl));
1863
1864    const Type* ElemType = E->getType()->getArrayElementTypeNoTypeQual();
1865    llvm::Constant *C;
1866    if (ElemType->isWideCharType()) {
1867      SmallString<32> RawChars;
1868      ConvertUTF8ToWideString(
1869          getContext().getTypeSizeInChars(ElemType).getQuantity(),
1870          FunctionName, RawChars);
1871      C = GetAddrOfConstantWideString(RawChars,
1872                                      GlobalVarName.c_str(),
1873                                      getContext(),
1874                                      E->getType(),
1875                                      E->getLocation(),
1876                                      CGM);
1877    } else {
1878      C = CGM.GetAddrOfConstantCString(FunctionName,
1879                                       GlobalVarName.c_str(),
1880                                       1);
1881    }
1882    return MakeAddrLValue(C, E->getType());
1883  }
1884  }
1885}
1886
1887/// Emit a type description suitable for use by a runtime sanitizer library. The
1888/// format of a type descriptor is
1889///
1890/// \code
1891///   { i16 TypeKind, i16 TypeInfo }
1892/// \endcode
1893///
1894/// followed by an array of i8 containing the type name. TypeKind is 0 for an
1895/// integer, 1 for a floating point value, and -1 for anything else.
1896llvm::Constant *CodeGenFunction::EmitCheckTypeDescriptor(QualType T) {
1897  // FIXME: Only emit each type's descriptor once.
1898  uint16_t TypeKind = -1;
1899  uint16_t TypeInfo = 0;
1900
1901  if (T->isIntegerType()) {
1902    TypeKind = 0;
1903    TypeInfo = (llvm::Log2_32(getContext().getTypeSize(T)) << 1) |
1904               (T->isSignedIntegerType() ? 1 : 0);
1905  } else if (T->isFloatingType()) {
1906    TypeKind = 1;
1907    TypeInfo = getContext().getTypeSize(T);
1908  }
1909
1910  // Format the type name as if for a diagnostic, including quotes and
1911  // optionally an 'aka'.
1912  SmallString<32> Buffer;
1913  CGM.getDiags().ConvertArgToString(DiagnosticsEngine::ak_qualtype,
1914                                    (intptr_t)T.getAsOpaquePtr(),
1915                                    0, 0, 0, 0, 0, 0, Buffer,
1916                                    ArrayRef<intptr_t>());
1917
1918  llvm::Constant *Components[] = {
1919    Builder.getInt16(TypeKind), Builder.getInt16(TypeInfo),
1920    llvm::ConstantDataArray::getString(getLLVMContext(), Buffer)
1921  };
1922  llvm::Constant *Descriptor = llvm::ConstantStruct::getAnon(Components);
1923
1924  llvm::GlobalVariable *GV =
1925    new llvm::GlobalVariable(CGM.getModule(), Descriptor->getType(),
1926                             /*isConstant=*/true,
1927                             llvm::GlobalVariable::PrivateLinkage,
1928                             Descriptor);
1929  GV->setUnnamedAddr(true);
1930  return GV;
1931}
1932
1933llvm::Value *CodeGenFunction::EmitCheckValue(llvm::Value *V) {
1934  llvm::Type *TargetTy = IntPtrTy;
1935
1936  // Integers which fit in intptr_t are zero-extended and passed directly.
1937  if (V->getType()->isIntegerTy() &&
1938      V->getType()->getIntegerBitWidth() <= TargetTy->getIntegerBitWidth())
1939    return Builder.CreateZExt(V, TargetTy);
1940
1941  // Pointers are passed directly, everything else is passed by address.
1942  if (!V->getType()->isPointerTy()) {
1943    llvm::Value *Ptr = Builder.CreateAlloca(V->getType());
1944    Builder.CreateStore(V, Ptr);
1945    V = Ptr;
1946  }
1947  return Builder.CreatePtrToInt(V, TargetTy);
1948}
1949
1950/// \brief Emit a representation of a SourceLocation for passing to a handler
1951/// in a sanitizer runtime library. The format for this data is:
1952/// \code
1953///   struct SourceLocation {
1954///     const char *Filename;
1955///     int32_t Line, Column;
1956///   };
1957/// \endcode
1958/// For an invalid SourceLocation, the Filename pointer is null.
1959llvm::Constant *CodeGenFunction::EmitCheckSourceLocation(SourceLocation Loc) {
1960  PresumedLoc PLoc = getContext().getSourceManager().getPresumedLoc(Loc);
1961
1962  llvm::Constant *Data[] = {
1963    // FIXME: Only emit each file name once.
1964    PLoc.isValid() ? cast<llvm::Constant>(
1965                       Builder.CreateGlobalStringPtr(PLoc.getFilename()))
1966                   : llvm::Constant::getNullValue(Int8PtrTy),
1967    Builder.getInt32(PLoc.getLine()),
1968    Builder.getInt32(PLoc.getColumn())
1969  };
1970
1971  return llvm::ConstantStruct::getAnon(Data);
1972}
1973
1974void CodeGenFunction::EmitCheck(llvm::Value *Checked, StringRef CheckName,
1975                                ArrayRef<llvm::Constant *> StaticArgs,
1976                                ArrayRef<llvm::Value *> DynamicArgs,
1977                                CheckRecoverableKind RecoverKind) {
1978  assert(SanOpts != &SanitizerOptions::Disabled);
1979
1980  if (CGM.getCodeGenOpts().SanitizeUndefinedTrapOnError) {
1981    assert (RecoverKind != CRK_AlwaysRecoverable &&
1982            "Runtime call required for AlwaysRecoverable kind!");
1983    return EmitTrapCheck(Checked);
1984  }
1985
1986  llvm::BasicBlock *Cont = createBasicBlock("cont");
1987
1988  llvm::BasicBlock *Handler = createBasicBlock("handler." + CheckName);
1989
1990  llvm::Instruction *Branch = Builder.CreateCondBr(Checked, Cont, Handler);
1991
1992  // Give hint that we very much don't expect to execute the handler
1993  // Value chosen to match UR_NONTAKEN_WEIGHT, see BranchProbabilityInfo.cpp
1994  llvm::MDBuilder MDHelper(getLLVMContext());
1995  llvm::MDNode *Node = MDHelper.createBranchWeights((1U << 20) - 1, 1);
1996  Branch->setMetadata(llvm::LLVMContext::MD_prof, Node);
1997
1998  EmitBlock(Handler);
1999
2000  llvm::Constant *Info = llvm::ConstantStruct::getAnon(StaticArgs);
2001  llvm::GlobalValue *InfoPtr =
2002      new llvm::GlobalVariable(CGM.getModule(), Info->getType(), false,
2003                               llvm::GlobalVariable::PrivateLinkage, Info);
2004  InfoPtr->setUnnamedAddr(true);
2005
2006  SmallVector<llvm::Value *, 4> Args;
2007  SmallVector<llvm::Type *, 4> ArgTypes;
2008  Args.reserve(DynamicArgs.size() + 1);
2009  ArgTypes.reserve(DynamicArgs.size() + 1);
2010
2011  // Handler functions take an i8* pointing to the (handler-specific) static
2012  // information block, followed by a sequence of intptr_t arguments
2013  // representing operand values.
2014  Args.push_back(Builder.CreateBitCast(InfoPtr, Int8PtrTy));
2015  ArgTypes.push_back(Int8PtrTy);
2016  for (size_t i = 0, n = DynamicArgs.size(); i != n; ++i) {
2017    Args.push_back(EmitCheckValue(DynamicArgs[i]));
2018    ArgTypes.push_back(IntPtrTy);
2019  }
2020
2021  bool Recover = (RecoverKind == CRK_AlwaysRecoverable) ||
2022                 ((RecoverKind == CRK_Recoverable) &&
2023                   CGM.getCodeGenOpts().SanitizeRecover);
2024
2025  llvm::FunctionType *FnType =
2026    llvm::FunctionType::get(CGM.VoidTy, ArgTypes, false);
2027  llvm::AttrBuilder B;
2028  if (!Recover) {
2029    B.addAttribute(llvm::Attribute::NoReturn)
2030     .addAttribute(llvm::Attribute::NoUnwind);
2031  }
2032  B.addAttribute(llvm::Attribute::UWTable);
2033
2034  // Checks that have two variants use a suffix to differentiate them
2035  bool NeedsAbortSuffix = (RecoverKind != CRK_Unrecoverable) &&
2036                           !CGM.getCodeGenOpts().SanitizeRecover;
2037  std::string FunctionName = ("__ubsan_handle_" + CheckName +
2038                              (NeedsAbortSuffix? "_abort" : "")).str();
2039  llvm::Value *Fn =
2040    CGM.CreateRuntimeFunction(FnType, FunctionName,
2041                              llvm::Attribute::get(getLLVMContext(), B));
2042  llvm::CallInst *HandlerCall = Builder.CreateCall(Fn, Args);
2043  if (Recover) {
2044    Builder.CreateBr(Cont);
2045  } else {
2046    HandlerCall->setDoesNotReturn();
2047    HandlerCall->setDoesNotThrow();
2048    Builder.CreateUnreachable();
2049  }
2050
2051  EmitBlock(Cont);
2052}
2053
2054void CodeGenFunction::EmitTrapCheck(llvm::Value *Checked) {
2055  llvm::BasicBlock *Cont = createBasicBlock("cont");
2056
2057  // If we're optimizing, collapse all calls to trap down to just one per
2058  // function to save on code size.
2059  if (!CGM.getCodeGenOpts().OptimizationLevel || !TrapBB) {
2060    TrapBB = createBasicBlock("trap");
2061    Builder.CreateCondBr(Checked, Cont, TrapBB);
2062    EmitBlock(TrapBB);
2063    llvm::Value *F = CGM.getIntrinsic(llvm::Intrinsic::trap);
2064    llvm::CallInst *TrapCall = Builder.CreateCall(F);
2065    TrapCall->setDoesNotReturn();
2066    TrapCall->setDoesNotThrow();
2067    Builder.CreateUnreachable();
2068  } else {
2069    Builder.CreateCondBr(Checked, Cont, TrapBB);
2070  }
2071
2072  EmitBlock(Cont);
2073}
2074
2075/// isSimpleArrayDecayOperand - If the specified expr is a simple decay from an
2076/// array to pointer, return the array subexpression.
2077static const Expr *isSimpleArrayDecayOperand(const Expr *E) {
2078  // If this isn't just an array->pointer decay, bail out.
2079  const CastExpr *CE = dyn_cast<CastExpr>(E);
2080  if (CE == 0 || CE->getCastKind() != CK_ArrayToPointerDecay)
2081    return 0;
2082
2083  // If this is a decay from variable width array, bail out.
2084  const Expr *SubExpr = CE->getSubExpr();
2085  if (SubExpr->getType()->isVariableArrayType())
2086    return 0;
2087
2088  return SubExpr;
2089}
2090
2091LValue CodeGenFunction::EmitArraySubscriptExpr(const ArraySubscriptExpr *E) {
2092  // The index must always be an integer, which is not an aggregate.  Emit it.
2093  llvm::Value *Idx = EmitScalarExpr(E->getIdx());
2094  QualType IdxTy  = E->getIdx()->getType();
2095  bool IdxSigned = IdxTy->isSignedIntegerOrEnumerationType();
2096
2097  // If the base is a vector type, then we are forming a vector element lvalue
2098  // with this subscript.
2099  if (E->getBase()->getType()->isVectorType()) {
2100    // Emit the vector as an lvalue to get its address.
2101    LValue LHS = EmitLValue(E->getBase());
2102    assert(LHS.isSimple() && "Can only subscript lvalue vectors here!");
2103    Idx = Builder.CreateIntCast(Idx, Int32Ty, IdxSigned, "vidx");
2104    return LValue::MakeVectorElt(LHS.getAddress(), Idx,
2105                                 E->getBase()->getType(), LHS.getAlignment());
2106  }
2107
2108  // Extend or truncate the index type to 32 or 64-bits.
2109  if (Idx->getType() != IntPtrTy)
2110    Idx = Builder.CreateIntCast(Idx, IntPtrTy, IdxSigned, "idxprom");
2111
2112  // We know that the pointer points to a type of the correct size, unless the
2113  // size is a VLA or Objective-C interface.
2114  llvm::Value *Address = 0;
2115  CharUnits ArrayAlignment;
2116  if (const VariableArrayType *vla =
2117        getContext().getAsVariableArrayType(E->getType())) {
2118    // The base must be a pointer, which is not an aggregate.  Emit
2119    // it.  It needs to be emitted first in case it's what captures
2120    // the VLA bounds.
2121    Address = EmitScalarExpr(E->getBase());
2122
2123    // The element count here is the total number of non-VLA elements.
2124    llvm::Value *numElements = getVLASize(vla).first;
2125
2126    // Effectively, the multiply by the VLA size is part of the GEP.
2127    // GEP indexes are signed, and scaling an index isn't permitted to
2128    // signed-overflow, so we use the same semantics for our explicit
2129    // multiply.  We suppress this if overflow is not undefined behavior.
2130    if (getLangOpts().isSignedOverflowDefined()) {
2131      Idx = Builder.CreateMul(Idx, numElements);
2132      Address = Builder.CreateGEP(Address, Idx, "arrayidx");
2133    } else {
2134      Idx = Builder.CreateNSWMul(Idx, numElements);
2135      Address = Builder.CreateInBoundsGEP(Address, Idx, "arrayidx");
2136    }
2137  } else if (const ObjCObjectType *OIT = E->getType()->getAs<ObjCObjectType>()){
2138    // Indexing over an interface, as in "NSString *P; P[4];"
2139    llvm::Value *InterfaceSize =
2140      llvm::ConstantInt::get(Idx->getType(),
2141          getContext().getTypeSizeInChars(OIT).getQuantity());
2142
2143    Idx = Builder.CreateMul(Idx, InterfaceSize);
2144
2145    // The base must be a pointer, which is not an aggregate.  Emit it.
2146    llvm::Value *Base = EmitScalarExpr(E->getBase());
2147    Address = EmitCastToVoidPtr(Base);
2148    Address = Builder.CreateGEP(Address, Idx, "arrayidx");
2149    Address = Builder.CreateBitCast(Address, Base->getType());
2150  } else if (const Expr *Array = isSimpleArrayDecayOperand(E->getBase())) {
2151    // If this is A[i] where A is an array, the frontend will have decayed the
2152    // base to be a ArrayToPointerDecay implicit cast.  While correct, it is
2153    // inefficient at -O0 to emit a "gep A, 0, 0" when codegen'ing it, then a
2154    // "gep x, i" here.  Emit one "gep A, 0, i".
2155    assert(Array->getType()->isArrayType() &&
2156           "Array to pointer decay must have array source type!");
2157    LValue ArrayLV = EmitLValue(Array);
2158    llvm::Value *ArrayPtr = ArrayLV.getAddress();
2159    llvm::Value *Zero = llvm::ConstantInt::get(Int32Ty, 0);
2160    llvm::Value *Args[] = { Zero, Idx };
2161
2162    // Propagate the alignment from the array itself to the result.
2163    ArrayAlignment = ArrayLV.getAlignment();
2164
2165    if (getLangOpts().isSignedOverflowDefined())
2166      Address = Builder.CreateGEP(ArrayPtr, Args, "arrayidx");
2167    else
2168      Address = Builder.CreateInBoundsGEP(ArrayPtr, Args, "arrayidx");
2169  } else {
2170    // The base must be a pointer, which is not an aggregate.  Emit it.
2171    llvm::Value *Base = EmitScalarExpr(E->getBase());
2172    if (getLangOpts().isSignedOverflowDefined())
2173      Address = Builder.CreateGEP(Base, Idx, "arrayidx");
2174    else
2175      Address = Builder.CreateInBoundsGEP(Base, Idx, "arrayidx");
2176  }
2177
2178  QualType T = E->getBase()->getType()->getPointeeType();
2179  assert(!T.isNull() &&
2180         "CodeGenFunction::EmitArraySubscriptExpr(): Illegal base type");
2181
2182
2183  // Limit the alignment to that of the result type.
2184  LValue LV;
2185  if (!ArrayAlignment.isZero()) {
2186    CharUnits Align = getContext().getTypeAlignInChars(T);
2187    ArrayAlignment = std::min(Align, ArrayAlignment);
2188    LV = MakeAddrLValue(Address, T, ArrayAlignment);
2189  } else {
2190    LV = MakeNaturalAlignAddrLValue(Address, T);
2191  }
2192
2193  LV.getQuals().setAddressSpace(E->getBase()->getType().getAddressSpace());
2194
2195  if (getLangOpts().ObjC1 &&
2196      getLangOpts().getGC() != LangOptions::NonGC) {
2197    LV.setNonGC(!E->isOBJCGCCandidate(getContext()));
2198    setObjCGCLValueClass(getContext(), E, LV);
2199  }
2200  return LV;
2201}
2202
2203static
2204llvm::Constant *GenerateConstantVector(CGBuilderTy &Builder,
2205                                       SmallVector<unsigned, 4> &Elts) {
2206  SmallVector<llvm::Constant*, 4> CElts;
2207  for (unsigned i = 0, e = Elts.size(); i != e; ++i)
2208    CElts.push_back(Builder.getInt32(Elts[i]));
2209
2210  return llvm::ConstantVector::get(CElts);
2211}
2212
2213LValue CodeGenFunction::
2214EmitExtVectorElementExpr(const ExtVectorElementExpr *E) {
2215  // Emit the base vector as an l-value.
2216  LValue Base;
2217
2218  // ExtVectorElementExpr's base can either be a vector or pointer to vector.
2219  if (E->isArrow()) {
2220    // If it is a pointer to a vector, emit the address and form an lvalue with
2221    // it.
2222    llvm::Value *Ptr = EmitScalarExpr(E->getBase());
2223    const PointerType *PT = E->getBase()->getType()->getAs<PointerType>();
2224    Base = MakeAddrLValue(Ptr, PT->getPointeeType());
2225    Base.getQuals().removeObjCGCAttr();
2226  } else if (E->getBase()->isGLValue()) {
2227    // Otherwise, if the base is an lvalue ( as in the case of foo.x.x),
2228    // emit the base as an lvalue.
2229    assert(E->getBase()->getType()->isVectorType());
2230    Base = EmitLValue(E->getBase());
2231  } else {
2232    // Otherwise, the base is a normal rvalue (as in (V+V).x), emit it as such.
2233    assert(E->getBase()->getType()->isVectorType() &&
2234           "Result must be a vector");
2235    llvm::Value *Vec = EmitScalarExpr(E->getBase());
2236
2237    // Store the vector to memory (because LValue wants an address).
2238    llvm::Value *VecMem = CreateMemTemp(E->getBase()->getType());
2239    Builder.CreateStore(Vec, VecMem);
2240    Base = MakeAddrLValue(VecMem, E->getBase()->getType());
2241  }
2242
2243  QualType type =
2244    E->getType().withCVRQualifiers(Base.getQuals().getCVRQualifiers());
2245
2246  // Encode the element access list into a vector of unsigned indices.
2247  SmallVector<unsigned, 4> Indices;
2248  E->getEncodedElementAccess(Indices);
2249
2250  if (Base.isSimple()) {
2251    llvm::Constant *CV = GenerateConstantVector(Builder, Indices);
2252    return LValue::MakeExtVectorElt(Base.getAddress(), CV, type,
2253                                    Base.getAlignment());
2254  }
2255  assert(Base.isExtVectorElt() && "Can only subscript lvalue vec elts here!");
2256
2257  llvm::Constant *BaseElts = Base.getExtVectorElts();
2258  SmallVector<llvm::Constant *, 4> CElts;
2259
2260  for (unsigned i = 0, e = Indices.size(); i != e; ++i)
2261    CElts.push_back(BaseElts->getAggregateElement(Indices[i]));
2262  llvm::Constant *CV = llvm::ConstantVector::get(CElts);
2263  return LValue::MakeExtVectorElt(Base.getExtVectorAddr(), CV, type,
2264                                  Base.getAlignment());
2265}
2266
2267LValue CodeGenFunction::EmitMemberExpr(const MemberExpr *E) {
2268  Expr *BaseExpr = E->getBase();
2269
2270  // If this is s.x, emit s as an lvalue.  If it is s->x, emit s as a scalar.
2271  LValue BaseLV;
2272  if (E->isArrow()) {
2273    llvm::Value *Ptr = EmitScalarExpr(BaseExpr);
2274    QualType PtrTy = BaseExpr->getType()->getPointeeType();
2275    EmitTypeCheck(TCK_MemberAccess, E->getExprLoc(), Ptr, PtrTy);
2276    BaseLV = MakeNaturalAlignAddrLValue(Ptr, PtrTy);
2277  } else
2278    BaseLV = EmitCheckedLValue(BaseExpr, TCK_MemberAccess);
2279
2280  NamedDecl *ND = E->getMemberDecl();
2281  if (FieldDecl *Field = dyn_cast<FieldDecl>(ND)) {
2282    LValue LV = EmitLValueForField(BaseLV, Field);
2283    setObjCGCLValueClass(getContext(), E, LV);
2284    return LV;
2285  }
2286
2287  if (VarDecl *VD = dyn_cast<VarDecl>(ND))
2288    return EmitGlobalVarDeclLValue(*this, E, VD);
2289
2290  if (const FunctionDecl *FD = dyn_cast<FunctionDecl>(ND))
2291    return EmitFunctionDeclLValue(*this, E, FD);
2292
2293  llvm_unreachable("Unhandled member declaration!");
2294}
2295
2296LValue CodeGenFunction::EmitLValueForField(LValue base,
2297                                           const FieldDecl *field) {
2298  if (field->isBitField()) {
2299    const CGRecordLayout &RL =
2300      CGM.getTypes().getCGRecordLayout(field->getParent());
2301    const CGBitFieldInfo &Info = RL.getBitFieldInfo(field);
2302    llvm::Value *Addr = base.getAddress();
2303    unsigned Idx = RL.getLLVMFieldNo(field);
2304    if (Idx != 0)
2305      // For structs, we GEP to the field that the record layout suggests.
2306      Addr = Builder.CreateStructGEP(Addr, Idx, field->getName());
2307    // Get the access type.
2308    llvm::Type *PtrTy = llvm::Type::getIntNPtrTy(
2309      getLLVMContext(), Info.StorageSize,
2310      CGM.getContext().getTargetAddressSpace(base.getType()));
2311    if (Addr->getType() != PtrTy)
2312      Addr = Builder.CreateBitCast(Addr, PtrTy);
2313
2314    QualType fieldType =
2315      field->getType().withCVRQualifiers(base.getVRQualifiers());
2316    return LValue::MakeBitfield(Addr, Info, fieldType, base.getAlignment());
2317  }
2318
2319  const RecordDecl *rec = field->getParent();
2320  QualType type = field->getType();
2321  CharUnits alignment = getContext().getDeclAlign(field);
2322
2323  // FIXME: It should be impossible to have an LValue without alignment for a
2324  // complete type.
2325  if (!base.getAlignment().isZero())
2326    alignment = std::min(alignment, base.getAlignment());
2327
2328  bool mayAlias = rec->hasAttr<MayAliasAttr>();
2329
2330  llvm::Value *addr = base.getAddress();
2331  unsigned cvr = base.getVRQualifiers();
2332  if (rec->isUnion()) {
2333    // For unions, there is no pointer adjustment.
2334    assert(!type->isReferenceType() && "union has reference member");
2335  } else {
2336    // For structs, we GEP to the field that the record layout suggests.
2337    unsigned idx = CGM.getTypes().getCGRecordLayout(rec).getLLVMFieldNo(field);
2338    addr = Builder.CreateStructGEP(addr, idx, field->getName());
2339
2340    // If this is a reference field, load the reference right now.
2341    if (const ReferenceType *refType = type->getAs<ReferenceType>()) {
2342      llvm::LoadInst *load = Builder.CreateLoad(addr, "ref");
2343      if (cvr & Qualifiers::Volatile) load->setVolatile(true);
2344      load->setAlignment(alignment.getQuantity());
2345
2346      if (CGM.shouldUseTBAA()) {
2347        llvm::MDNode *tbaa;
2348        if (mayAlias)
2349          tbaa = CGM.getTBAAInfo(getContext().CharTy);
2350        else
2351          tbaa = CGM.getTBAAInfo(type);
2352        CGM.DecorateInstruction(load, tbaa);
2353      }
2354
2355      addr = load;
2356      mayAlias = false;
2357      type = refType->getPointeeType();
2358      if (type->isIncompleteType())
2359        alignment = CharUnits();
2360      else
2361        alignment = getContext().getTypeAlignInChars(type);
2362      cvr = 0; // qualifiers don't recursively apply to referencee
2363    }
2364  }
2365
2366  // Make sure that the address is pointing to the right type.  This is critical
2367  // for both unions and structs.  A union needs a bitcast, a struct element
2368  // will need a bitcast if the LLVM type laid out doesn't match the desired
2369  // type.
2370  addr = EmitBitCastOfLValueToProperType(*this, addr,
2371                                         CGM.getTypes().ConvertTypeForMem(type),
2372                                         field->getName());
2373
2374  if (field->hasAttr<AnnotateAttr>())
2375    addr = EmitFieldAnnotations(field, addr);
2376
2377  LValue LV = MakeAddrLValue(addr, type, alignment);
2378  LV.getQuals().addCVRQualifiers(cvr);
2379
2380  // __weak attribute on a field is ignored.
2381  if (LV.getQuals().getObjCGCAttr() == Qualifiers::Weak)
2382    LV.getQuals().removeObjCGCAttr();
2383
2384  // Fields of may_alias structs act like 'char' for TBAA purposes.
2385  // FIXME: this should get propagated down through anonymous structs
2386  // and unions.
2387  if (mayAlias && LV.getTBAAInfo())
2388    LV.setTBAAInfo(CGM.getTBAAInfo(getContext().CharTy));
2389
2390  return LV;
2391}
2392
2393LValue
2394CodeGenFunction::EmitLValueForFieldInitialization(LValue Base,
2395                                                  const FieldDecl *Field) {
2396  QualType FieldType = Field->getType();
2397
2398  if (!FieldType->isReferenceType())
2399    return EmitLValueForField(Base, Field);
2400
2401  const CGRecordLayout &RL =
2402    CGM.getTypes().getCGRecordLayout(Field->getParent());
2403  unsigned idx = RL.getLLVMFieldNo(Field);
2404  llvm::Value *V = Builder.CreateStructGEP(Base.getAddress(), idx);
2405  assert(!FieldType.getObjCGCAttr() && "fields cannot have GC attrs");
2406
2407  // Make sure that the address is pointing to the right type.  This is critical
2408  // for both unions and structs.  A union needs a bitcast, a struct element
2409  // will need a bitcast if the LLVM type laid out doesn't match the desired
2410  // type.
2411  llvm::Type *llvmType = ConvertTypeForMem(FieldType);
2412  V = EmitBitCastOfLValueToProperType(*this, V, llvmType, Field->getName());
2413
2414  CharUnits Alignment = getContext().getDeclAlign(Field);
2415
2416  // FIXME: It should be impossible to have an LValue without alignment for a
2417  // complete type.
2418  if (!Base.getAlignment().isZero())
2419    Alignment = std::min(Alignment, Base.getAlignment());
2420
2421  return MakeAddrLValue(V, FieldType, Alignment);
2422}
2423
2424LValue CodeGenFunction::EmitCompoundLiteralLValue(const CompoundLiteralExpr *E){
2425  if (E->isFileScope()) {
2426    llvm::Value *GlobalPtr = CGM.GetAddrOfConstantCompoundLiteral(E);
2427    return MakeAddrLValue(GlobalPtr, E->getType());
2428  }
2429  if (E->getType()->isVariablyModifiedType())
2430    // make sure to emit the VLA size.
2431    EmitVariablyModifiedType(E->getType());
2432
2433  llvm::Value *DeclPtr = CreateMemTemp(E->getType(), ".compoundliteral");
2434  const Expr *InitExpr = E->getInitializer();
2435  LValue Result = MakeAddrLValue(DeclPtr, E->getType());
2436
2437  EmitAnyExprToMem(InitExpr, DeclPtr, E->getType().getQualifiers(),
2438                   /*Init*/ true);
2439
2440  return Result;
2441}
2442
2443LValue CodeGenFunction::EmitInitListLValue(const InitListExpr *E) {
2444  if (!E->isGLValue())
2445    // Initializing an aggregate temporary in C++11: T{...}.
2446    return EmitAggExprToLValue(E);
2447
2448  // An lvalue initializer list must be initializing a reference.
2449  assert(E->getNumInits() == 1 && "reference init with multiple values");
2450  return EmitLValue(E->getInit(0));
2451}
2452
2453LValue CodeGenFunction::
2454EmitConditionalOperatorLValue(const AbstractConditionalOperator *expr) {
2455  if (!expr->isGLValue()) {
2456    // ?: here should be an aggregate.
2457    assert((hasAggregateLLVMType(expr->getType()) &&
2458            !expr->getType()->isAnyComplexType()) &&
2459           "Unexpected conditional operator!");
2460    return EmitAggExprToLValue(expr);
2461  }
2462
2463  OpaqueValueMapping binding(*this, expr);
2464
2465  const Expr *condExpr = expr->getCond();
2466  bool CondExprBool;
2467  if (ConstantFoldsToSimpleInteger(condExpr, CondExprBool)) {
2468    const Expr *live = expr->getTrueExpr(), *dead = expr->getFalseExpr();
2469    if (!CondExprBool) std::swap(live, dead);
2470
2471    if (!ContainsLabel(dead))
2472      return EmitLValue(live);
2473  }
2474
2475  llvm::BasicBlock *lhsBlock = createBasicBlock("cond.true");
2476  llvm::BasicBlock *rhsBlock = createBasicBlock("cond.false");
2477  llvm::BasicBlock *contBlock = createBasicBlock("cond.end");
2478
2479  ConditionalEvaluation eval(*this);
2480  EmitBranchOnBoolExpr(condExpr, lhsBlock, rhsBlock);
2481
2482  // Any temporaries created here are conditional.
2483  EmitBlock(lhsBlock);
2484  eval.begin(*this);
2485  LValue lhs = EmitLValue(expr->getTrueExpr());
2486  eval.end(*this);
2487
2488  if (!lhs.isSimple())
2489    return EmitUnsupportedLValue(expr, "conditional operator");
2490
2491  lhsBlock = Builder.GetInsertBlock();
2492  Builder.CreateBr(contBlock);
2493
2494  // Any temporaries created here are conditional.
2495  EmitBlock(rhsBlock);
2496  eval.begin(*this);
2497  LValue rhs = EmitLValue(expr->getFalseExpr());
2498  eval.end(*this);
2499  if (!rhs.isSimple())
2500    return EmitUnsupportedLValue(expr, "conditional operator");
2501  rhsBlock = Builder.GetInsertBlock();
2502
2503  EmitBlock(contBlock);
2504
2505  llvm::PHINode *phi = Builder.CreatePHI(lhs.getAddress()->getType(), 2,
2506                                         "cond-lvalue");
2507  phi->addIncoming(lhs.getAddress(), lhsBlock);
2508  phi->addIncoming(rhs.getAddress(), rhsBlock);
2509  return MakeAddrLValue(phi, expr->getType());
2510}
2511
2512/// EmitCastLValue - Casts are never lvalues unless that cast is to a reference
2513/// type. If the cast is to a reference, we can have the usual lvalue result,
2514/// otherwise if a cast is needed by the code generator in an lvalue context,
2515/// then it must mean that we need the address of an aggregate in order to
2516/// access one of its members.  This can happen for all the reasons that casts
2517/// are permitted with aggregate result, including noop aggregate casts, and
2518/// cast from scalar to union.
2519LValue CodeGenFunction::EmitCastLValue(const CastExpr *E) {
2520  switch (E->getCastKind()) {
2521  case CK_ToVoid:
2522    return EmitUnsupportedLValue(E, "unexpected cast lvalue");
2523
2524  case CK_Dependent:
2525    llvm_unreachable("dependent cast kind in IR gen!");
2526
2527  case CK_BuiltinFnToFnPtr:
2528    llvm_unreachable("builtin functions are handled elsewhere");
2529
2530  // These two casts are currently treated as no-ops, although they could
2531  // potentially be real operations depending on the target's ABI.
2532  case CK_NonAtomicToAtomic:
2533  case CK_AtomicToNonAtomic:
2534
2535  case CK_NoOp:
2536  case CK_LValueToRValue:
2537    if (!E->getSubExpr()->Classify(getContext()).isPRValue()
2538        || E->getType()->isRecordType())
2539      return EmitLValue(E->getSubExpr());
2540    // Fall through to synthesize a temporary.
2541
2542  case CK_BitCast:
2543  case CK_ArrayToPointerDecay:
2544  case CK_FunctionToPointerDecay:
2545  case CK_NullToMemberPointer:
2546  case CK_NullToPointer:
2547  case CK_IntegralToPointer:
2548  case CK_PointerToIntegral:
2549  case CK_PointerToBoolean:
2550  case CK_VectorSplat:
2551  case CK_IntegralCast:
2552  case CK_IntegralToBoolean:
2553  case CK_IntegralToFloating:
2554  case CK_FloatingToIntegral:
2555  case CK_FloatingToBoolean:
2556  case CK_FloatingCast:
2557  case CK_FloatingRealToComplex:
2558  case CK_FloatingComplexToReal:
2559  case CK_FloatingComplexToBoolean:
2560  case CK_FloatingComplexCast:
2561  case CK_FloatingComplexToIntegralComplex:
2562  case CK_IntegralRealToComplex:
2563  case CK_IntegralComplexToReal:
2564  case CK_IntegralComplexToBoolean:
2565  case CK_IntegralComplexCast:
2566  case CK_IntegralComplexToFloatingComplex:
2567  case CK_DerivedToBaseMemberPointer:
2568  case CK_BaseToDerivedMemberPointer:
2569  case CK_MemberPointerToBoolean:
2570  case CK_ReinterpretMemberPointer:
2571  case CK_AnyPointerToBlockPointerCast:
2572  case CK_ARCProduceObject:
2573  case CK_ARCConsumeObject:
2574  case CK_ARCReclaimReturnedObject:
2575  case CK_ARCExtendBlockObject:
2576  case CK_CopyAndAutoreleaseBlockObject: {
2577    // These casts only produce lvalues when we're binding a reference to a
2578    // temporary realized from a (converted) pure rvalue. Emit the expression
2579    // as a value, copy it into a temporary, and return an lvalue referring to
2580    // that temporary.
2581    llvm::Value *V = CreateMemTemp(E->getType(), "ref.temp");
2582    EmitAnyExprToMem(E, V, E->getType().getQualifiers(), false);
2583    return MakeAddrLValue(V, E->getType());
2584  }
2585
2586  case CK_Dynamic: {
2587    LValue LV = EmitLValue(E->getSubExpr());
2588    llvm::Value *V = LV.getAddress();
2589    const CXXDynamicCastExpr *DCE = cast<CXXDynamicCastExpr>(E);
2590    return MakeAddrLValue(EmitDynamicCast(V, DCE), E->getType());
2591  }
2592
2593  case CK_ConstructorConversion:
2594  case CK_UserDefinedConversion:
2595  case CK_CPointerToObjCPointerCast:
2596  case CK_BlockPointerToObjCPointerCast:
2597    return EmitLValue(E->getSubExpr());
2598
2599  case CK_UncheckedDerivedToBase:
2600  case CK_DerivedToBase: {
2601    const RecordType *DerivedClassTy =
2602      E->getSubExpr()->getType()->getAs<RecordType>();
2603    CXXRecordDecl *DerivedClassDecl =
2604      cast<CXXRecordDecl>(DerivedClassTy->getDecl());
2605
2606    LValue LV = EmitLValue(E->getSubExpr());
2607    llvm::Value *This = LV.getAddress();
2608
2609    // Perform the derived-to-base conversion
2610    llvm::Value *Base =
2611      GetAddressOfBaseClass(This, DerivedClassDecl,
2612                            E->path_begin(), E->path_end(),
2613                            /*NullCheckValue=*/false);
2614
2615    return MakeAddrLValue(Base, E->getType());
2616  }
2617  case CK_ToUnion:
2618    return EmitAggExprToLValue(E);
2619  case CK_BaseToDerived: {
2620    const RecordType *DerivedClassTy = E->getType()->getAs<RecordType>();
2621    CXXRecordDecl *DerivedClassDecl =
2622      cast<CXXRecordDecl>(DerivedClassTy->getDecl());
2623
2624    LValue LV = EmitLValue(E->getSubExpr());
2625
2626    // Perform the base-to-derived conversion
2627    llvm::Value *Derived =
2628      GetAddressOfDerivedClass(LV.getAddress(), DerivedClassDecl,
2629                               E->path_begin(), E->path_end(),
2630                               /*NullCheckValue=*/false);
2631
2632    return MakeAddrLValue(Derived, E->getType());
2633  }
2634  case CK_LValueBitCast: {
2635    // This must be a reinterpret_cast (or c-style equivalent).
2636    const ExplicitCastExpr *CE = cast<ExplicitCastExpr>(E);
2637
2638    LValue LV = EmitLValue(E->getSubExpr());
2639    llvm::Value *V = Builder.CreateBitCast(LV.getAddress(),
2640                                           ConvertType(CE->getTypeAsWritten()));
2641    return MakeAddrLValue(V, E->getType());
2642  }
2643  case CK_ObjCObjectLValueCast: {
2644    LValue LV = EmitLValue(E->getSubExpr());
2645    QualType ToType = getContext().getLValueReferenceType(E->getType());
2646    llvm::Value *V = Builder.CreateBitCast(LV.getAddress(),
2647                                           ConvertType(ToType));
2648    return MakeAddrLValue(V, E->getType());
2649  }
2650  case CK_ZeroToOCLEvent:
2651    llvm_unreachable("NULL to OpenCL event lvalue cast is not valid");
2652  }
2653
2654  llvm_unreachable("Unhandled lvalue cast kind?");
2655}
2656
2657LValue CodeGenFunction::EmitNullInitializationLValue(
2658                                              const CXXScalarValueInitExpr *E) {
2659  QualType Ty = E->getType();
2660  LValue LV = MakeAddrLValue(CreateMemTemp(Ty), Ty);
2661  EmitNullInitialization(LV.getAddress(), Ty);
2662  return LV;
2663}
2664
2665LValue CodeGenFunction::EmitOpaqueValueLValue(const OpaqueValueExpr *e) {
2666  assert(OpaqueValueMappingData::shouldBindAsLValue(e));
2667  return getOpaqueLValueMapping(e);
2668}
2669
2670LValue CodeGenFunction::EmitMaterializeTemporaryExpr(
2671                                           const MaterializeTemporaryExpr *E) {
2672  RValue RV = EmitReferenceBindingToExpr(E, /*InitializedDecl=*/0);
2673  return MakeAddrLValue(RV.getScalarVal(), E->getType());
2674}
2675
2676RValue CodeGenFunction::EmitRValueForField(LValue LV,
2677                                           const FieldDecl *FD) {
2678  QualType FT = FD->getType();
2679  LValue FieldLV = EmitLValueForField(LV, FD);
2680  if (FT->isAnyComplexType())
2681    return RValue::getComplex(
2682        LoadComplexFromAddr(FieldLV.getAddress(),
2683                            FieldLV.isVolatileQualified()));
2684  else if (CodeGenFunction::hasAggregateLLVMType(FT))
2685    return FieldLV.asAggregateRValue();
2686
2687  return EmitLoadOfLValue(FieldLV);
2688}
2689
2690//===--------------------------------------------------------------------===//
2691//                             Expression Emission
2692//===--------------------------------------------------------------------===//
2693
2694RValue CodeGenFunction::EmitCallExpr(const CallExpr *E,
2695                                     ReturnValueSlot ReturnValue) {
2696  if (CGDebugInfo *DI = getDebugInfo())
2697    DI->EmitLocation(Builder, E->getLocStart());
2698
2699  // Builtins never have block type.
2700  if (E->getCallee()->getType()->isBlockPointerType())
2701    return EmitBlockCallExpr(E, ReturnValue);
2702
2703  if (const CXXMemberCallExpr *CE = dyn_cast<CXXMemberCallExpr>(E))
2704    return EmitCXXMemberCallExpr(CE, ReturnValue);
2705
2706  if (const CUDAKernelCallExpr *CE = dyn_cast<CUDAKernelCallExpr>(E))
2707    return EmitCUDAKernelCallExpr(CE, ReturnValue);
2708
2709  const Decl *TargetDecl = E->getCalleeDecl();
2710  if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(TargetDecl)) {
2711    if (unsigned builtinID = FD->getBuiltinID())
2712      return EmitBuiltinExpr(FD, builtinID, E);
2713  }
2714
2715  if (const CXXOperatorCallExpr *CE = dyn_cast<CXXOperatorCallExpr>(E))
2716    if (const CXXMethodDecl *MD = dyn_cast_or_null<CXXMethodDecl>(TargetDecl))
2717      return EmitCXXOperatorMemberCallExpr(CE, MD, ReturnValue);
2718
2719  if (const CXXPseudoDestructorExpr *PseudoDtor
2720          = dyn_cast<CXXPseudoDestructorExpr>(E->getCallee()->IgnoreParens())) {
2721    QualType DestroyedType = PseudoDtor->getDestroyedType();
2722    if (getLangOpts().ObjCAutoRefCount &&
2723        DestroyedType->isObjCLifetimeType() &&
2724        (DestroyedType.getObjCLifetime() == Qualifiers::OCL_Strong ||
2725         DestroyedType.getObjCLifetime() == Qualifiers::OCL_Weak)) {
2726      // Automatic Reference Counting:
2727      //   If the pseudo-expression names a retainable object with weak or
2728      //   strong lifetime, the object shall be released.
2729      Expr *BaseExpr = PseudoDtor->getBase();
2730      llvm::Value *BaseValue = NULL;
2731      Qualifiers BaseQuals;
2732
2733      // If this is s.x, emit s as an lvalue. If it is s->x, emit s as a scalar.
2734      if (PseudoDtor->isArrow()) {
2735        BaseValue = EmitScalarExpr(BaseExpr);
2736        const PointerType *PTy = BaseExpr->getType()->getAs<PointerType>();
2737        BaseQuals = PTy->getPointeeType().getQualifiers();
2738      } else {
2739        LValue BaseLV = EmitLValue(BaseExpr);
2740        BaseValue = BaseLV.getAddress();
2741        QualType BaseTy = BaseExpr->getType();
2742        BaseQuals = BaseTy.getQualifiers();
2743      }
2744
2745      switch (PseudoDtor->getDestroyedType().getObjCLifetime()) {
2746      case Qualifiers::OCL_None:
2747      case Qualifiers::OCL_ExplicitNone:
2748      case Qualifiers::OCL_Autoreleasing:
2749        break;
2750
2751      case Qualifiers::OCL_Strong:
2752        EmitARCRelease(Builder.CreateLoad(BaseValue,
2753                          PseudoDtor->getDestroyedType().isVolatileQualified()),
2754                       /*precise*/ true);
2755        break;
2756
2757      case Qualifiers::OCL_Weak:
2758        EmitARCDestroyWeak(BaseValue);
2759        break;
2760      }
2761    } else {
2762      // C++ [expr.pseudo]p1:
2763      //   The result shall only be used as the operand for the function call
2764      //   operator (), and the result of such a call has type void. The only
2765      //   effect is the evaluation of the postfix-expression before the dot or
2766      //   arrow.
2767      EmitScalarExpr(E->getCallee());
2768    }
2769
2770    return RValue::get(0);
2771  }
2772
2773  llvm::Value *Callee = EmitScalarExpr(E->getCallee());
2774  return EmitCall(E->getCallee()->getType(), Callee, ReturnValue,
2775                  E->arg_begin(), E->arg_end(), TargetDecl);
2776}
2777
2778LValue CodeGenFunction::EmitBinaryOperatorLValue(const BinaryOperator *E) {
2779  // Comma expressions just emit their LHS then their RHS as an l-value.
2780  if (E->getOpcode() == BO_Comma) {
2781    EmitIgnoredExpr(E->getLHS());
2782    EnsureInsertPoint();
2783    return EmitLValue(E->getRHS());
2784  }
2785
2786  if (E->getOpcode() == BO_PtrMemD ||
2787      E->getOpcode() == BO_PtrMemI)
2788    return EmitPointerToDataMemberBinaryExpr(E);
2789
2790  assert(E->getOpcode() == BO_Assign && "unexpected binary l-value");
2791
2792  // Note that in all of these cases, __block variables need the RHS
2793  // evaluated first just in case the variable gets moved by the RHS.
2794
2795  if (!hasAggregateLLVMType(E->getType())) {
2796    switch (E->getLHS()->getType().getObjCLifetime()) {
2797    case Qualifiers::OCL_Strong:
2798      return EmitARCStoreStrong(E, /*ignored*/ false).first;
2799
2800    case Qualifiers::OCL_Autoreleasing:
2801      return EmitARCStoreAutoreleasing(E).first;
2802
2803    // No reason to do any of these differently.
2804    case Qualifiers::OCL_None:
2805    case Qualifiers::OCL_ExplicitNone:
2806    case Qualifiers::OCL_Weak:
2807      break;
2808    }
2809
2810    RValue RV = EmitAnyExpr(E->getRHS());
2811    LValue LV = EmitCheckedLValue(E->getLHS(), TCK_Store);
2812    EmitStoreThroughLValue(RV, LV);
2813    return LV;
2814  }
2815
2816  if (E->getType()->isAnyComplexType())
2817    return EmitComplexAssignmentLValue(E);
2818
2819  return EmitAggExprToLValue(E);
2820}
2821
2822LValue CodeGenFunction::EmitCallExprLValue(const CallExpr *E) {
2823  RValue RV = EmitCallExpr(E);
2824
2825  if (!RV.isScalar())
2826    return MakeAddrLValue(RV.getAggregateAddr(), E->getType());
2827
2828  assert(E->getCallReturnType()->isReferenceType() &&
2829         "Can't have a scalar return unless the return type is a "
2830         "reference type!");
2831
2832  return MakeAddrLValue(RV.getScalarVal(), E->getType());
2833}
2834
2835LValue CodeGenFunction::EmitVAArgExprLValue(const VAArgExpr *E) {
2836  // FIXME: This shouldn't require another copy.
2837  return EmitAggExprToLValue(E);
2838}
2839
2840LValue CodeGenFunction::EmitCXXConstructLValue(const CXXConstructExpr *E) {
2841  assert(E->getType()->getAsCXXRecordDecl()->hasTrivialDestructor()
2842         && "binding l-value to type which needs a temporary");
2843  AggValueSlot Slot = CreateAggTemp(E->getType());
2844  EmitCXXConstructExpr(E, Slot);
2845  return MakeAddrLValue(Slot.getAddr(), E->getType());
2846}
2847
2848LValue
2849CodeGenFunction::EmitCXXTypeidLValue(const CXXTypeidExpr *E) {
2850  return MakeAddrLValue(EmitCXXTypeidExpr(E), E->getType());
2851}
2852
2853llvm::Value *CodeGenFunction::EmitCXXUuidofExpr(const CXXUuidofExpr *E) {
2854  return CGM.GetAddrOfUuidDescriptor(E);
2855}
2856
2857LValue CodeGenFunction::EmitCXXUuidofLValue(const CXXUuidofExpr *E) {
2858  return MakeAddrLValue(EmitCXXUuidofExpr(E), E->getType());
2859}
2860
2861LValue
2862CodeGenFunction::EmitCXXBindTemporaryLValue(const CXXBindTemporaryExpr *E) {
2863  AggValueSlot Slot = CreateAggTemp(E->getType(), "temp.lvalue");
2864  Slot.setExternallyDestructed();
2865  EmitAggExpr(E->getSubExpr(), Slot);
2866  EmitCXXTemporary(E->getTemporary(), E->getType(), Slot.getAddr());
2867  return MakeAddrLValue(Slot.getAddr(), E->getType());
2868}
2869
2870LValue
2871CodeGenFunction::EmitLambdaLValue(const LambdaExpr *E) {
2872  AggValueSlot Slot = CreateAggTemp(E->getType(), "temp.lvalue");
2873  EmitLambdaExpr(E, Slot);
2874  return MakeAddrLValue(Slot.getAddr(), E->getType());
2875}
2876
2877LValue CodeGenFunction::EmitObjCMessageExprLValue(const ObjCMessageExpr *E) {
2878  RValue RV = EmitObjCMessageExpr(E);
2879
2880  if (!RV.isScalar())
2881    return MakeAddrLValue(RV.getAggregateAddr(), E->getType());
2882
2883  assert(E->getMethodDecl()->getResultType()->isReferenceType() &&
2884         "Can't have a scalar return unless the return type is a "
2885         "reference type!");
2886
2887  return MakeAddrLValue(RV.getScalarVal(), E->getType());
2888}
2889
2890LValue CodeGenFunction::EmitObjCSelectorLValue(const ObjCSelectorExpr *E) {
2891  llvm::Value *V =
2892    CGM.getObjCRuntime().GetSelector(Builder, E->getSelector(), true);
2893  return MakeAddrLValue(V, E->getType());
2894}
2895
2896llvm::Value *CodeGenFunction::EmitIvarOffset(const ObjCInterfaceDecl *Interface,
2897                                             const ObjCIvarDecl *Ivar) {
2898  return CGM.getObjCRuntime().EmitIvarOffset(*this, Interface, Ivar);
2899}
2900
2901LValue CodeGenFunction::EmitLValueForIvar(QualType ObjectTy,
2902                                          llvm::Value *BaseValue,
2903                                          const ObjCIvarDecl *Ivar,
2904                                          unsigned CVRQualifiers) {
2905  return CGM.getObjCRuntime().EmitObjCValueForIvar(*this, ObjectTy, BaseValue,
2906                                                   Ivar, CVRQualifiers);
2907}
2908
2909LValue CodeGenFunction::EmitObjCIvarRefLValue(const ObjCIvarRefExpr *E) {
2910  // FIXME: A lot of the code below could be shared with EmitMemberExpr.
2911  llvm::Value *BaseValue = 0;
2912  const Expr *BaseExpr = E->getBase();
2913  Qualifiers BaseQuals;
2914  QualType ObjectTy;
2915  if (E->isArrow()) {
2916    BaseValue = EmitScalarExpr(BaseExpr);
2917    ObjectTy = BaseExpr->getType()->getPointeeType();
2918    BaseQuals = ObjectTy.getQualifiers();
2919  } else {
2920    LValue BaseLV = EmitLValue(BaseExpr);
2921    // FIXME: this isn't right for bitfields.
2922    BaseValue = BaseLV.getAddress();
2923    ObjectTy = BaseExpr->getType();
2924    BaseQuals = ObjectTy.getQualifiers();
2925  }
2926
2927  LValue LV =
2928    EmitLValueForIvar(ObjectTy, BaseValue, E->getDecl(),
2929                      BaseQuals.getCVRQualifiers());
2930  setObjCGCLValueClass(getContext(), E, LV);
2931  return LV;
2932}
2933
2934LValue CodeGenFunction::EmitStmtExprLValue(const StmtExpr *E) {
2935  // Can only get l-value for message expression returning aggregate type
2936  RValue RV = EmitAnyExprToTemp(E);
2937  return MakeAddrLValue(RV.getAggregateAddr(), E->getType());
2938}
2939
2940RValue CodeGenFunction::EmitCall(QualType CalleeType, llvm::Value *Callee,
2941                                 ReturnValueSlot ReturnValue,
2942                                 CallExpr::const_arg_iterator ArgBeg,
2943                                 CallExpr::const_arg_iterator ArgEnd,
2944                                 const Decl *TargetDecl) {
2945  // Get the actual function type. The callee type will always be a pointer to
2946  // function type or a block pointer type.
2947  assert(CalleeType->isFunctionPointerType() &&
2948         "Call must have function pointer type!");
2949
2950  CalleeType = getContext().getCanonicalType(CalleeType);
2951
2952  const FunctionType *FnType
2953    = cast<FunctionType>(cast<PointerType>(CalleeType)->getPointeeType());
2954
2955  CallArgList Args;
2956  EmitCallArgs(Args, dyn_cast<FunctionProtoType>(FnType), ArgBeg, ArgEnd);
2957
2958  const CGFunctionInfo &FnInfo =
2959    CGM.getTypes().arrangeFreeFunctionCall(Args, FnType);
2960
2961  // C99 6.5.2.2p6:
2962  //   If the expression that denotes the called function has a type
2963  //   that does not include a prototype, [the default argument
2964  //   promotions are performed]. If the number of arguments does not
2965  //   equal the number of parameters, the behavior is undefined. If
2966  //   the function is defined with a type that includes a prototype,
2967  //   and either the prototype ends with an ellipsis (, ...) or the
2968  //   types of the arguments after promotion are not compatible with
2969  //   the types of the parameters, the behavior is undefined. If the
2970  //   function is defined with a type that does not include a
2971  //   prototype, and the types of the arguments after promotion are
2972  //   not compatible with those of the parameters after promotion,
2973  //   the behavior is undefined [except in some trivial cases].
2974  // That is, in the general case, we should assume that a call
2975  // through an unprototyped function type works like a *non-variadic*
2976  // call.  The way we make this work is to cast to the exact type
2977  // of the promoted arguments.
2978  if (isa<FunctionNoProtoType>(FnType)) {
2979    llvm::Type *CalleeTy = getTypes().GetFunctionType(FnInfo);
2980    CalleeTy = CalleeTy->getPointerTo();
2981    Callee = Builder.CreateBitCast(Callee, CalleeTy, "callee.knr.cast");
2982  }
2983
2984  return EmitCall(FnInfo, Callee, ReturnValue, Args, TargetDecl);
2985}
2986
2987LValue CodeGenFunction::
2988EmitPointerToDataMemberBinaryExpr(const BinaryOperator *E) {
2989  llvm::Value *BaseV;
2990  if (E->getOpcode() == BO_PtrMemI)
2991    BaseV = EmitScalarExpr(E->getLHS());
2992  else
2993    BaseV = EmitLValue(E->getLHS()).getAddress();
2994
2995  llvm::Value *OffsetV = EmitScalarExpr(E->getRHS());
2996
2997  const MemberPointerType *MPT
2998    = E->getRHS()->getType()->getAs<MemberPointerType>();
2999
3000  llvm::Value *AddV =
3001    CGM.getCXXABI().EmitMemberDataPointerAddress(*this, BaseV, OffsetV, MPT);
3002
3003  return MakeAddrLValue(AddV, MPT->getPointeeType());
3004}
3005
3006static void
3007EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *E, llvm::Value *Dest,
3008             llvm::Value *Ptr, llvm::Value *Val1, llvm::Value *Val2,
3009             uint64_t Size, unsigned Align, llvm::AtomicOrdering Order) {
3010  llvm::AtomicRMWInst::BinOp Op = llvm::AtomicRMWInst::Add;
3011  llvm::Instruction::BinaryOps PostOp = (llvm::Instruction::BinaryOps)0;
3012
3013  switch (E->getOp()) {
3014  case AtomicExpr::AO__c11_atomic_init:
3015    llvm_unreachable("Already handled!");
3016
3017  case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
3018  case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
3019  case AtomicExpr::AO__atomic_compare_exchange:
3020  case AtomicExpr::AO__atomic_compare_exchange_n: {
3021    // Note that cmpxchg only supports specifying one ordering and
3022    // doesn't support weak cmpxchg, at least at the moment.
3023    llvm::LoadInst *LoadVal1 = CGF.Builder.CreateLoad(Val1);
3024    LoadVal1->setAlignment(Align);
3025    llvm::LoadInst *LoadVal2 = CGF.Builder.CreateLoad(Val2);
3026    LoadVal2->setAlignment(Align);
3027    llvm::AtomicCmpXchgInst *CXI =
3028        CGF.Builder.CreateAtomicCmpXchg(Ptr, LoadVal1, LoadVal2, Order);
3029    CXI->setVolatile(E->isVolatile());
3030    llvm::StoreInst *StoreVal1 = CGF.Builder.CreateStore(CXI, Val1);
3031    StoreVal1->setAlignment(Align);
3032    llvm::Value *Cmp = CGF.Builder.CreateICmpEQ(CXI, LoadVal1);
3033    CGF.EmitStoreOfScalar(Cmp, CGF.MakeAddrLValue(Dest, E->getType()));
3034    return;
3035  }
3036
3037  case AtomicExpr::AO__c11_atomic_load:
3038  case AtomicExpr::AO__atomic_load_n:
3039  case AtomicExpr::AO__atomic_load: {
3040    llvm::LoadInst *Load = CGF.Builder.CreateLoad(Ptr);
3041    Load->setAtomic(Order);
3042    Load->setAlignment(Size);
3043    Load->setVolatile(E->isVolatile());
3044    llvm::StoreInst *StoreDest = CGF.Builder.CreateStore(Load, Dest);
3045    StoreDest->setAlignment(Align);
3046    return;
3047  }
3048
3049  case AtomicExpr::AO__c11_atomic_store:
3050  case AtomicExpr::AO__atomic_store:
3051  case AtomicExpr::AO__atomic_store_n: {
3052    assert(!Dest && "Store does not return a value");
3053    llvm::LoadInst *LoadVal1 = CGF.Builder.CreateLoad(Val1);
3054    LoadVal1->setAlignment(Align);
3055    llvm::StoreInst *Store = CGF.Builder.CreateStore(LoadVal1, Ptr);
3056    Store->setAtomic(Order);
3057    Store->setAlignment(Size);
3058    Store->setVolatile(E->isVolatile());
3059    return;
3060  }
3061
3062  case AtomicExpr::AO__c11_atomic_exchange:
3063  case AtomicExpr::AO__atomic_exchange_n:
3064  case AtomicExpr::AO__atomic_exchange:
3065    Op = llvm::AtomicRMWInst::Xchg;
3066    break;
3067
3068  case AtomicExpr::AO__atomic_add_fetch:
3069    PostOp = llvm::Instruction::Add;
3070    // Fall through.
3071  case AtomicExpr::AO__c11_atomic_fetch_add:
3072  case AtomicExpr::AO__atomic_fetch_add:
3073    Op = llvm::AtomicRMWInst::Add;
3074    break;
3075
3076  case AtomicExpr::AO__atomic_sub_fetch:
3077    PostOp = llvm::Instruction::Sub;
3078    // Fall through.
3079  case AtomicExpr::AO__c11_atomic_fetch_sub:
3080  case AtomicExpr::AO__atomic_fetch_sub:
3081    Op = llvm::AtomicRMWInst::Sub;
3082    break;
3083
3084  case AtomicExpr::AO__atomic_and_fetch:
3085    PostOp = llvm::Instruction::And;
3086    // Fall through.
3087  case AtomicExpr::AO__c11_atomic_fetch_and:
3088  case AtomicExpr::AO__atomic_fetch_and:
3089    Op = llvm::AtomicRMWInst::And;
3090    break;
3091
3092  case AtomicExpr::AO__atomic_or_fetch:
3093    PostOp = llvm::Instruction::Or;
3094    // Fall through.
3095  case AtomicExpr::AO__c11_atomic_fetch_or:
3096  case AtomicExpr::AO__atomic_fetch_or:
3097    Op = llvm::AtomicRMWInst::Or;
3098    break;
3099
3100  case AtomicExpr::AO__atomic_xor_fetch:
3101    PostOp = llvm::Instruction::Xor;
3102    // Fall through.
3103  case AtomicExpr::AO__c11_atomic_fetch_xor:
3104  case AtomicExpr::AO__atomic_fetch_xor:
3105    Op = llvm::AtomicRMWInst::Xor;
3106    break;
3107
3108  case AtomicExpr::AO__atomic_nand_fetch:
3109    PostOp = llvm::Instruction::And;
3110    // Fall through.
3111  case AtomicExpr::AO__atomic_fetch_nand:
3112    Op = llvm::AtomicRMWInst::Nand;
3113    break;
3114  }
3115
3116  llvm::LoadInst *LoadVal1 = CGF.Builder.CreateLoad(Val1);
3117  LoadVal1->setAlignment(Align);
3118  llvm::AtomicRMWInst *RMWI =
3119      CGF.Builder.CreateAtomicRMW(Op, Ptr, LoadVal1, Order);
3120  RMWI->setVolatile(E->isVolatile());
3121
3122  // For __atomic_*_fetch operations, perform the operation again to
3123  // determine the value which was written.
3124  llvm::Value *Result = RMWI;
3125  if (PostOp)
3126    Result = CGF.Builder.CreateBinOp(PostOp, RMWI, LoadVal1);
3127  if (E->getOp() == AtomicExpr::AO__atomic_nand_fetch)
3128    Result = CGF.Builder.CreateNot(Result);
3129  llvm::StoreInst *StoreDest = CGF.Builder.CreateStore(Result, Dest);
3130  StoreDest->setAlignment(Align);
3131}
3132
3133// This function emits any expression (scalar, complex, or aggregate)
3134// into a temporary alloca.
3135static llvm::Value *
3136EmitValToTemp(CodeGenFunction &CGF, Expr *E) {
3137  llvm::Value *DeclPtr = CGF.CreateMemTemp(E->getType(), ".atomictmp");
3138  CGF.EmitAnyExprToMem(E, DeclPtr, E->getType().getQualifiers(),
3139                       /*Init*/ true);
3140  return DeclPtr;
3141}
3142
3143static RValue ConvertTempToRValue(CodeGenFunction &CGF, QualType Ty,
3144                                  llvm::Value *Dest) {
3145  if (Ty->isAnyComplexType())
3146    return RValue::getComplex(CGF.LoadComplexFromAddr(Dest, false));
3147  if (CGF.hasAggregateLLVMType(Ty))
3148    return RValue::getAggregate(Dest);
3149  return RValue::get(CGF.EmitLoadOfScalar(CGF.MakeAddrLValue(Dest, Ty)));
3150}
3151
3152RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E, llvm::Value *Dest) {
3153  QualType AtomicTy = E->getPtr()->getType()->getPointeeType();
3154  QualType MemTy = AtomicTy;
3155  if (const AtomicType *AT = AtomicTy->getAs<AtomicType>())
3156    MemTy = AT->getValueType();
3157  CharUnits sizeChars = getContext().getTypeSizeInChars(AtomicTy);
3158  uint64_t Size = sizeChars.getQuantity();
3159  CharUnits alignChars = getContext().getTypeAlignInChars(AtomicTy);
3160  unsigned Align = alignChars.getQuantity();
3161  unsigned MaxInlineWidthInBits =
3162    getContext().getTargetInfo().getMaxAtomicInlineWidth();
3163  bool UseLibcall = (Size != Align ||
3164                     getContext().toBits(sizeChars) > MaxInlineWidthInBits);
3165
3166  llvm::Value *Ptr, *Order, *OrderFail = 0, *Val1 = 0, *Val2 = 0;
3167  Ptr = EmitScalarExpr(E->getPtr());
3168
3169  if (E->getOp() == AtomicExpr::AO__c11_atomic_init) {
3170    assert(!Dest && "Init does not return a value");
3171    if (!hasAggregateLLVMType(E->getVal1()->getType())) {
3172      QualType PointeeType
3173        = E->getPtr()->getType()->getAs<PointerType>()->getPointeeType();
3174      EmitScalarInit(EmitScalarExpr(E->getVal1()),
3175                     LValue::MakeAddr(Ptr, PointeeType, alignChars,
3176                                      getContext()));
3177    } else if (E->getType()->isAnyComplexType()) {
3178      EmitComplexExprIntoAddr(E->getVal1(), Ptr, E->isVolatile());
3179    } else {
3180      AggValueSlot Slot = AggValueSlot::forAddr(Ptr, alignChars,
3181                                        AtomicTy.getQualifiers(),
3182                                        AggValueSlot::IsNotDestructed,
3183                                        AggValueSlot::DoesNotNeedGCBarriers,
3184                                        AggValueSlot::IsNotAliased);
3185      EmitAggExpr(E->getVal1(), Slot);
3186    }
3187    return RValue::get(0);
3188  }
3189
3190  Order = EmitScalarExpr(E->getOrder());
3191
3192  switch (E->getOp()) {
3193  case AtomicExpr::AO__c11_atomic_init:
3194    llvm_unreachable("Already handled!");
3195
3196  case AtomicExpr::AO__c11_atomic_load:
3197  case AtomicExpr::AO__atomic_load_n:
3198    break;
3199
3200  case AtomicExpr::AO__atomic_load:
3201    Dest = EmitScalarExpr(E->getVal1());
3202    break;
3203
3204  case AtomicExpr::AO__atomic_store:
3205    Val1 = EmitScalarExpr(E->getVal1());
3206    break;
3207
3208  case AtomicExpr::AO__atomic_exchange:
3209    Val1 = EmitScalarExpr(E->getVal1());
3210    Dest = EmitScalarExpr(E->getVal2());
3211    break;
3212
3213  case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
3214  case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
3215  case AtomicExpr::AO__atomic_compare_exchange_n:
3216  case AtomicExpr::AO__atomic_compare_exchange:
3217    Val1 = EmitScalarExpr(E->getVal1());
3218    if (E->getOp() == AtomicExpr::AO__atomic_compare_exchange)
3219      Val2 = EmitScalarExpr(E->getVal2());
3220    else
3221      Val2 = EmitValToTemp(*this, E->getVal2());
3222    OrderFail = EmitScalarExpr(E->getOrderFail());
3223    // Evaluate and discard the 'weak' argument.
3224    if (E->getNumSubExprs() == 6)
3225      EmitScalarExpr(E->getWeak());
3226    break;
3227
3228  case AtomicExpr::AO__c11_atomic_fetch_add:
3229  case AtomicExpr::AO__c11_atomic_fetch_sub:
3230    if (MemTy->isPointerType()) {
3231      // For pointer arithmetic, we're required to do a bit of math:
3232      // adding 1 to an int* is not the same as adding 1 to a uintptr_t.
3233      // ... but only for the C11 builtins. The GNU builtins expect the
3234      // user to multiply by sizeof(T).
3235      QualType Val1Ty = E->getVal1()->getType();
3236      llvm::Value *Val1Scalar = EmitScalarExpr(E->getVal1());
3237      CharUnits PointeeIncAmt =
3238          getContext().getTypeSizeInChars(MemTy->getPointeeType());
3239      Val1Scalar = Builder.CreateMul(Val1Scalar, CGM.getSize(PointeeIncAmt));
3240      Val1 = CreateMemTemp(Val1Ty, ".atomictmp");
3241      EmitStoreOfScalar(Val1Scalar, MakeAddrLValue(Val1, Val1Ty));
3242      break;
3243    }
3244    // Fall through.
3245  case AtomicExpr::AO__atomic_fetch_add:
3246  case AtomicExpr::AO__atomic_fetch_sub:
3247  case AtomicExpr::AO__atomic_add_fetch:
3248  case AtomicExpr::AO__atomic_sub_fetch:
3249  case AtomicExpr::AO__c11_atomic_store:
3250  case AtomicExpr::AO__c11_atomic_exchange:
3251  case AtomicExpr::AO__atomic_store_n:
3252  case AtomicExpr::AO__atomic_exchange_n:
3253  case AtomicExpr::AO__c11_atomic_fetch_and:
3254  case AtomicExpr::AO__c11_atomic_fetch_or:
3255  case AtomicExpr::AO__c11_atomic_fetch_xor:
3256  case AtomicExpr::AO__atomic_fetch_and:
3257  case AtomicExpr::AO__atomic_fetch_or:
3258  case AtomicExpr::AO__atomic_fetch_xor:
3259  case AtomicExpr::AO__atomic_fetch_nand:
3260  case AtomicExpr::AO__atomic_and_fetch:
3261  case AtomicExpr::AO__atomic_or_fetch:
3262  case AtomicExpr::AO__atomic_xor_fetch:
3263  case AtomicExpr::AO__atomic_nand_fetch:
3264    Val1 = EmitValToTemp(*this, E->getVal1());
3265    break;
3266  }
3267
3268  if (!E->getType()->isVoidType() && !Dest)
3269    Dest = CreateMemTemp(E->getType(), ".atomicdst");
3270
3271  // Use a library call.  See: http://gcc.gnu.org/wiki/Atomic/GCCMM/LIbrary .
3272  if (UseLibcall) {
3273
3274    SmallVector<QualType, 5> Params;
3275    CallArgList Args;
3276    // Size is always the first parameter
3277    Args.add(RValue::get(llvm::ConstantInt::get(SizeTy, Size)),
3278             getContext().getSizeType());
3279    // Atomic address is always the second parameter
3280    Args.add(RValue::get(EmitCastToVoidPtr(Ptr)),
3281             getContext().VoidPtrTy);
3282
3283    const char* LibCallName;
3284    QualType RetTy = getContext().VoidTy;
3285    switch (E->getOp()) {
3286    // There is only one libcall for compare an exchange, because there is no
3287    // optimisation benefit possible from a libcall version of a weak compare
3288    // and exchange.
3289    // bool __atomic_compare_exchange(size_t size, void *obj, void *expected,
3290    //                                void *desired, int success, int failure)
3291    case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
3292    case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
3293    case AtomicExpr::AO__atomic_compare_exchange:
3294    case AtomicExpr::AO__atomic_compare_exchange_n:
3295      LibCallName = "__atomic_compare_exchange";
3296      RetTy = getContext().BoolTy;
3297      Args.add(RValue::get(EmitCastToVoidPtr(Val1)),
3298               getContext().VoidPtrTy);
3299      Args.add(RValue::get(EmitCastToVoidPtr(Val2)),
3300               getContext().VoidPtrTy);
3301      Args.add(RValue::get(Order),
3302               getContext().IntTy);
3303      Order = OrderFail;
3304      break;
3305    // void __atomic_exchange(size_t size, void *mem, void *val, void *return,
3306    //                        int order)
3307    case AtomicExpr::AO__c11_atomic_exchange:
3308    case AtomicExpr::AO__atomic_exchange_n:
3309    case AtomicExpr::AO__atomic_exchange:
3310      LibCallName = "__atomic_exchange";
3311      Args.add(RValue::get(EmitCastToVoidPtr(Val1)),
3312               getContext().VoidPtrTy);
3313      Args.add(RValue::get(EmitCastToVoidPtr(Dest)),
3314               getContext().VoidPtrTy);
3315      break;
3316    // void __atomic_store(size_t size, void *mem, void *val, int order)
3317    case AtomicExpr::AO__c11_atomic_store:
3318    case AtomicExpr::AO__atomic_store:
3319    case AtomicExpr::AO__atomic_store_n:
3320      LibCallName = "__atomic_store";
3321      Args.add(RValue::get(EmitCastToVoidPtr(Val1)),
3322               getContext().VoidPtrTy);
3323      break;
3324    // void __atomic_load(size_t size, void *mem, void *return, int order)
3325    case AtomicExpr::AO__c11_atomic_load:
3326    case AtomicExpr::AO__atomic_load:
3327    case AtomicExpr::AO__atomic_load_n:
3328      LibCallName = "__atomic_load";
3329      Args.add(RValue::get(EmitCastToVoidPtr(Dest)),
3330               getContext().VoidPtrTy);
3331      break;
3332#if 0
3333    // These are only defined for 1-16 byte integers.  It is not clear what
3334    // their semantics would be on anything else...
3335    case AtomicExpr::Add:   LibCallName = "__atomic_fetch_add_generic"; break;
3336    case AtomicExpr::Sub:   LibCallName = "__atomic_fetch_sub_generic"; break;
3337    case AtomicExpr::And:   LibCallName = "__atomic_fetch_and_generic"; break;
3338    case AtomicExpr::Or:    LibCallName = "__atomic_fetch_or_generic"; break;
3339    case AtomicExpr::Xor:   LibCallName = "__atomic_fetch_xor_generic"; break;
3340#endif
3341    default: return EmitUnsupportedRValue(E, "atomic library call");
3342    }
3343    // order is always the last parameter
3344    Args.add(RValue::get(Order),
3345             getContext().IntTy);
3346
3347    const CGFunctionInfo &FuncInfo =
3348        CGM.getTypes().arrangeFreeFunctionCall(RetTy, Args,
3349            FunctionType::ExtInfo(), RequiredArgs::All);
3350    llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(FuncInfo);
3351    llvm::Constant *Func = CGM.CreateRuntimeFunction(FTy, LibCallName);
3352    RValue Res = EmitCall(FuncInfo, Func, ReturnValueSlot(), Args);
3353    if (E->isCmpXChg())
3354      return Res;
3355    if (E->getType()->isVoidType())
3356      return RValue::get(0);
3357    return ConvertTempToRValue(*this, E->getType(), Dest);
3358  }
3359
3360  bool IsStore = E->getOp() == AtomicExpr::AO__c11_atomic_store ||
3361                 E->getOp() == AtomicExpr::AO__atomic_store ||
3362                 E->getOp() == AtomicExpr::AO__atomic_store_n;
3363  bool IsLoad = E->getOp() == AtomicExpr::AO__c11_atomic_load ||
3364                E->getOp() == AtomicExpr::AO__atomic_load ||
3365                E->getOp() == AtomicExpr::AO__atomic_load_n;
3366
3367  llvm::Type *IPtrTy =
3368      llvm::IntegerType::get(getLLVMContext(), Size * 8)->getPointerTo();
3369  llvm::Value *OrigDest = Dest;
3370  Ptr = Builder.CreateBitCast(Ptr, IPtrTy);
3371  if (Val1) Val1 = Builder.CreateBitCast(Val1, IPtrTy);
3372  if (Val2) Val2 = Builder.CreateBitCast(Val2, IPtrTy);
3373  if (Dest && !E->isCmpXChg()) Dest = Builder.CreateBitCast(Dest, IPtrTy);
3374
3375  if (isa<llvm::ConstantInt>(Order)) {
3376    int ord = cast<llvm::ConstantInt>(Order)->getZExtValue();
3377    switch (ord) {
3378    case 0:  // memory_order_relaxed
3379      EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, Size, Align,
3380                   llvm::Monotonic);
3381      break;
3382    case 1:  // memory_order_consume
3383    case 2:  // memory_order_acquire
3384      if (IsStore)
3385        break; // Avoid crashing on code with undefined behavior
3386      EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, Size, Align,
3387                   llvm::Acquire);
3388      break;
3389    case 3:  // memory_order_release
3390      if (IsLoad)
3391        break; // Avoid crashing on code with undefined behavior
3392      EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, Size, Align,
3393                   llvm::Release);
3394      break;
3395    case 4:  // memory_order_acq_rel
3396      if (IsLoad || IsStore)
3397        break; // Avoid crashing on code with undefined behavior
3398      EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, Size, Align,
3399                   llvm::AcquireRelease);
3400      break;
3401    case 5:  // memory_order_seq_cst
3402      EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, Size, Align,
3403                   llvm::SequentiallyConsistent);
3404      break;
3405    default: // invalid order
3406      // We should not ever get here normally, but it's hard to
3407      // enforce that in general.
3408      break;
3409    }
3410    if (E->getType()->isVoidType())
3411      return RValue::get(0);
3412    return ConvertTempToRValue(*this, E->getType(), OrigDest);
3413  }
3414
3415  // Long case, when Order isn't obviously constant.
3416
3417  // Create all the relevant BB's
3418  llvm::BasicBlock *MonotonicBB = 0, *AcquireBB = 0, *ReleaseBB = 0,
3419                   *AcqRelBB = 0, *SeqCstBB = 0;
3420  MonotonicBB = createBasicBlock("monotonic", CurFn);
3421  if (!IsStore)
3422    AcquireBB = createBasicBlock("acquire", CurFn);
3423  if (!IsLoad)
3424    ReleaseBB = createBasicBlock("release", CurFn);
3425  if (!IsLoad && !IsStore)
3426    AcqRelBB = createBasicBlock("acqrel", CurFn);
3427  SeqCstBB = createBasicBlock("seqcst", CurFn);
3428  llvm::BasicBlock *ContBB = createBasicBlock("atomic.continue", CurFn);
3429
3430  // Create the switch for the split
3431  // MonotonicBB is arbitrarily chosen as the default case; in practice, this
3432  // doesn't matter unless someone is crazy enough to use something that
3433  // doesn't fold to a constant for the ordering.
3434  Order = Builder.CreateIntCast(Order, Builder.getInt32Ty(), false);
3435  llvm::SwitchInst *SI = Builder.CreateSwitch(Order, MonotonicBB);
3436
3437  // Emit all the different atomics
3438  Builder.SetInsertPoint(MonotonicBB);
3439  EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, Size, Align,
3440               llvm::Monotonic);
3441  Builder.CreateBr(ContBB);
3442  if (!IsStore) {
3443    Builder.SetInsertPoint(AcquireBB);
3444    EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, Size, Align,
3445                 llvm::Acquire);
3446    Builder.CreateBr(ContBB);
3447    SI->addCase(Builder.getInt32(1), AcquireBB);
3448    SI->addCase(Builder.getInt32(2), AcquireBB);
3449  }
3450  if (!IsLoad) {
3451    Builder.SetInsertPoint(ReleaseBB);
3452    EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, Size, Align,
3453                 llvm::Release);
3454    Builder.CreateBr(ContBB);
3455    SI->addCase(Builder.getInt32(3), ReleaseBB);
3456  }
3457  if (!IsLoad && !IsStore) {
3458    Builder.SetInsertPoint(AcqRelBB);
3459    EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, Size, Align,
3460                 llvm::AcquireRelease);
3461    Builder.CreateBr(ContBB);
3462    SI->addCase(Builder.getInt32(4), AcqRelBB);
3463  }
3464  Builder.SetInsertPoint(SeqCstBB);
3465  EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, Size, Align,
3466               llvm::SequentiallyConsistent);
3467  Builder.CreateBr(ContBB);
3468  SI->addCase(Builder.getInt32(5), SeqCstBB);
3469
3470  // Cleanup and return
3471  Builder.SetInsertPoint(ContBB);
3472  if (E->getType()->isVoidType())
3473    return RValue::get(0);
3474  return ConvertTempToRValue(*this, E->getType(), OrigDest);
3475}
3476
3477void CodeGenFunction::SetFPAccuracy(llvm::Value *Val, float Accuracy) {
3478  assert(Val->getType()->isFPOrFPVectorTy());
3479  if (Accuracy == 0.0 || !isa<llvm::Instruction>(Val))
3480    return;
3481
3482  llvm::MDBuilder MDHelper(getLLVMContext());
3483  llvm::MDNode *Node = MDHelper.createFPMath(Accuracy);
3484
3485  cast<llvm::Instruction>(Val)->setMetadata(llvm::LLVMContext::MD_fpmath, Node);
3486}
3487
3488namespace {
3489  struct LValueOrRValue {
3490    LValue LV;
3491    RValue RV;
3492  };
3493}
3494
3495static LValueOrRValue emitPseudoObjectExpr(CodeGenFunction &CGF,
3496                                           const PseudoObjectExpr *E,
3497                                           bool forLValue,
3498                                           AggValueSlot slot) {
3499  SmallVector<CodeGenFunction::OpaqueValueMappingData, 4> opaques;
3500
3501  // Find the result expression, if any.
3502  const Expr *resultExpr = E->getResultExpr();
3503  LValueOrRValue result;
3504
3505  for (PseudoObjectExpr::const_semantics_iterator
3506         i = E->semantics_begin(), e = E->semantics_end(); i != e; ++i) {
3507    const Expr *semantic = *i;
3508
3509    // If this semantic expression is an opaque value, bind it
3510    // to the result of its source expression.
3511    if (const OpaqueValueExpr *ov = dyn_cast<OpaqueValueExpr>(semantic)) {
3512
3513      // If this is the result expression, we may need to evaluate
3514      // directly into the slot.
3515      typedef CodeGenFunction::OpaqueValueMappingData OVMA;
3516      OVMA opaqueData;
3517      if (ov == resultExpr && ov->isRValue() && !forLValue &&
3518          CodeGenFunction::hasAggregateLLVMType(ov->getType()) &&
3519          !ov->getType()->isAnyComplexType()) {
3520        CGF.EmitAggExpr(ov->getSourceExpr(), slot);
3521
3522        LValue LV = CGF.MakeAddrLValue(slot.getAddr(), ov->getType());
3523        opaqueData = OVMA::bind(CGF, ov, LV);
3524        result.RV = slot.asRValue();
3525
3526      // Otherwise, emit as normal.
3527      } else {
3528        opaqueData = OVMA::bind(CGF, ov, ov->getSourceExpr());
3529
3530        // If this is the result, also evaluate the result now.
3531        if (ov == resultExpr) {
3532          if (forLValue)
3533            result.LV = CGF.EmitLValue(ov);
3534          else
3535            result.RV = CGF.EmitAnyExpr(ov, slot);
3536        }
3537      }
3538
3539      opaques.push_back(opaqueData);
3540
3541    // Otherwise, if the expression is the result, evaluate it
3542    // and remember the result.
3543    } else if (semantic == resultExpr) {
3544      if (forLValue)
3545        result.LV = CGF.EmitLValue(semantic);
3546      else
3547        result.RV = CGF.EmitAnyExpr(semantic, slot);
3548
3549    // Otherwise, evaluate the expression in an ignored context.
3550    } else {
3551      CGF.EmitIgnoredExpr(semantic);
3552    }
3553  }
3554
3555  // Unbind all the opaques now.
3556  for (unsigned i = 0, e = opaques.size(); i != e; ++i)
3557    opaques[i].unbind(CGF);
3558
3559  return result;
3560}
3561
3562RValue CodeGenFunction::EmitPseudoObjectRValue(const PseudoObjectExpr *E,
3563                                               AggValueSlot slot) {
3564  return emitPseudoObjectExpr(*this, E, false, slot).RV;
3565}
3566
3567LValue CodeGenFunction::EmitPseudoObjectLValue(const PseudoObjectExpr *E) {
3568  return emitPseudoObjectExpr(*this, E, true, AggValueSlot::ignored()).LV;
3569}
3570