CGExpr.cpp revision 28e06354b5d4dd539be6781e388ff27c0dffd807 
1//===--- CGExpr.cpp - Emit LLVM Code from Expressions ---------------------===//
2//
3//                     The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10// This contains code to emit Expr nodes as LLVM code.
11//
12//===----------------------------------------------------------------------===//
13
14#include "CodeGenFunction.h"
15#include "CodeGenModule.h"
16#include "CGCall.h"
17#include "CGCXXABI.h"
18#include "CGDebugInfo.h"
19#include "CGRecordLayout.h"
20#include "CGObjCRuntime.h"
21#include "TargetInfo.h"
22#include "clang/AST/ASTContext.h"
23#include "clang/AST/DeclObjC.h"
24#include "clang/Frontend/CodeGenOptions.h"
25#include "llvm/Intrinsics.h"
26#include "llvm/Target/TargetData.h"
27using namespace clang;
28using namespace CodeGen;
29
30//===--------------------------------------------------------------------===//
31//                        Miscellaneous Helper Methods
32//===--------------------------------------------------------------------===//
33
34llvm::Value *CodeGenFunction::EmitCastToVoidPtr(llvm::Value *value) {
35  unsigned addressSpace =
36    cast<llvm::PointerType>(value->getType())->getAddressSpace();
37
38  llvm::PointerType *destType = Int8PtrTy;
39  if (addressSpace)
40    destType = llvm::Type::getInt8PtrTy(getLLVMContext(), addressSpace);
41
42  if (value->getType() == destType) return value;
43  return Builder.CreateBitCast(value, destType);
44}
45
46/// CreateTempAlloca - This creates a alloca and inserts it into the entry
47/// block.
48llvm::AllocaInst *CodeGenFunction::CreateTempAlloca(llvm::Type *Ty,
49                                                    const Twine &Name) {
50  if (!Builder.isNamePreserving())
51    return new llvm::AllocaInst(Ty, 0, "", AllocaInsertPt);
52  return new llvm::AllocaInst(Ty, 0, Name, AllocaInsertPt);
53}
54
55void CodeGenFunction::InitTempAlloca(llvm::AllocaInst *Var,
56                                     llvm::Value *Init) {
57  llvm::StoreInst *Store = new llvm::StoreInst(Init, Var);
58  llvm::BasicBlock *Block = AllocaInsertPt->getParent();
59  Block->getInstList().insertAfter(&*AllocaInsertPt, Store);
60}
61
62llvm::AllocaInst *CodeGenFunction::CreateIRTemp(QualType Ty,
63                                                const Twine &Name) {
64  llvm::AllocaInst *Alloc = CreateTempAlloca(ConvertType(Ty), Name);
65  // FIXME: Should we prefer the preferred type alignment here?
66  CharUnits Align = getContext().getTypeAlignInChars(Ty);
67  Alloc->setAlignment(Align.getQuantity());
68  return Alloc;
69}
70
71llvm::AllocaInst *CodeGenFunction::CreateMemTemp(QualType Ty,
72                                                 const Twine &Name) {
73  llvm::AllocaInst *Alloc = CreateTempAlloca(ConvertTypeForMem(Ty), Name);
74  // FIXME: Should we prefer the preferred type alignment here?
75  CharUnits Align = getContext().getTypeAlignInChars(Ty);
76  Alloc->setAlignment(Align.getQuantity());
77  return Alloc;
78}
79
80/// EvaluateExprAsBool - Perform the usual unary conversions on the specified
81/// expression and compare the result against zero, returning an Int1Ty value.
82llvm::Value *CodeGenFunction::EvaluateExprAsBool(const Expr *E) {
83  if (const MemberPointerType *MPT = E->getType()->getAs<MemberPointerType>()) {
84    llvm::Value *MemPtr = EmitScalarExpr(E);
85    return CGM.getCXXABI().EmitMemberPointerIsNotNull(*this, MemPtr, MPT);
86  }
87
88  QualType BoolTy = getContext().BoolTy;
89  if (!E->getType()->isAnyComplexType())
90    return EmitScalarConversion(EmitScalarExpr(E), E->getType(), BoolTy);
91
92  return EmitComplexToScalarConversion(EmitComplexExpr(E), E->getType(),BoolTy);
93}
94
95/// EmitIgnoredExpr - Emit code to compute the specified expression,
96/// ignoring the result.
97void CodeGenFunction::EmitIgnoredExpr(const Expr *E) {
98  if (E->isRValue())
99    return (void) EmitAnyExpr(E, AggValueSlot::ignored(), true);
100
101  // Just emit it as an l-value and drop the result.
102  EmitLValue(E);
103}
104
105/// EmitAnyExpr - Emit code to compute the specified expression which
106/// can have any type.  The result is returned as an RValue struct.
107/// If this is an aggregate expression, AggSlot indicates where the
108/// result should be returned.
109RValue CodeGenFunction::EmitAnyExpr(const Expr *E, AggValueSlot AggSlot,
110                                    bool IgnoreResult) {
111  if (!hasAggregateLLVMType(E->getType()))
112    return RValue::get(EmitScalarExpr(E, IgnoreResult));
113  else if (E->getType()->isAnyComplexType())
114    return RValue::getComplex(EmitComplexExpr(E, IgnoreResult, IgnoreResult));
115
116  EmitAggExpr(E, AggSlot, IgnoreResult);
117  return AggSlot.asRValue();
118}
119
120/// EmitAnyExprToTemp - Similary to EmitAnyExpr(), however, the result will
121/// always be accessible even if no aggregate location is provided.
122RValue CodeGenFunction::EmitAnyExprToTemp(const Expr *E) {
123  AggValueSlot AggSlot = AggValueSlot::ignored();
124
125  if (hasAggregateLLVMType(E->getType()) &&
126      !E->getType()->isAnyComplexType())
127    AggSlot = CreateAggTemp(E->getType(), "agg.tmp");
128  return EmitAnyExpr(E, AggSlot);
129}
130
131/// EmitAnyExprToMem - Evaluate an expression into a given memory
132/// location.
133void CodeGenFunction::EmitAnyExprToMem(const Expr *E,
134                                       llvm::Value *Location,
135                                       Qualifiers Quals,
136                                       bool IsInit) {
137  if (E->getType()->isAnyComplexType())
138    EmitComplexExprIntoAddr(E, Location, Quals.hasVolatile());
139  else if (hasAggregateLLVMType(E->getType()))
140    EmitAggExpr(E, AggValueSlot::forAddr(Location, Quals,
141                                         AggValueSlot::IsDestructed_t(IsInit),
142                                         AggValueSlot::DoesNotNeedGCBarriers,
143                                         AggValueSlot::IsAliased_t(!IsInit)));
144  else {
145    RValue RV = RValue::get(EmitScalarExpr(E, /*Ignore*/ false));
146    LValue LV = MakeAddrLValue(Location, E->getType());
147    EmitStoreThroughLValue(RV, LV);
148  }
149}
150
151namespace {
152/// \brief An adjustment to be made to the temporary created when emitting a
153/// reference binding, which accesses a particular subobject of that temporary.
154  struct SubobjectAdjustment {
155    enum { DerivedToBaseAdjustment, FieldAdjustment } Kind;
156
157    union {
158      struct {
159        const CastExpr *BasePath;
160        const CXXRecordDecl *DerivedClass;
161      } DerivedToBase;
162
163      FieldDecl *Field;
164    };
165
166    SubobjectAdjustment(const CastExpr *BasePath,
167                        const CXXRecordDecl *DerivedClass)
168      : Kind(DerivedToBaseAdjustment) {
169      DerivedToBase.BasePath = BasePath;
170      DerivedToBase.DerivedClass = DerivedClass;
171    }
172
173    SubobjectAdjustment(FieldDecl *Field)
174      : Kind(FieldAdjustment) {
175      this->Field = Field;
176    }
177  };
178}
179
180static llvm::Value *
181CreateReferenceTemporary(CodeGenFunction &CGF, QualType Type,
182                         const NamedDecl *InitializedDecl) {
183  if (const VarDecl *VD = dyn_cast_or_null<VarDecl>(InitializedDecl)) {
184    if (VD->hasGlobalStorage()) {
185      llvm::SmallString<256> Name;
186      llvm::raw_svector_ostream Out(Name);
187      CGF.CGM.getCXXABI().getMangleContext().mangleReferenceTemporary(VD, Out);
188      Out.flush();
189
190      llvm::Type *RefTempTy = CGF.ConvertTypeForMem(Type);
191
192      // Create the reference temporary.
193      llvm::GlobalValue *RefTemp =
194        new llvm::GlobalVariable(CGF.CGM.getModule(),
195                                 RefTempTy, /*isConstant=*/false,
196                                 llvm::GlobalValue::InternalLinkage,
197                                 llvm::Constant::getNullValue(RefTempTy),
198                                 Name.str());
199      return RefTemp;
200    }
201  }
202
203  return CGF.CreateMemTemp(Type, "ref.tmp");
204}
205
206static llvm::Value *
207EmitExprForReferenceBinding(CodeGenFunction &CGF, const Expr *E,
208                            llvm::Value *&ReferenceTemporary,
209                            const CXXDestructorDecl *&ReferenceTemporaryDtor,
210                            QualType &ObjCARCReferenceLifetimeType,
211                            const NamedDecl *InitializedDecl) {
212  // Look through expressions for materialized temporaries (for now).
213  if (const MaterializeTemporaryExpr *M
214                                      = dyn_cast<MaterializeTemporaryExpr>(E)) {
215    // Objective-C++ ARC:
216    //   If we are binding a reference to a temporary that has ownership, we
217    //   need to perform retain/release operations on the temporary.
218    if (CGF.getContext().getLangOptions().ObjCAutoRefCount &&
219        E->getType()->isObjCLifetimeType() &&
220        (E->getType().getObjCLifetime() == Qualifiers::OCL_Strong ||
221         E->getType().getObjCLifetime() == Qualifiers::OCL_Weak ||
222         E->getType().getObjCLifetime() == Qualifiers::OCL_Autoreleasing))
223      ObjCARCReferenceLifetimeType = E->getType();
224
225    E = M->GetTemporaryExpr();
226  }
227
228  if (const CXXDefaultArgExpr *DAE = dyn_cast<CXXDefaultArgExpr>(E))
229    E = DAE->getExpr();
230
231  if (const ExprWithCleanups *TE = dyn_cast<ExprWithCleanups>(E)) {
232    CodeGenFunction::RunCleanupsScope Scope(CGF);
233
234    return EmitExprForReferenceBinding(CGF, TE->getSubExpr(),
235                                       ReferenceTemporary,
236                                       ReferenceTemporaryDtor,
237                                       ObjCARCReferenceLifetimeType,
238                                       InitializedDecl);
239  }
240
241  if (const ObjCPropertyRefExpr *PRE =
242      dyn_cast<ObjCPropertyRefExpr>(E->IgnoreParenImpCasts()))
243    if (PRE->getGetterResultType()->isReferenceType())
244      E = PRE;
245
246  RValue RV;
247  if (E->isGLValue()) {
248    // Emit the expression as an lvalue.
249    LValue LV = CGF.EmitLValue(E);
250    if (LV.isPropertyRef()) {
251      RV = CGF.EmitLoadOfPropertyRefLValue(LV);
252      return RV.getScalarVal();
253    }
254
255    if (LV.isSimple())
256      return LV.getAddress();
257
258    // We have to load the lvalue.
259    RV = CGF.EmitLoadOfLValue(LV);
260  } else {
261    if (!ObjCARCReferenceLifetimeType.isNull()) {
262      ReferenceTemporary = CreateReferenceTemporary(CGF,
263                                                  ObjCARCReferenceLifetimeType,
264                                                    InitializedDecl);
265
266
267      LValue RefTempDst = CGF.MakeAddrLValue(ReferenceTemporary,
268                                             ObjCARCReferenceLifetimeType);
269
270      CGF.EmitScalarInit(E, dyn_cast_or_null<ValueDecl>(InitializedDecl),
271                         RefTempDst, false);
272
273      bool ExtendsLifeOfTemporary = false;
274      if (const VarDecl *Var = dyn_cast_or_null<VarDecl>(InitializedDecl)) {
275        if (Var->extendsLifetimeOfTemporary())
276          ExtendsLifeOfTemporary = true;
277      } else if (InitializedDecl && isa<FieldDecl>(InitializedDecl)) {
278        ExtendsLifeOfTemporary = true;
279      }
280
281      if (!ExtendsLifeOfTemporary) {
282        // Since the lifetime of this temporary isn't going to be extended,
283        // we need to clean it up ourselves at the end of the full expression.
284        switch (ObjCARCReferenceLifetimeType.getObjCLifetime()) {
285        case Qualifiers::OCL_None:
286        case Qualifiers::OCL_ExplicitNone:
287        case Qualifiers::OCL_Autoreleasing:
288          break;
289
290        case Qualifiers::OCL_Strong: {
291          assert(!ObjCARCReferenceLifetimeType->isArrayType());
292          CleanupKind cleanupKind = CGF.getARCCleanupKind();
293          CGF.pushDestroy(cleanupKind,
294                          ReferenceTemporary,
295                          ObjCARCReferenceLifetimeType,
296                          CodeGenFunction::destroyARCStrongImprecise,
297                          cleanupKind & EHCleanup);
298          break;
299        }
300
301        case Qualifiers::OCL_Weak:
302          assert(!ObjCARCReferenceLifetimeType->isArrayType());
303          CGF.pushDestroy(NormalAndEHCleanup,
304                          ReferenceTemporary,
305                          ObjCARCReferenceLifetimeType,
306                          CodeGenFunction::destroyARCWeak,
307                          /*useEHCleanupForArray*/ true);
308          break;
309        }
310
311        ObjCARCReferenceLifetimeType = QualType();
312      }
313
314      return ReferenceTemporary;
315    }
316
317    SmallVector<SubobjectAdjustment, 2> Adjustments;
318    while (true) {
319      E = E->IgnoreParens();
320
321      if (const CastExpr *CE = dyn_cast<CastExpr>(E)) {
322        if ((CE->getCastKind() == CK_DerivedToBase ||
323             CE->getCastKind() == CK_UncheckedDerivedToBase) &&
324            E->getType()->isRecordType()) {
325          E = CE->getSubExpr();
326          CXXRecordDecl *Derived
327            = cast<CXXRecordDecl>(E->getType()->getAs<RecordType>()->getDecl());
328          Adjustments.push_back(SubobjectAdjustment(CE, Derived));
329          continue;
330        }
331
332        if (CE->getCastKind() == CK_NoOp) {
333          E = CE->getSubExpr();
334          continue;
335        }
336      } else if (const MemberExpr *ME = dyn_cast<MemberExpr>(E)) {
337        if (!ME->isArrow() && ME->getBase()->isRValue()) {
338          assert(ME->getBase()->getType()->isRecordType());
339          if (FieldDecl *Field = dyn_cast<FieldDecl>(ME->getMemberDecl())) {
340            E = ME->getBase();
341            Adjustments.push_back(SubobjectAdjustment(Field));
342            continue;
343          }
344        }
345      }
346
347      if (const OpaqueValueExpr *opaque = dyn_cast<OpaqueValueExpr>(E))
348        if (opaque->getType()->isRecordType())
349          return CGF.EmitOpaqueValueLValue(opaque).getAddress();
350
351      // Nothing changed.
352      break;
353    }
354
355    // Create a reference temporary if necessary.
356    AggValueSlot AggSlot = AggValueSlot::ignored();
357    if (CGF.hasAggregateLLVMType(E->getType()) &&
358        !E->getType()->isAnyComplexType()) {
359      ReferenceTemporary = CreateReferenceTemporary(CGF, E->getType(),
360                                                    InitializedDecl);
361      AggValueSlot::IsDestructed_t isDestructed
362        = AggValueSlot::IsDestructed_t(InitializedDecl != 0);
363      AggSlot = AggValueSlot::forAddr(ReferenceTemporary, Qualifiers(),
364                                      isDestructed,
365                                      AggValueSlot::DoesNotNeedGCBarriers,
366                                      AggValueSlot::IsNotAliased);
367    }
368
369    if (InitializedDecl) {
370      // Get the destructor for the reference temporary.
371      if (const RecordType *RT = E->getType()->getAs<RecordType>()) {
372        CXXRecordDecl *ClassDecl = cast<CXXRecordDecl>(RT->getDecl());
373        if (!ClassDecl->hasTrivialDestructor())
374          ReferenceTemporaryDtor = ClassDecl->getDestructor();
375      }
376    }
377
378    RV = CGF.EmitAnyExpr(E, AggSlot);
379
380    // Check if need to perform derived-to-base casts and/or field accesses, to
381    // get from the temporary object we created (and, potentially, for which we
382    // extended the lifetime) to the subobject we're binding the reference to.
383    if (!Adjustments.empty()) {
384      llvm::Value *Object = RV.getAggregateAddr();
385      for (unsigned I = Adjustments.size(); I != 0; --I) {
386        SubobjectAdjustment &Adjustment = Adjustments[I-1];
387        switch (Adjustment.Kind) {
388        case SubobjectAdjustment::DerivedToBaseAdjustment:
389          Object =
390              CGF.GetAddressOfBaseClass(Object,
391                                        Adjustment.DerivedToBase.DerivedClass,
392                              Adjustment.DerivedToBase.BasePath->path_begin(),
393                              Adjustment.DerivedToBase.BasePath->path_end(),
394                                        /*NullCheckValue=*/false);
395          break;
396
397        case SubobjectAdjustment::FieldAdjustment: {
398          LValue LV =
399            CGF.EmitLValueForField(Object, Adjustment.Field, 0);
400          if (LV.isSimple()) {
401            Object = LV.getAddress();
402            break;
403          }
404
405          // For non-simple lvalues, we actually have to create a copy of
406          // the object we're binding to.
407          QualType T = Adjustment.Field->getType().getNonReferenceType()
408                                                  .getUnqualifiedType();
409          Object = CreateReferenceTemporary(CGF, T, InitializedDecl);
410          LValue TempLV = CGF.MakeAddrLValue(Object,
411                                             Adjustment.Field->getType());
412          CGF.EmitStoreThroughLValue(CGF.EmitLoadOfLValue(LV), TempLV);
413          break;
414        }
415
416        }
417      }
418
419      return Object;
420    }
421  }
422
423  if (RV.isAggregate())
424    return RV.getAggregateAddr();
425
426  // Create a temporary variable that we can bind the reference to.
427  ReferenceTemporary = CreateReferenceTemporary(CGF, E->getType(),
428                                                InitializedDecl);
429
430
431  unsigned Alignment =
432    CGF.getContext().getTypeAlignInChars(E->getType()).getQuantity();
433  if (RV.isScalar())
434    CGF.EmitStoreOfScalar(RV.getScalarVal(), ReferenceTemporary,
435                          /*Volatile=*/false, Alignment, E->getType());
436  else
437    CGF.StoreComplexToAddr(RV.getComplexVal(), ReferenceTemporary,
438                           /*Volatile=*/false);
439  return ReferenceTemporary;
440}
441
442RValue
443CodeGenFunction::EmitReferenceBindingToExpr(const Expr *E,
444                                            const NamedDecl *InitializedDecl) {
445  llvm::Value *ReferenceTemporary = 0;
446  const CXXDestructorDecl *ReferenceTemporaryDtor = 0;
447  QualType ObjCARCReferenceLifetimeType;
448  llvm::Value *Value = EmitExprForReferenceBinding(*this, E, ReferenceTemporary,
449                                                   ReferenceTemporaryDtor,
450                                                   ObjCARCReferenceLifetimeType,
451                                                   InitializedDecl);
452  if (!ReferenceTemporaryDtor && ObjCARCReferenceLifetimeType.isNull())
453    return RValue::get(Value);
454
455  // Make sure to call the destructor for the reference temporary.
456  const VarDecl *VD = dyn_cast_or_null<VarDecl>(InitializedDecl);
457  if (VD && VD->hasGlobalStorage()) {
458    if (ReferenceTemporaryDtor) {
459      llvm::Constant *DtorFn =
460        CGM.GetAddrOfCXXDestructor(ReferenceTemporaryDtor, Dtor_Complete);
461      EmitCXXGlobalDtorRegistration(DtorFn,
462                                    cast<llvm::Constant>(ReferenceTemporary));
463    } else {
464      assert(!ObjCARCReferenceLifetimeType.isNull());
465      // Note: We intentionally do not register a global "destructor" to
466      // release the object.
467    }
468
469    return RValue::get(Value);
470  }
471
472  if (ReferenceTemporaryDtor)
473    PushDestructorCleanup(ReferenceTemporaryDtor, ReferenceTemporary);
474  else {
475    switch (ObjCARCReferenceLifetimeType.getObjCLifetime()) {
476    case Qualifiers::OCL_None:
477      llvm_unreachable(
478                      "Not a reference temporary that needs to be deallocated");
479    case Qualifiers::OCL_ExplicitNone:
480    case Qualifiers::OCL_Autoreleasing:
481      // Nothing to do.
482      break;
483
484    case Qualifiers::OCL_Strong: {
485      bool precise = VD && VD->hasAttr<ObjCPreciseLifetimeAttr>();
486      CleanupKind cleanupKind = getARCCleanupKind();
487      // This local is a GCC and MSVC compiler workaround.
488      Destroyer *destroyer = precise ? &destroyARCStrongPrecise :
489                                       &destroyARCStrongImprecise;
490      pushDestroy(cleanupKind, ReferenceTemporary, ObjCARCReferenceLifetimeType,
491                  *destroyer, cleanupKind & EHCleanup);
492      break;
493    }
494
495    case Qualifiers::OCL_Weak: {
496      // This local is a GCC and MSVC compiler workaround.
497      Destroyer *destroyer = &destroyARCWeak;
498      // __weak objects always get EH cleanups; otherwise, exceptions
499      // could cause really nasty crashes instead of mere leaks.
500      pushDestroy(NormalAndEHCleanup, ReferenceTemporary,
501                  ObjCARCReferenceLifetimeType, *destroyer, true);
502      break;
503    }
504    }
505  }
506
507  return RValue::get(Value);
508}
509
510
511/// getAccessedFieldNo - Given an encoded value and a result number, return the
512/// input field number being accessed.
513unsigned CodeGenFunction::getAccessedFieldNo(unsigned Idx,
514                                             const llvm::Constant *Elts) {
515  if (isa<llvm::ConstantAggregateZero>(Elts))
516    return 0;
517
518  return cast<llvm::ConstantInt>(Elts->getOperand(Idx))->getZExtValue();
519}
520
521void CodeGenFunction::EmitCheck(llvm::Value *Address, unsigned Size) {
522  if (!CatchUndefined)
523    return;
524
525  // This needs to be to the standard address space.
526  Address = Builder.CreateBitCast(Address, Int8PtrTy);
527
528  llvm::Value *F = CGM.getIntrinsic(llvm::Intrinsic::objectsize, IntPtrTy);
529
530  // In time, people may want to control this and use a 1 here.
531  llvm::Value *Arg = Builder.getFalse();
532  llvm::Value *C = Builder.CreateCall2(F, Address, Arg);
533  llvm::BasicBlock *Cont = createBasicBlock();
534  llvm::BasicBlock *Check = createBasicBlock();
535  llvm::Value *NegativeOne = llvm::ConstantInt::get(IntPtrTy, -1ULL);
536  Builder.CreateCondBr(Builder.CreateICmpEQ(C, NegativeOne), Cont, Check);
537
538  EmitBlock(Check);
539  Builder.CreateCondBr(Builder.CreateICmpUGE(C,
540                                        llvm::ConstantInt::get(IntPtrTy, Size)),
541                       Cont, getTrapBB());
542  EmitBlock(Cont);
543}
544
545
546CodeGenFunction::ComplexPairTy CodeGenFunction::
547EmitComplexPrePostIncDec(const UnaryOperator *E, LValue LV,
548                         bool isInc, bool isPre) {
549  ComplexPairTy InVal = LoadComplexFromAddr(LV.getAddress(),
550                                            LV.isVolatileQualified());
551
552  llvm::Value *NextVal;
553  if (isa<llvm::IntegerType>(InVal.first->getType())) {
554    uint64_t AmountVal = isInc ? 1 : -1;
555    NextVal = llvm::ConstantInt::get(InVal.first->getType(), AmountVal, true);
556
557    // Add the inc/dec to the real part.
558    NextVal = Builder.CreateAdd(InVal.first, NextVal, isInc ? "inc" : "dec");
559  } else {
560    QualType ElemTy = E->getType()->getAs<ComplexType>()->getElementType();
561    llvm::APFloat FVal(getContext().getFloatTypeSemantics(ElemTy), 1);
562    if (!isInc)
563      FVal.changeSign();
564    NextVal = llvm::ConstantFP::get(getLLVMContext(), FVal);
565
566    // Add the inc/dec to the real part.
567    NextVal = Builder.CreateFAdd(InVal.first, NextVal, isInc ? "inc" : "dec");
568  }
569
570  ComplexPairTy IncVal(NextVal, InVal.second);
571
572  // Store the updated result through the lvalue.
573  StoreComplexToAddr(IncVal, LV.getAddress(), LV.isVolatileQualified());
574
575  // If this is a postinc, return the value read from memory, otherwise use the
576  // updated value.
577  return isPre ? IncVal : InVal;
578}
579
580
581//===----------------------------------------------------------------------===//
582//                         LValue Expression Emission
583//===----------------------------------------------------------------------===//
584
585RValue CodeGenFunction::GetUndefRValue(QualType Ty) {
586  if (Ty->isVoidType())
587    return RValue::get(0);
588
589  if (const ComplexType *CTy = Ty->getAs<ComplexType>()) {
590    llvm::Type *EltTy = ConvertType(CTy->getElementType());
591    llvm::Value *U = llvm::UndefValue::get(EltTy);
592    return RValue::getComplex(std::make_pair(U, U));
593  }
594
595  // If this is a use of an undefined aggregate type, the aggregate must have an
596  // identifiable address.  Just because the contents of the value are undefined
597  // doesn't mean that the address can't be taken and compared.
598  if (hasAggregateLLVMType(Ty)) {
599    llvm::Value *DestPtr = CreateMemTemp(Ty, "undef.agg.tmp");
600    return RValue::getAggregate(DestPtr);
601  }
602
603  return RValue::get(llvm::UndefValue::get(ConvertType(Ty)));
604}
605
606RValue CodeGenFunction::EmitUnsupportedRValue(const Expr *E,
607                                              const char *Name) {
608  ErrorUnsupported(E, Name);
609  return GetUndefRValue(E->getType());
610}
611
612LValue CodeGenFunction::EmitUnsupportedLValue(const Expr *E,
613                                              const char *Name) {
614  ErrorUnsupported(E, Name);
615  llvm::Type *Ty = llvm::PointerType::getUnqual(ConvertType(E->getType()));
616  return MakeAddrLValue(llvm::UndefValue::get(Ty), E->getType());
617}
618
619LValue CodeGenFunction::EmitCheckedLValue(const Expr *E) {
620  LValue LV = EmitLValue(E);
621  if (!isa<DeclRefExpr>(E) && !LV.isBitField() && LV.isSimple())
622    EmitCheck(LV.getAddress(),
623              getContext().getTypeSizeInChars(E->getType()).getQuantity());
624  return LV;
625}
626
627/// EmitLValue - Emit code to compute a designator that specifies the location
628/// of the expression.
629///
630/// This can return one of two things: a simple address or a bitfield reference.
631/// In either case, the LLVM Value* in the LValue structure is guaranteed to be
632/// an LLVM pointer type.
633///
634/// If this returns a bitfield reference, nothing about the pointee type of the
635/// LLVM value is known: For example, it may not be a pointer to an integer.
636///
637/// If this returns a normal address, and if the lvalue's C type is fixed size,
638/// this method guarantees that the returned pointer type will point to an LLVM
639/// type of the same size of the lvalue's type.  If the lvalue has a variable
640/// length type, this is not possible.
641///
642LValue CodeGenFunction::EmitLValue(const Expr *E) {
643  switch (E->getStmtClass()) {
644  default: return EmitUnsupportedLValue(E, "l-value expression");
645
646  case Expr::ObjCSelectorExprClass:
647  return EmitObjCSelectorLValue(cast<ObjCSelectorExpr>(E));
648  case Expr::ObjCIsaExprClass:
649    return EmitObjCIsaExpr(cast<ObjCIsaExpr>(E));
650  case Expr::BinaryOperatorClass:
651    return EmitBinaryOperatorLValue(cast<BinaryOperator>(E));
652  case Expr::CompoundAssignOperatorClass:
653    if (!E->getType()->isAnyComplexType())
654      return EmitCompoundAssignmentLValue(cast<CompoundAssignOperator>(E));
655    return EmitComplexCompoundAssignmentLValue(cast<CompoundAssignOperator>(E));
656  case Expr::CallExprClass:
657  case Expr::CXXMemberCallExprClass:
658  case Expr::CXXOperatorCallExprClass:
659    return EmitCallExprLValue(cast<CallExpr>(E));
660  case Expr::VAArgExprClass:
661    return EmitVAArgExprLValue(cast<VAArgExpr>(E));
662  case Expr::DeclRefExprClass:
663    return EmitDeclRefLValue(cast<DeclRefExpr>(E));
664  case Expr::ParenExprClass:
665    return EmitLValue(cast<ParenExpr>(E)->getSubExpr());
666  case Expr::GenericSelectionExprClass:
667    return EmitLValue(cast<GenericSelectionExpr>(E)->getResultExpr());
668  case Expr::PredefinedExprClass:
669    return EmitPredefinedLValue(cast<PredefinedExpr>(E));
670  case Expr::StringLiteralClass:
671    return EmitStringLiteralLValue(cast<StringLiteral>(E));
672  case Expr::ObjCEncodeExprClass:
673    return EmitObjCEncodeExprLValue(cast<ObjCEncodeExpr>(E));
674
675  case Expr::BlockDeclRefExprClass:
676    return EmitBlockDeclRefLValue(cast<BlockDeclRefExpr>(E));
677
678  case Expr::CXXTemporaryObjectExprClass:
679  case Expr::CXXConstructExprClass:
680    return EmitCXXConstructLValue(cast<CXXConstructExpr>(E));
681  case Expr::CXXBindTemporaryExprClass:
682    return EmitCXXBindTemporaryLValue(cast<CXXBindTemporaryExpr>(E));
683  case Expr::ExprWithCleanupsClass:
684    return EmitExprWithCleanupsLValue(cast<ExprWithCleanups>(E));
685  case Expr::CXXScalarValueInitExprClass:
686    return EmitNullInitializationLValue(cast<CXXScalarValueInitExpr>(E));
687  case Expr::CXXDefaultArgExprClass:
688    return EmitLValue(cast<CXXDefaultArgExpr>(E)->getExpr());
689  case Expr::CXXTypeidExprClass:
690    return EmitCXXTypeidLValue(cast<CXXTypeidExpr>(E));
691
692  case Expr::ObjCMessageExprClass:
693    return EmitObjCMessageExprLValue(cast<ObjCMessageExpr>(E));
694  case Expr::ObjCIvarRefExprClass:
695    return EmitObjCIvarRefLValue(cast<ObjCIvarRefExpr>(E));
696  case Expr::ObjCPropertyRefExprClass:
697    return EmitObjCPropertyRefLValue(cast<ObjCPropertyRefExpr>(E));
698  case Expr::StmtExprClass:
699    return EmitStmtExprLValue(cast<StmtExpr>(E));
700  case Expr::UnaryOperatorClass:
701    return EmitUnaryOpLValue(cast<UnaryOperator>(E));
702  case Expr::ArraySubscriptExprClass:
703    return EmitArraySubscriptExpr(cast<ArraySubscriptExpr>(E));
704  case Expr::ExtVectorElementExprClass:
705    return EmitExtVectorElementExpr(cast<ExtVectorElementExpr>(E));
706  case Expr::MemberExprClass:
707    return EmitMemberExpr(cast<MemberExpr>(E));
708  case Expr::CompoundLiteralExprClass:
709    return EmitCompoundLiteralLValue(cast<CompoundLiteralExpr>(E));
710  case Expr::ConditionalOperatorClass:
711    return EmitConditionalOperatorLValue(cast<ConditionalOperator>(E));
712  case Expr::BinaryConditionalOperatorClass:
713    return EmitConditionalOperatorLValue(cast<BinaryConditionalOperator>(E));
714  case Expr::ChooseExprClass:
715    return EmitLValue(cast<ChooseExpr>(E)->getChosenSubExpr(getContext()));
716  case Expr::OpaqueValueExprClass:
717    return EmitOpaqueValueLValue(cast<OpaqueValueExpr>(E));
718  case Expr::SubstNonTypeTemplateParmExprClass:
719    return EmitLValue(cast<SubstNonTypeTemplateParmExpr>(E)->getReplacement());
720  case Expr::ImplicitCastExprClass:
721  case Expr::CStyleCastExprClass:
722  case Expr::CXXFunctionalCastExprClass:
723  case Expr::CXXStaticCastExprClass:
724  case Expr::CXXDynamicCastExprClass:
725  case Expr::CXXReinterpretCastExprClass:
726  case Expr::CXXConstCastExprClass:
727  case Expr::ObjCBridgedCastExprClass:
728    return EmitCastLValue(cast<CastExpr>(E));
729
730  case Expr::MaterializeTemporaryExprClass:
731    return EmitMaterializeTemporaryExpr(cast<MaterializeTemporaryExpr>(E));
732  }
733}
734
735llvm::Value *CodeGenFunction::EmitLoadOfScalar(LValue lvalue) {
736  return EmitLoadOfScalar(lvalue.getAddress(), lvalue.isVolatile(),
737                          lvalue.getAlignment(), lvalue.getType(),
738                          lvalue.getTBAAInfo());
739}
740
741llvm::Value *CodeGenFunction::EmitLoadOfScalar(llvm::Value *Addr, bool Volatile,
742                                              unsigned Alignment, QualType Ty,
743                                              llvm::MDNode *TBAAInfo) {
744  llvm::LoadInst *Load = Builder.CreateLoad(Addr);
745  if (Volatile)
746    Load->setVolatile(true);
747  if (Alignment)
748    Load->setAlignment(Alignment);
749  if (TBAAInfo)
750    CGM.DecorateInstruction(Load, TBAAInfo);
751
752  return EmitFromMemory(Load, Ty);
753}
754
755static bool isBooleanUnderlyingType(QualType Ty) {
756  if (const EnumType *ET = dyn_cast<EnumType>(Ty))
757    return ET->getDecl()->getIntegerType()->isBooleanType();
758  return false;
759}
760
761llvm::Value *CodeGenFunction::EmitToMemory(llvm::Value *Value, QualType Ty) {
762  // Bool has a different representation in memory than in registers.
763  if (Ty->isBooleanType() || isBooleanUnderlyingType(Ty)) {
764    // This should really always be an i1, but sometimes it's already
765    // an i8, and it's awkward to track those cases down.
766    if (Value->getType()->isIntegerTy(1))
767      return Builder.CreateZExt(Value, Builder.getInt8Ty(), "frombool");
768    assert(Value->getType()->isIntegerTy(8) && "value rep of bool not i1/i8");
769  }
770
771  return Value;
772}
773
774llvm::Value *CodeGenFunction::EmitFromMemory(llvm::Value *Value, QualType Ty) {
775  // Bool has a different representation in memory than in registers.
776  if (Ty->isBooleanType() || isBooleanUnderlyingType(Ty)) {
777    assert(Value->getType()->isIntegerTy(8) && "memory rep of bool not i8");
778    return Builder.CreateTrunc(Value, Builder.getInt1Ty(), "tobool");
779  }
780
781  return Value;
782}
783
784void CodeGenFunction::EmitStoreOfScalar(llvm::Value *Value, llvm::Value *Addr,
785                                        bool Volatile, unsigned Alignment,
786                                        QualType Ty,
787                                        llvm::MDNode *TBAAInfo) {
788  Value = EmitToMemory(Value, Ty);
789
790  llvm::StoreInst *Store = Builder.CreateStore(Value, Addr, Volatile);
791  if (Alignment)
792    Store->setAlignment(Alignment);
793  if (TBAAInfo)
794    CGM.DecorateInstruction(Store, TBAAInfo);
795}
796
797void CodeGenFunction::EmitStoreOfScalar(llvm::Value *value, LValue lvalue) {
798  EmitStoreOfScalar(value, lvalue.getAddress(), lvalue.isVolatile(),
799                    lvalue.getAlignment(), lvalue.getType(),
800                    lvalue.getTBAAInfo());
801}
802
803/// EmitLoadOfLValue - Given an expression that represents a value lvalue, this
804/// method emits the address of the lvalue, then loads the result as an rvalue,
805/// returning the rvalue.
806RValue CodeGenFunction::EmitLoadOfLValue(LValue LV) {
807  if (LV.isObjCWeak()) {
808    // load of a __weak object.
809    llvm::Value *AddrWeakObj = LV.getAddress();
810    return RValue::get(CGM.getObjCRuntime().EmitObjCWeakRead(*this,
811                                                             AddrWeakObj));
812  }
813  if (LV.getQuals().getObjCLifetime() == Qualifiers::OCL_Weak)
814    return RValue::get(EmitARCLoadWeak(LV.getAddress()));
815
816  if (LV.isSimple()) {
817    assert(!LV.getType()->isFunctionType());
818
819    // Everything needs a load.
820    return RValue::get(EmitLoadOfScalar(LV));
821  }
822
823  if (LV.isVectorElt()) {
824    llvm::Value *Vec = Builder.CreateLoad(LV.getVectorAddr(),
825                                          LV.isVolatileQualified());
826    return RValue::get(Builder.CreateExtractElement(Vec, LV.getVectorIdx(),
827                                                    "vecext"));
828  }
829
830  // If this is a reference to a subset of the elements of a vector, either
831  // shuffle the input or extract/insert them as appropriate.
832  if (LV.isExtVectorElt())
833    return EmitLoadOfExtVectorElementLValue(LV);
834
835  if (LV.isBitField())
836    return EmitLoadOfBitfieldLValue(LV);
837
838  assert(LV.isPropertyRef() && "Unknown LValue type!");
839  return EmitLoadOfPropertyRefLValue(LV);
840}
841
842RValue CodeGenFunction::EmitLoadOfBitfieldLValue(LValue LV) {
843  const CGBitFieldInfo &Info = LV.getBitFieldInfo();
844
845  // Get the output type.
846  llvm::Type *ResLTy = ConvertType(LV.getType());
847  unsigned ResSizeInBits = CGM.getTargetData().getTypeSizeInBits(ResLTy);
848
849  // Compute the result as an OR of all of the individual component accesses.
850  llvm::Value *Res = 0;
851  for (unsigned i = 0, e = Info.getNumComponents(); i != e; ++i) {
852    const CGBitFieldInfo::AccessInfo &AI = Info.getComponent(i);
853
854    // Get the field pointer.
855    llvm::Value *Ptr = LV.getBitFieldBaseAddr();
856
857    // Only offset by the field index if used, so that incoming values are not
858    // required to be structures.
859    if (AI.FieldIndex)
860      Ptr = Builder.CreateStructGEP(Ptr, AI.FieldIndex, "bf.field");
861
862    // Offset by the byte offset, if used.
863    if (!AI.FieldByteOffset.isZero()) {
864      Ptr = EmitCastToVoidPtr(Ptr);
865      Ptr = Builder.CreateConstGEP1_32(Ptr, AI.FieldByteOffset.getQuantity(),
866                                       "bf.field.offs");
867    }
868
869    // Cast to the access type.
870    llvm::Type *PTy = llvm::Type::getIntNPtrTy(getLLVMContext(),
871                                                     AI.AccessWidth,
872                       CGM.getContext().getTargetAddressSpace(LV.getType()));
873    Ptr = Builder.CreateBitCast(Ptr, PTy);
874
875    // Perform the load.
876    llvm::LoadInst *Load = Builder.CreateLoad(Ptr, LV.isVolatileQualified());
877    if (!AI.AccessAlignment.isZero())
878      Load->setAlignment(AI.AccessAlignment.getQuantity());
879
880    // Shift out unused low bits and mask out unused high bits.
881    llvm::Value *Val = Load;
882    if (AI.FieldBitStart)
883      Val = Builder.CreateLShr(Load, AI.FieldBitStart);
884    Val = Builder.CreateAnd(Val, llvm::APInt::getLowBitsSet(AI.AccessWidth,
885                                                            AI.TargetBitWidth),
886                            "bf.clear");
887
888    // Extend or truncate to the target size.
889    if (AI.AccessWidth < ResSizeInBits)
890      Val = Builder.CreateZExt(Val, ResLTy);
891    else if (AI.AccessWidth > ResSizeInBits)
892      Val = Builder.CreateTrunc(Val, ResLTy);
893
894    // Shift into place, and OR into the result.
895    if (AI.TargetBitOffset)
896      Val = Builder.CreateShl(Val, AI.TargetBitOffset);
897    Res = Res ? Builder.CreateOr(Res, Val) : Val;
898  }
899
900  // If the bit-field is signed, perform the sign-extension.
901  //
902  // FIXME: This can easily be folded into the load of the high bits, which
903  // could also eliminate the mask of high bits in some situations.
904  if (Info.isSigned()) {
905    unsigned ExtraBits = ResSizeInBits - Info.getSize();
906    if (ExtraBits)
907      Res = Builder.CreateAShr(Builder.CreateShl(Res, ExtraBits),
908                               ExtraBits, "bf.val.sext");
909  }
910
911  return RValue::get(Res);
912}
913
914// If this is a reference to a subset of the elements of a vector, create an
915// appropriate shufflevector.
916RValue CodeGenFunction::EmitLoadOfExtVectorElementLValue(LValue LV) {
917  llvm::Value *Vec = Builder.CreateLoad(LV.getExtVectorAddr(),
918                                        LV.isVolatileQualified());
919
920  const llvm::Constant *Elts = LV.getExtVectorElts();
921
922  // If the result of the expression is a non-vector type, we must be extracting
923  // a single element.  Just codegen as an extractelement.
924  const VectorType *ExprVT = LV.getType()->getAs<VectorType>();
925  if (!ExprVT) {
926    unsigned InIdx = getAccessedFieldNo(0, Elts);
927    llvm::Value *Elt = llvm::ConstantInt::get(Int32Ty, InIdx);
928    return RValue::get(Builder.CreateExtractElement(Vec, Elt));
929  }
930
931  // Always use shuffle vector to try to retain the original program structure
932  unsigned NumResultElts = ExprVT->getNumElements();
933
934  SmallVector<llvm::Constant*, 4> Mask;
935  for (unsigned i = 0; i != NumResultElts; ++i) {
936    unsigned InIdx = getAccessedFieldNo(i, Elts);
937    Mask.push_back(llvm::ConstantInt::get(Int32Ty, InIdx));
938  }
939
940  llvm::Value *MaskV = llvm::ConstantVector::get(Mask);
941  Vec = Builder.CreateShuffleVector(Vec, llvm::UndefValue::get(Vec->getType()),
942                                    MaskV);
943  return RValue::get(Vec);
944}
945
946
947
948/// EmitStoreThroughLValue - Store the specified rvalue into the specified
949/// lvalue, where both are guaranteed to the have the same type, and that type
950/// is 'Ty'.
951void CodeGenFunction::EmitStoreThroughLValue(RValue Src, LValue Dst) {
952  if (!Dst.isSimple()) {
953    if (Dst.isVectorElt()) {
954      // Read/modify/write the vector, inserting the new element.
955      llvm::Value *Vec = Builder.CreateLoad(Dst.getVectorAddr(),
956                                            Dst.isVolatileQualified());
957      Vec = Builder.CreateInsertElement(Vec, Src.getScalarVal(),
958                                        Dst.getVectorIdx(), "vecins");
959      Builder.CreateStore(Vec, Dst.getVectorAddr(),Dst.isVolatileQualified());
960      return;
961    }
962
963    // If this is an update of extended vector elements, insert them as
964    // appropriate.
965    if (Dst.isExtVectorElt())
966      return EmitStoreThroughExtVectorComponentLValue(Src, Dst);
967
968    if (Dst.isBitField())
969      return EmitStoreThroughBitfieldLValue(Src, Dst);
970
971    assert(Dst.isPropertyRef() && "Unknown LValue type");
972    return EmitStoreThroughPropertyRefLValue(Src, Dst);
973  }
974
975  // There's special magic for assigning into an ARC-qualified l-value.
976  if (Qualifiers::ObjCLifetime Lifetime = Dst.getQuals().getObjCLifetime()) {
977    switch (Lifetime) {
978    case Qualifiers::OCL_None:
979      llvm_unreachable("present but none");
980
981    case Qualifiers::OCL_ExplicitNone:
982      // nothing special
983      break;
984
985    case Qualifiers::OCL_Strong:
986      EmitARCStoreStrong(Dst, Src.getScalarVal(), /*ignore*/ true);
987      return;
988
989    case Qualifiers::OCL_Weak:
990      EmitARCStoreWeak(Dst.getAddress(), Src.getScalarVal(), /*ignore*/ true);
991      return;
992
993    case Qualifiers::OCL_Autoreleasing:
994      Src = RValue::get(EmitObjCExtendObjectLifetime(Dst.getType(),
995                                                     Src.getScalarVal()));
996      // fall into the normal path
997      break;
998    }
999  }
1000
1001  if (Dst.isObjCWeak() && !Dst.isNonGC()) {
1002    // load of a __weak object.
1003    llvm::Value *LvalueDst = Dst.getAddress();
1004    llvm::Value *src = Src.getScalarVal();
1005     CGM.getObjCRuntime().EmitObjCWeakAssign(*this, src, LvalueDst);
1006    return;
1007  }
1008
1009  if (Dst.isObjCStrong() && !Dst.isNonGC()) {
1010    // load of a __strong object.
1011    llvm::Value *LvalueDst = Dst.getAddress();
1012    llvm::Value *src = Src.getScalarVal();
1013    if (Dst.isObjCIvar()) {
1014      assert(Dst.getBaseIvarExp() && "BaseIvarExp is NULL");
1015      llvm::Type *ResultType = ConvertType(getContext().LongTy);
1016      llvm::Value *RHS = EmitScalarExpr(Dst.getBaseIvarExp());
1017      llvm::Value *dst = RHS;
1018      RHS = Builder.CreatePtrToInt(RHS, ResultType, "sub.ptr.rhs.cast");
1019      llvm::Value *LHS =
1020        Builder.CreatePtrToInt(LvalueDst, ResultType, "sub.ptr.lhs.cast");
1021      llvm::Value *BytesBetween = Builder.CreateSub(LHS, RHS, "ivar.offset");
1022      CGM.getObjCRuntime().EmitObjCIvarAssign(*this, src, dst,
1023                                              BytesBetween);
1024    } else if (Dst.isGlobalObjCRef()) {
1025      CGM.getObjCRuntime().EmitObjCGlobalAssign(*this, src, LvalueDst,
1026                                                Dst.isThreadLocalRef());
1027    }
1028    else
1029      CGM.getObjCRuntime().EmitObjCStrongCastAssign(*this, src, LvalueDst);
1030    return;
1031  }
1032
1033  assert(Src.isScalar() && "Can't emit an agg store with this method");
1034  EmitStoreOfScalar(Src.getScalarVal(), Dst);
1035}
1036
1037void CodeGenFunction::EmitStoreThroughBitfieldLValue(RValue Src, LValue Dst,
1038                                                     llvm::Value **Result) {
1039  const CGBitFieldInfo &Info = Dst.getBitFieldInfo();
1040
1041  // Get the output type.
1042  llvm::Type *ResLTy = ConvertTypeForMem(Dst.getType());
1043  unsigned ResSizeInBits = CGM.getTargetData().getTypeSizeInBits(ResLTy);
1044
1045  // Get the source value, truncated to the width of the bit-field.
1046  llvm::Value *SrcVal = Src.getScalarVal();
1047
1048  if (Dst.getType()->isBooleanType())
1049    SrcVal = Builder.CreateIntCast(SrcVal, ResLTy, /*IsSigned=*/false);
1050
1051  SrcVal = Builder.CreateAnd(SrcVal, llvm::APInt::getLowBitsSet(ResSizeInBits,
1052                                                                Info.getSize()),
1053                             "bf.value");
1054
1055  // Return the new value of the bit-field, if requested.
1056  if (Result) {
1057    // Cast back to the proper type for result.
1058    llvm::Type *SrcTy = Src.getScalarVal()->getType();
1059    llvm::Value *ReloadVal = Builder.CreateIntCast(SrcVal, SrcTy, false,
1060                                                   "bf.reload.val");
1061
1062    // Sign extend if necessary.
1063    if (Info.isSigned()) {
1064      unsigned ExtraBits = ResSizeInBits - Info.getSize();
1065      if (ExtraBits)
1066        ReloadVal = Builder.CreateAShr(Builder.CreateShl(ReloadVal, ExtraBits),
1067                                       ExtraBits, "bf.reload.sext");
1068    }
1069
1070    *Result = ReloadVal;
1071  }
1072
1073  // Iterate over the components, writing each piece to memory.
1074  for (unsigned i = 0, e = Info.getNumComponents(); i != e; ++i) {
1075    const CGBitFieldInfo::AccessInfo &AI = Info.getComponent(i);
1076
1077    // Get the field pointer.
1078    llvm::Value *Ptr = Dst.getBitFieldBaseAddr();
1079    unsigned addressSpace =
1080      cast<llvm::PointerType>(Ptr->getType())->getAddressSpace();
1081
1082    // Only offset by the field index if used, so that incoming values are not
1083    // required to be structures.
1084    if (AI.FieldIndex)
1085      Ptr = Builder.CreateStructGEP(Ptr, AI.FieldIndex, "bf.field");
1086
1087    // Offset by the byte offset, if used.
1088    if (!AI.FieldByteOffset.isZero()) {
1089      Ptr = EmitCastToVoidPtr(Ptr);
1090      Ptr = Builder.CreateConstGEP1_32(Ptr, AI.FieldByteOffset.getQuantity(),
1091                                       "bf.field.offs");
1092    }
1093
1094    // Cast to the access type.
1095    llvm::Type *AccessLTy =
1096      llvm::Type::getIntNTy(getLLVMContext(), AI.AccessWidth);
1097
1098    llvm::Type *PTy = AccessLTy->getPointerTo(addressSpace);
1099    Ptr = Builder.CreateBitCast(Ptr, PTy);
1100
1101    // Extract the piece of the bit-field value to write in this access, limited
1102    // to the values that are part of this access.
1103    llvm::Value *Val = SrcVal;
1104    if (AI.TargetBitOffset)
1105      Val = Builder.CreateLShr(Val, AI.TargetBitOffset);
1106    Val = Builder.CreateAnd(Val, llvm::APInt::getLowBitsSet(ResSizeInBits,
1107                                                            AI.TargetBitWidth));
1108
1109    // Extend or truncate to the access size.
1110    if (ResSizeInBits < AI.AccessWidth)
1111      Val = Builder.CreateZExt(Val, AccessLTy);
1112    else if (ResSizeInBits > AI.AccessWidth)
1113      Val = Builder.CreateTrunc(Val, AccessLTy);
1114
1115    // Shift into the position in memory.
1116    if (AI.FieldBitStart)
1117      Val = Builder.CreateShl(Val, AI.FieldBitStart);
1118
1119    // If necessary, load and OR in bits that are outside of the bit-field.
1120    if (AI.TargetBitWidth != AI.AccessWidth) {
1121      llvm::LoadInst *Load = Builder.CreateLoad(Ptr, Dst.isVolatileQualified());
1122      if (!AI.AccessAlignment.isZero())
1123        Load->setAlignment(AI.AccessAlignment.getQuantity());
1124
1125      // Compute the mask for zeroing the bits that are part of the bit-field.
1126      llvm::APInt InvMask =
1127        ~llvm::APInt::getBitsSet(AI.AccessWidth, AI.FieldBitStart,
1128                                 AI.FieldBitStart + AI.TargetBitWidth);
1129
1130      // Apply the mask and OR in to the value to write.
1131      Val = Builder.CreateOr(Builder.CreateAnd(Load, InvMask), Val);
1132    }
1133
1134    // Write the value.
1135    llvm::StoreInst *Store = Builder.CreateStore(Val, Ptr,
1136                                                 Dst.isVolatileQualified());
1137    if (!AI.AccessAlignment.isZero())
1138      Store->setAlignment(AI.AccessAlignment.getQuantity());
1139  }
1140}
1141
1142void CodeGenFunction::EmitStoreThroughExtVectorComponentLValue(RValue Src,
1143                                                               LValue Dst) {
1144  // This access turns into a read/modify/write of the vector.  Load the input
1145  // value now.
1146  llvm::Value *Vec = Builder.CreateLoad(Dst.getExtVectorAddr(),
1147                                        Dst.isVolatileQualified());
1148  const llvm::Constant *Elts = Dst.getExtVectorElts();
1149
1150  llvm::Value *SrcVal = Src.getScalarVal();
1151
1152  if (const VectorType *VTy = Dst.getType()->getAs<VectorType>()) {
1153    unsigned NumSrcElts = VTy->getNumElements();
1154    unsigned NumDstElts =
1155       cast<llvm::VectorType>(Vec->getType())->getNumElements();
1156    if (NumDstElts == NumSrcElts) {
1157      // Use shuffle vector is the src and destination are the same number of
1158      // elements and restore the vector mask since it is on the side it will be
1159      // stored.
1160      SmallVector<llvm::Constant*, 4> Mask(NumDstElts);
1161      for (unsigned i = 0; i != NumSrcElts; ++i) {
1162        unsigned InIdx = getAccessedFieldNo(i, Elts);
1163        Mask[InIdx] = llvm::ConstantInt::get(Int32Ty, i);
1164      }
1165
1166      llvm::Value *MaskV = llvm::ConstantVector::get(Mask);
1167      Vec = Builder.CreateShuffleVector(SrcVal,
1168                                        llvm::UndefValue::get(Vec->getType()),
1169                                        MaskV);
1170    } else if (NumDstElts > NumSrcElts) {
1171      // Extended the source vector to the same length and then shuffle it
1172      // into the destination.
1173      // FIXME: since we're shuffling with undef, can we just use the indices
1174      //        into that?  This could be simpler.
1175      SmallVector<llvm::Constant*, 4> ExtMask;
1176      unsigned i;
1177      for (i = 0; i != NumSrcElts; ++i)
1178        ExtMask.push_back(llvm::ConstantInt::get(Int32Ty, i));
1179      for (; i != NumDstElts; ++i)
1180        ExtMask.push_back(llvm::UndefValue::get(Int32Ty));
1181      llvm::Value *ExtMaskV = llvm::ConstantVector::get(ExtMask);
1182      llvm::Value *ExtSrcVal =
1183        Builder.CreateShuffleVector(SrcVal,
1184                                    llvm::UndefValue::get(SrcVal->getType()),
1185                                    ExtMaskV);
1186      // build identity
1187      SmallVector<llvm::Constant*, 4> Mask;
1188      for (unsigned i = 0; i != NumDstElts; ++i)
1189        Mask.push_back(llvm::ConstantInt::get(Int32Ty, i));
1190
1191      // modify when what gets shuffled in
1192      for (unsigned i = 0; i != NumSrcElts; ++i) {
1193        unsigned Idx = getAccessedFieldNo(i, Elts);
1194        Mask[Idx] = llvm::ConstantInt::get(Int32Ty, i+NumDstElts);
1195      }
1196      llvm::Value *MaskV = llvm::ConstantVector::get(Mask);
1197      Vec = Builder.CreateShuffleVector(Vec, ExtSrcVal, MaskV);
1198    } else {
1199      // We should never shorten the vector
1200      llvm_unreachable("unexpected shorten vector length");
1201    }
1202  } else {
1203    // If the Src is a scalar (not a vector) it must be updating one element.
1204    unsigned InIdx = getAccessedFieldNo(0, Elts);
1205    llvm::Value *Elt = llvm::ConstantInt::get(Int32Ty, InIdx);
1206    Vec = Builder.CreateInsertElement(Vec, SrcVal, Elt);
1207  }
1208
1209  Builder.CreateStore(Vec, Dst.getExtVectorAddr(), Dst.isVolatileQualified());
1210}
1211
1212// setObjCGCLValueClass - sets class of he lvalue for the purpose of
1213// generating write-barries API. It is currently a global, ivar,
1214// or neither.
1215static void setObjCGCLValueClass(const ASTContext &Ctx, const Expr *E,
1216                                 LValue &LV,
1217                                 bool IsMemberAccess=false) {
1218  if (Ctx.getLangOptions().getGC() == LangOptions::NonGC)
1219    return;
1220
1221  if (isa<ObjCIvarRefExpr>(E)) {
1222    QualType ExpTy = E->getType();
1223    if (IsMemberAccess && ExpTy->isPointerType()) {
1224      // If ivar is a structure pointer, assigning to field of
1225      // this struct follows gcc's behavior and makes it a non-ivar
1226      // writer-barrier conservatively.
1227      ExpTy = ExpTy->getAs<PointerType>()->getPointeeType();
1228      if (ExpTy->isRecordType()) {
1229        LV.setObjCIvar(false);
1230        return;
1231      }
1232    }
1233    LV.setObjCIvar(true);
1234    ObjCIvarRefExpr *Exp = cast<ObjCIvarRefExpr>(const_cast<Expr*>(E));
1235    LV.setBaseIvarExp(Exp->getBase());
1236    LV.setObjCArray(E->getType()->isArrayType());
1237    return;
1238  }
1239
1240  if (const DeclRefExpr *Exp = dyn_cast<DeclRefExpr>(E)) {
1241    if (const VarDecl *VD = dyn_cast<VarDecl>(Exp->getDecl())) {
1242      if (VD->hasGlobalStorage()) {
1243        LV.setGlobalObjCRef(true);
1244        LV.setThreadLocalRef(VD->isThreadSpecified());
1245      }
1246    }
1247    LV.setObjCArray(E->getType()->isArrayType());
1248    return;
1249  }
1250
1251  if (const UnaryOperator *Exp = dyn_cast<UnaryOperator>(E)) {
1252    setObjCGCLValueClass(Ctx, Exp->getSubExpr(), LV, IsMemberAccess);
1253    return;
1254  }
1255
1256  if (const ParenExpr *Exp = dyn_cast<ParenExpr>(E)) {
1257    setObjCGCLValueClass(Ctx, Exp->getSubExpr(), LV, IsMemberAccess);
1258    if (LV.isObjCIvar()) {
1259      // If cast is to a structure pointer, follow gcc's behavior and make it
1260      // a non-ivar write-barrier.
1261      QualType ExpTy = E->getType();
1262      if (ExpTy->isPointerType())
1263        ExpTy = ExpTy->getAs<PointerType>()->getPointeeType();
1264      if (ExpTy->isRecordType())
1265        LV.setObjCIvar(false);
1266    }
1267    return;
1268  }
1269
1270  if (const GenericSelectionExpr *Exp = dyn_cast<GenericSelectionExpr>(E)) {
1271    setObjCGCLValueClass(Ctx, Exp->getResultExpr(), LV);
1272    return;
1273  }
1274
1275  if (const ImplicitCastExpr *Exp = dyn_cast<ImplicitCastExpr>(E)) {
1276    setObjCGCLValueClass(Ctx, Exp->getSubExpr(), LV, IsMemberAccess);
1277    return;
1278  }
1279
1280  if (const CStyleCastExpr *Exp = dyn_cast<CStyleCastExpr>(E)) {
1281    setObjCGCLValueClass(Ctx, Exp->getSubExpr(), LV, IsMemberAccess);
1282    return;
1283  }
1284
1285  if (const ObjCBridgedCastExpr *Exp = dyn_cast<ObjCBridgedCastExpr>(E)) {
1286    setObjCGCLValueClass(Ctx, Exp->getSubExpr(), LV, IsMemberAccess);
1287    return;
1288  }
1289
1290  if (const ArraySubscriptExpr *Exp = dyn_cast<ArraySubscriptExpr>(E)) {
1291    setObjCGCLValueClass(Ctx, Exp->getBase(), LV);
1292    if (LV.isObjCIvar() && !LV.isObjCArray())
1293      // Using array syntax to assigning to what an ivar points to is not
1294      // same as assigning to the ivar itself. {id *Names;} Names[i] = 0;
1295      LV.setObjCIvar(false);
1296    else if (LV.isGlobalObjCRef() && !LV.isObjCArray())
1297      // Using array syntax to assigning to what global points to is not
1298      // same as assigning to the global itself. {id *G;} G[i] = 0;
1299      LV.setGlobalObjCRef(false);
1300    return;
1301  }
1302
1303  if (const MemberExpr *Exp = dyn_cast<MemberExpr>(E)) {
1304    setObjCGCLValueClass(Ctx, Exp->getBase(), LV, true);
1305    // We don't know if member is an 'ivar', but this flag is looked at
1306    // only in the context of LV.isObjCIvar().
1307    LV.setObjCArray(E->getType()->isArrayType());
1308    return;
1309  }
1310}
1311
1312static llvm::Value *
1313EmitBitCastOfLValueToProperType(CodeGenFunction &CGF,
1314                                llvm::Value *V, llvm::Type *IRType,
1315                                StringRef Name = StringRef()) {
1316  unsigned AS = cast<llvm::PointerType>(V->getType())->getAddressSpace();
1317  return CGF.Builder.CreateBitCast(V, IRType->getPointerTo(AS), Name);
1318}
1319
1320static LValue EmitGlobalVarDeclLValue(CodeGenFunction &CGF,
1321                                      const Expr *E, const VarDecl *VD) {
1322  assert((VD->hasExternalStorage() || VD->isFileVarDecl()) &&
1323         "Var decl must have external storage or be a file var decl!");
1324
1325  llvm::Value *V = CGF.CGM.GetAddrOfGlobalVar(VD);
1326  if (VD->getType()->isReferenceType())
1327    V = CGF.Builder.CreateLoad(V);
1328
1329  V = EmitBitCastOfLValueToProperType(CGF, V,
1330                                CGF.getTypes().ConvertTypeForMem(E->getType()));
1331
1332  unsigned Alignment = CGF.getContext().getDeclAlign(VD).getQuantity();
1333  LValue LV = CGF.MakeAddrLValue(V, E->getType(), Alignment);
1334  setObjCGCLValueClass(CGF.getContext(), E, LV);
1335  return LV;
1336}
1337
1338static LValue EmitFunctionDeclLValue(CodeGenFunction &CGF,
1339                                     const Expr *E, const FunctionDecl *FD) {
1340  llvm::Value *V = CGF.CGM.GetAddrOfFunction(FD);
1341  if (!FD->hasPrototype()) {
1342    if (const FunctionProtoType *Proto =
1343            FD->getType()->getAs<FunctionProtoType>()) {
1344      // Ugly case: for a K&R-style definition, the type of the definition
1345      // isn't the same as the type of a use.  Correct for this with a
1346      // bitcast.
1347      QualType NoProtoType =
1348          CGF.getContext().getFunctionNoProtoType(Proto->getResultType());
1349      NoProtoType = CGF.getContext().getPointerType(NoProtoType);
1350      V = CGF.Builder.CreateBitCast(V, CGF.ConvertType(NoProtoType));
1351    }
1352  }
1353  unsigned Alignment = CGF.getContext().getDeclAlign(FD).getQuantity();
1354  return CGF.MakeAddrLValue(V, E->getType(), Alignment);
1355}
1356
1357LValue CodeGenFunction::EmitDeclRefLValue(const DeclRefExpr *E) {
1358  const NamedDecl *ND = E->getDecl();
1359  unsigned Alignment = getContext().getDeclAlign(ND).getQuantity();
1360
1361  if (ND->hasAttr<WeakRefAttr>()) {
1362    const ValueDecl *VD = cast<ValueDecl>(ND);
1363    llvm::Constant *Aliasee = CGM.GetWeakRefReference(VD);
1364    return MakeAddrLValue(Aliasee, E->getType(), Alignment);
1365  }
1366
1367  if (const VarDecl *VD = dyn_cast<VarDecl>(ND)) {
1368
1369    // Check if this is a global variable.
1370    if (VD->hasExternalStorage() || VD->isFileVarDecl())
1371      return EmitGlobalVarDeclLValue(*this, E, VD);
1372
1373    bool NonGCable = VD->hasLocalStorage() &&
1374                     !VD->getType()->isReferenceType() &&
1375                     !VD->hasAttr<BlocksAttr>();
1376
1377    llvm::Value *V = LocalDeclMap[VD];
1378    if (!V && VD->isStaticLocal())
1379      V = CGM.getStaticLocalDeclAddress(VD);
1380    assert(V && "DeclRefExpr not entered in LocalDeclMap?");
1381
1382    if (VD->hasAttr<BlocksAttr>())
1383      V = BuildBlockByrefAddress(V, VD);
1384
1385    if (VD->getType()->isReferenceType())
1386      V = Builder.CreateLoad(V);
1387
1388    V = EmitBitCastOfLValueToProperType(*this, V,
1389                                    getTypes().ConvertTypeForMem(E->getType()));
1390
1391    LValue LV = MakeAddrLValue(V, E->getType(), Alignment);
1392    if (NonGCable) {
1393      LV.getQuals().removeObjCGCAttr();
1394      LV.setNonGC(true);
1395    }
1396    setObjCGCLValueClass(getContext(), E, LV);
1397    return LV;
1398  }
1399
1400  if (const FunctionDecl *fn = dyn_cast<FunctionDecl>(ND))
1401    return EmitFunctionDeclLValue(*this, E, fn);
1402
1403  llvm_unreachable("Unhandled DeclRefExpr");
1404
1405  // an invalid LValue, but the assert will
1406  // ensure that this point is never reached.
1407  return LValue();
1408}
1409
1410LValue CodeGenFunction::EmitBlockDeclRefLValue(const BlockDeclRefExpr *E) {
1411  unsigned Alignment =
1412    getContext().getDeclAlign(E->getDecl()).getQuantity();
1413  return MakeAddrLValue(GetAddrOfBlockDecl(E), E->getType(), Alignment);
1414}
1415
1416LValue CodeGenFunction::EmitUnaryOpLValue(const UnaryOperator *E) {
1417  // __extension__ doesn't affect lvalue-ness.
1418  if (E->getOpcode() == UO_Extension)
1419    return EmitLValue(E->getSubExpr());
1420
1421  QualType ExprTy = getContext().getCanonicalType(E->getSubExpr()->getType());
1422  switch (E->getOpcode()) {
1423  default: llvm_unreachable("Unknown unary operator lvalue!");
1424  case UO_Deref: {
1425    QualType T = E->getSubExpr()->getType()->getPointeeType();
1426    assert(!T.isNull() && "CodeGenFunction::EmitUnaryOpLValue: Illegal type");
1427
1428    LValue LV = MakeAddrLValue(EmitScalarExpr(E->getSubExpr()), T);
1429    LV.getQuals().setAddressSpace(ExprTy.getAddressSpace());
1430
1431    // We should not generate __weak write barrier on indirect reference
1432    // of a pointer to object; as in void foo (__weak id *param); *param = 0;
1433    // But, we continue to generate __strong write barrier on indirect write
1434    // into a pointer to object.
1435    if (getContext().getLangOptions().ObjC1 &&
1436        getContext().getLangOptions().getGC() != LangOptions::NonGC &&
1437        LV.isObjCWeak())
1438      LV.setNonGC(!E->isOBJCGCCandidate(getContext()));
1439    return LV;
1440  }
1441  case UO_Real:
1442  case UO_Imag: {
1443    LValue LV = EmitLValue(E->getSubExpr());
1444    assert(LV.isSimple() && "real/imag on non-ordinary l-value");
1445    llvm::Value *Addr = LV.getAddress();
1446
1447    // real and imag are valid on scalars.  This is a faster way of
1448    // testing that.
1449    if (!cast<llvm::PointerType>(Addr->getType())
1450           ->getElementType()->isStructTy()) {
1451      assert(E->getSubExpr()->getType()->isArithmeticType());
1452      return LV;
1453    }
1454
1455    assert(E->getSubExpr()->getType()->isAnyComplexType());
1456
1457    unsigned Idx = E->getOpcode() == UO_Imag;
1458    return MakeAddrLValue(Builder.CreateStructGEP(LV.getAddress(),
1459                                                  Idx, "idx"),
1460                          ExprTy);
1461  }
1462  case UO_PreInc:
1463  case UO_PreDec: {
1464    LValue LV = EmitLValue(E->getSubExpr());
1465    bool isInc = E->getOpcode() == UO_PreInc;
1466
1467    if (E->getType()->isAnyComplexType())
1468      EmitComplexPrePostIncDec(E, LV, isInc, true/*isPre*/);
1469    else
1470      EmitScalarPrePostIncDec(E, LV, isInc, true/*isPre*/);
1471    return LV;
1472  }
1473  }
1474}
1475
1476LValue CodeGenFunction::EmitStringLiteralLValue(const StringLiteral *E) {
1477  return MakeAddrLValue(CGM.GetAddrOfConstantStringFromLiteral(E),
1478                        E->getType());
1479}
1480
1481LValue CodeGenFunction::EmitObjCEncodeExprLValue(const ObjCEncodeExpr *E) {
1482  return MakeAddrLValue(CGM.GetAddrOfConstantStringFromObjCEncode(E),
1483                        E->getType());
1484}
1485
1486
1487LValue CodeGenFunction::EmitPredefinedLValue(const PredefinedExpr *E) {
1488  switch (E->getIdentType()) {
1489  default:
1490    return EmitUnsupportedLValue(E, "predefined expression");
1491
1492  case PredefinedExpr::Func:
1493  case PredefinedExpr::Function:
1494  case PredefinedExpr::PrettyFunction: {
1495    unsigned Type = E->getIdentType();
1496    std::string GlobalVarName;
1497
1498    switch (Type) {
1499    default: llvm_unreachable("Invalid type");
1500    case PredefinedExpr::Func:
1501      GlobalVarName = "__func__.";
1502      break;
1503    case PredefinedExpr::Function:
1504      GlobalVarName = "__FUNCTION__.";
1505      break;
1506    case PredefinedExpr::PrettyFunction:
1507      GlobalVarName = "__PRETTY_FUNCTION__.";
1508      break;
1509    }
1510
1511    StringRef FnName = CurFn->getName();
1512    if (FnName.startswith("\01"))
1513      FnName = FnName.substr(1);
1514    GlobalVarName += FnName;
1515
1516    const Decl *CurDecl = CurCodeDecl;
1517    if (CurDecl == 0)
1518      CurDecl = getContext().getTranslationUnitDecl();
1519
1520    std::string FunctionName =
1521        (isa<BlockDecl>(CurDecl)
1522         ? FnName.str()
1523         : PredefinedExpr::ComputeName((PredefinedExpr::IdentType)Type, CurDecl));
1524
1525    llvm::Constant *C =
1526      CGM.GetAddrOfConstantCString(FunctionName, GlobalVarName.c_str());
1527    return MakeAddrLValue(C, E->getType());
1528  }
1529  }
1530}
1531
1532llvm::BasicBlock *CodeGenFunction::getTrapBB() {
1533  const CodeGenOptions &GCO = CGM.getCodeGenOpts();
1534
1535  // If we are not optimzing, don't collapse all calls to trap in the function
1536  // to the same call, that way, in the debugger they can see which operation
1537  // did in fact fail.  If we are optimizing, we collapse all calls to trap down
1538  // to just one per function to save on codesize.
1539  if (GCO.OptimizationLevel && TrapBB)
1540    return TrapBB;
1541
1542  llvm::BasicBlock *Cont = 0;
1543  if (HaveInsertPoint()) {
1544    Cont = createBasicBlock("cont");
1545    EmitBranch(Cont);
1546  }
1547  TrapBB = createBasicBlock("trap");
1548  EmitBlock(TrapBB);
1549
1550  llvm::Value *F = CGM.getIntrinsic(llvm::Intrinsic::trap);
1551  llvm::CallInst *TrapCall = Builder.CreateCall(F);
1552  TrapCall->setDoesNotReturn();
1553  TrapCall->setDoesNotThrow();
1554  Builder.CreateUnreachable();
1555
1556  if (Cont)
1557    EmitBlock(Cont);
1558  return TrapBB;
1559}
1560
1561/// isSimpleArrayDecayOperand - If the specified expr is a simple decay from an
1562/// array to pointer, return the array subexpression.
1563static const Expr *isSimpleArrayDecayOperand(const Expr *E) {
1564  // If this isn't just an array->pointer decay, bail out.
1565  const CastExpr *CE = dyn_cast<CastExpr>(E);
1566  if (CE == 0 || CE->getCastKind() != CK_ArrayToPointerDecay)
1567    return 0;
1568
1569  // If this is a decay from variable width array, bail out.
1570  const Expr *SubExpr = CE->getSubExpr();
1571  if (SubExpr->getType()->isVariableArrayType())
1572    return 0;
1573
1574  return SubExpr;
1575}
1576
1577LValue CodeGenFunction::EmitArraySubscriptExpr(const ArraySubscriptExpr *E) {
1578  // The index must always be an integer, which is not an aggregate.  Emit it.
1579  llvm::Value *Idx = EmitScalarExpr(E->getIdx());
1580  QualType IdxTy  = E->getIdx()->getType();
1581  bool IdxSigned = IdxTy->isSignedIntegerOrEnumerationType();
1582
1583  // If the base is a vector type, then we are forming a vector element lvalue
1584  // with this subscript.
1585  if (E->getBase()->getType()->isVectorType()) {
1586    // Emit the vector as an lvalue to get its address.
1587    LValue LHS = EmitLValue(E->getBase());
1588    assert(LHS.isSimple() && "Can only subscript lvalue vectors here!");
1589    Idx = Builder.CreateIntCast(Idx, Int32Ty, IdxSigned, "vidx");
1590    return LValue::MakeVectorElt(LHS.getAddress(), Idx,
1591                                 E->getBase()->getType());
1592  }
1593
1594  // Extend or truncate the index type to 32 or 64-bits.
1595  if (Idx->getType() != IntPtrTy)
1596    Idx = Builder.CreateIntCast(Idx, IntPtrTy, IdxSigned, "idxprom");
1597
1598  // FIXME: As llvm implements the object size checking, this can come out.
1599  if (CatchUndefined) {
1600    if (const ImplicitCastExpr *ICE = dyn_cast<ImplicitCastExpr>(E->getBase())){
1601      if (const DeclRefExpr *DRE = dyn_cast<DeclRefExpr>(ICE->getSubExpr())) {
1602        if (ICE->getCastKind() == CK_ArrayToPointerDecay) {
1603          if (const ConstantArrayType *CAT
1604              = getContext().getAsConstantArrayType(DRE->getType())) {
1605            llvm::APInt Size = CAT->getSize();
1606            llvm::BasicBlock *Cont = createBasicBlock("cont");
1607            Builder.CreateCondBr(Builder.CreateICmpULE(Idx,
1608                                  llvm::ConstantInt::get(Idx->getType(), Size)),
1609                                 Cont, getTrapBB());
1610            EmitBlock(Cont);
1611          }
1612        }
1613      }
1614    }
1615  }
1616
1617  // We know that the pointer points to a type of the correct size, unless the
1618  // size is a VLA or Objective-C interface.
1619  llvm::Value *Address = 0;
1620  unsigned ArrayAlignment = 0;
1621  if (const VariableArrayType *vla =
1622        getContext().getAsVariableArrayType(E->getType())) {
1623    // The base must be a pointer, which is not an aggregate.  Emit
1624    // it.  It needs to be emitted first in case it's what captures
1625    // the VLA bounds.
1626    Address = EmitScalarExpr(E->getBase());
1627
1628    // The element count here is the total number of non-VLA elements.
1629    llvm::Value *numElements = getVLASize(vla).first;
1630
1631    // Effectively, the multiply by the VLA size is part of the GEP.
1632    // GEP indexes are signed, and scaling an index isn't permitted to
1633    // signed-overflow, so we use the same semantics for our explicit
1634    // multiply.  We suppress this if overflow is not undefined behavior.
1635    if (getLangOptions().isSignedOverflowDefined()) {
1636      Idx = Builder.CreateMul(Idx, numElements);
1637      Address = Builder.CreateGEP(Address, Idx, "arrayidx");
1638    } else {
1639      Idx = Builder.CreateNSWMul(Idx, numElements);
1640      Address = Builder.CreateInBoundsGEP(Address, Idx, "arrayidx");
1641    }
1642  } else if (const ObjCObjectType *OIT = E->getType()->getAs<ObjCObjectType>()){
1643    // Indexing over an interface, as in "NSString *P; P[4];"
1644    llvm::Value *InterfaceSize =
1645      llvm::ConstantInt::get(Idx->getType(),
1646          getContext().getTypeSizeInChars(OIT).getQuantity());
1647
1648    Idx = Builder.CreateMul(Idx, InterfaceSize);
1649
1650    // The base must be a pointer, which is not an aggregate.  Emit it.
1651    llvm::Value *Base = EmitScalarExpr(E->getBase());
1652    Address = EmitCastToVoidPtr(Base);
1653    Address = Builder.CreateGEP(Address, Idx, "arrayidx");
1654    Address = Builder.CreateBitCast(Address, Base->getType());
1655  } else if (const Expr *Array = isSimpleArrayDecayOperand(E->getBase())) {
1656    // If this is A[i] where A is an array, the frontend will have decayed the
1657    // base to be a ArrayToPointerDecay implicit cast.  While correct, it is
1658    // inefficient at -O0 to emit a "gep A, 0, 0" when codegen'ing it, then a
1659    // "gep x, i" here.  Emit one "gep A, 0, i".
1660    assert(Array->getType()->isArrayType() &&
1661           "Array to pointer decay must have array source type!");
1662    LValue ArrayLV = EmitLValue(Array);
1663    llvm::Value *ArrayPtr = ArrayLV.getAddress();
1664    llvm::Value *Zero = llvm::ConstantInt::get(Int32Ty, 0);
1665    llvm::Value *Args[] = { Zero, Idx };
1666
1667    // Propagate the alignment from the array itself to the result.
1668    ArrayAlignment = ArrayLV.getAlignment();
1669
1670    if (getContext().getLangOptions().isSignedOverflowDefined())
1671      Address = Builder.CreateGEP(ArrayPtr, Args, "arrayidx");
1672    else
1673      Address = Builder.CreateInBoundsGEP(ArrayPtr, Args, "arrayidx");
1674  } else {
1675    // The base must be a pointer, which is not an aggregate.  Emit it.
1676    llvm::Value *Base = EmitScalarExpr(E->getBase());
1677    if (getContext().getLangOptions().isSignedOverflowDefined())
1678      Address = Builder.CreateGEP(Base, Idx, "arrayidx");
1679    else
1680      Address = Builder.CreateInBoundsGEP(Base, Idx, "arrayidx");
1681  }
1682
1683  QualType T = E->getBase()->getType()->getPointeeType();
1684  assert(!T.isNull() &&
1685         "CodeGenFunction::EmitArraySubscriptExpr(): Illegal base type");
1686
1687  // Limit the alignment to that of the result type.
1688  if (ArrayAlignment) {
1689    unsigned Align = getContext().getTypeAlignInChars(T).getQuantity();
1690    ArrayAlignment = std::min(Align, ArrayAlignment);
1691  }
1692
1693  LValue LV = MakeAddrLValue(Address, T, ArrayAlignment);
1694  LV.getQuals().setAddressSpace(E->getBase()->getType().getAddressSpace());
1695
1696  if (getContext().getLangOptions().ObjC1 &&
1697      getContext().getLangOptions().getGC() != LangOptions::NonGC) {
1698    LV.setNonGC(!E->isOBJCGCCandidate(getContext()));
1699    setObjCGCLValueClass(getContext(), E, LV);
1700  }
1701  return LV;
1702}
1703
1704static
1705llvm::Constant *GenerateConstantVector(llvm::LLVMContext &VMContext,
1706                                       SmallVector<unsigned, 4> &Elts) {
1707  SmallVector<llvm::Constant*, 4> CElts;
1708
1709  llvm::Type *Int32Ty = llvm::Type::getInt32Ty(VMContext);
1710  for (unsigned i = 0, e = Elts.size(); i != e; ++i)
1711    CElts.push_back(llvm::ConstantInt::get(Int32Ty, Elts[i]));
1712
1713  return llvm::ConstantVector::get(CElts);
1714}
1715
1716LValue CodeGenFunction::
1717EmitExtVectorElementExpr(const ExtVectorElementExpr *E) {
1718  // Emit the base vector as an l-value.
1719  LValue Base;
1720
1721  // ExtVectorElementExpr's base can either be a vector or pointer to vector.
1722  if (E->isArrow()) {
1723    // If it is a pointer to a vector, emit the address and form an lvalue with
1724    // it.
1725    llvm::Value *Ptr = EmitScalarExpr(E->getBase());
1726    const PointerType *PT = E->getBase()->getType()->getAs<PointerType>();
1727    Base = MakeAddrLValue(Ptr, PT->getPointeeType());
1728    Base.getQuals().removeObjCGCAttr();
1729  } else if (E->getBase()->isGLValue()) {
1730    // Otherwise, if the base is an lvalue ( as in the case of foo.x.x),
1731    // emit the base as an lvalue.
1732    assert(E->getBase()->getType()->isVectorType());
1733    Base = EmitLValue(E->getBase());
1734  } else {
1735    // Otherwise, the base is a normal rvalue (as in (V+V).x), emit it as such.
1736    assert(E->getBase()->getType()->isVectorType() &&
1737           "Result must be a vector");
1738    llvm::Value *Vec = EmitScalarExpr(E->getBase());
1739
1740    // Store the vector to memory (because LValue wants an address).
1741    llvm::Value *VecMem = CreateMemTemp(E->getBase()->getType());
1742    Builder.CreateStore(Vec, VecMem);
1743    Base = MakeAddrLValue(VecMem, E->getBase()->getType());
1744  }
1745
1746  QualType type =
1747    E->getType().withCVRQualifiers(Base.getQuals().getCVRQualifiers());
1748
1749  // Encode the element access list into a vector of unsigned indices.
1750  SmallVector<unsigned, 4> Indices;
1751  E->getEncodedElementAccess(Indices);
1752
1753  if (Base.isSimple()) {
1754    llvm::Constant *CV = GenerateConstantVector(getLLVMContext(), Indices);
1755    return LValue::MakeExtVectorElt(Base.getAddress(), CV, type);
1756  }
1757  assert(Base.isExtVectorElt() && "Can only subscript lvalue vec elts here!");
1758
1759  llvm::Constant *BaseElts = Base.getExtVectorElts();
1760  SmallVector<llvm::Constant *, 4> CElts;
1761
1762  for (unsigned i = 0, e = Indices.size(); i != e; ++i) {
1763    if (isa<llvm::ConstantAggregateZero>(BaseElts))
1764      CElts.push_back(llvm::ConstantInt::get(Int32Ty, 0));
1765    else
1766      CElts.push_back(cast<llvm::Constant>(BaseElts->getOperand(Indices[i])));
1767  }
1768  llvm::Constant *CV = llvm::ConstantVector::get(CElts);
1769  return LValue::MakeExtVectorElt(Base.getExtVectorAddr(), CV, type);
1770}
1771
1772LValue CodeGenFunction::EmitMemberExpr(const MemberExpr *E) {
1773  bool isNonGC = false;
1774  Expr *BaseExpr = E->getBase();
1775  llvm::Value *BaseValue = NULL;
1776  Qualifiers BaseQuals;
1777
1778  // If this is s.x, emit s as an lvalue.  If it is s->x, emit s as a scalar.
1779  if (E->isArrow()) {
1780    BaseValue = EmitScalarExpr(BaseExpr);
1781    const PointerType *PTy =
1782      BaseExpr->getType()->getAs<PointerType>();
1783    BaseQuals = PTy->getPointeeType().getQualifiers();
1784  } else {
1785    LValue BaseLV = EmitLValue(BaseExpr);
1786    if (BaseLV.isNonGC())
1787      isNonGC = true;
1788    // FIXME: this isn't right for bitfields.
1789    BaseValue = BaseLV.getAddress();
1790    QualType BaseTy = BaseExpr->getType();
1791    BaseQuals = BaseTy.getQualifiers();
1792  }
1793
1794  NamedDecl *ND = E->getMemberDecl();
1795  if (FieldDecl *Field = dyn_cast<FieldDecl>(ND)) {
1796    LValue LV = EmitLValueForField(BaseValue, Field,
1797                                   BaseQuals.getCVRQualifiers());
1798    LV.setNonGC(isNonGC);
1799    setObjCGCLValueClass(getContext(), E, LV);
1800    return LV;
1801  }
1802
1803  if (VarDecl *VD = dyn_cast<VarDecl>(ND))
1804    return EmitGlobalVarDeclLValue(*this, E, VD);
1805
1806  if (const FunctionDecl *FD = dyn_cast<FunctionDecl>(ND))
1807    return EmitFunctionDeclLValue(*this, E, FD);
1808
1809  llvm_unreachable("Unhandled member declaration!");
1810}
1811
1812LValue CodeGenFunction::EmitLValueForBitfield(llvm::Value *BaseValue,
1813                                              const FieldDecl *Field,
1814                                              unsigned CVRQualifiers) {
1815  const CGRecordLayout &RL =
1816    CGM.getTypes().getCGRecordLayout(Field->getParent());
1817  const CGBitFieldInfo &Info = RL.getBitFieldInfo(Field);
1818  return LValue::MakeBitfield(BaseValue, Info,
1819                          Field->getType().withCVRQualifiers(CVRQualifiers));
1820}
1821
1822/// EmitLValueForAnonRecordField - Given that the field is a member of
1823/// an anonymous struct or union buried inside a record, and given
1824/// that the base value is a pointer to the enclosing record, derive
1825/// an lvalue for the ultimate field.
1826LValue CodeGenFunction::EmitLValueForAnonRecordField(llvm::Value *BaseValue,
1827                                             const IndirectFieldDecl *Field,
1828                                                     unsigned CVRQualifiers) {
1829  IndirectFieldDecl::chain_iterator I = Field->chain_begin(),
1830    IEnd = Field->chain_end();
1831  while (true) {
1832    LValue LV = EmitLValueForField(BaseValue, cast<FieldDecl>(*I),
1833                                   CVRQualifiers);
1834    if (++I == IEnd) return LV;
1835
1836    assert(LV.isSimple());
1837    BaseValue = LV.getAddress();
1838    CVRQualifiers |= LV.getVRQualifiers();
1839  }
1840}
1841
1842LValue CodeGenFunction::EmitLValueForField(llvm::Value *baseAddr,
1843                                           const FieldDecl *field,
1844                                           unsigned cvr) {
1845  if (field->isBitField())
1846    return EmitLValueForBitfield(baseAddr, field, cvr);
1847
1848  const RecordDecl *rec = field->getParent();
1849  QualType type = field->getType();
1850
1851  bool mayAlias = rec->hasAttr<MayAliasAttr>();
1852
1853  llvm::Value *addr = baseAddr;
1854  if (rec->isUnion()) {
1855    // For unions, there is no pointer adjustment.
1856    assert(!type->isReferenceType() && "union has reference member");
1857  } else {
1858    // For structs, we GEP to the field that the record layout suggests.
1859    unsigned idx = CGM.getTypes().getCGRecordLayout(rec).getLLVMFieldNo(field);
1860    addr = Builder.CreateStructGEP(addr, idx, field->getName());
1861
1862    // If this is a reference field, load the reference right now.
1863    if (const ReferenceType *refType = type->getAs<ReferenceType>()) {
1864      llvm::LoadInst *load = Builder.CreateLoad(addr, "ref");
1865      if (cvr & Qualifiers::Volatile) load->setVolatile(true);
1866
1867      if (CGM.shouldUseTBAA()) {
1868        llvm::MDNode *tbaa;
1869        if (mayAlias)
1870          tbaa = CGM.getTBAAInfo(getContext().CharTy);
1871        else
1872          tbaa = CGM.getTBAAInfo(type);
1873        CGM.DecorateInstruction(load, tbaa);
1874      }
1875
1876      addr = load;
1877      mayAlias = false;
1878      type = refType->getPointeeType();
1879      cvr = 0; // qualifiers don't recursively apply to referencee
1880    }
1881  }
1882
1883  // Make sure that the address is pointing to the right type.  This is critical
1884  // for both unions and structs.  A union needs a bitcast, a struct element
1885  // will need a bitcast if the LLVM type laid out doesn't match the desired
1886  // type.
1887  addr = EmitBitCastOfLValueToProperType(*this, addr,
1888                                         CGM.getTypes().ConvertTypeForMem(type),
1889                                         field->getName());
1890
1891  if (field->hasAttr<AnnotateAttr>())
1892    addr = EmitFieldAnnotations(field, addr);
1893
1894  unsigned alignment = getContext().getDeclAlign(field).getQuantity();
1895  LValue LV = MakeAddrLValue(addr, type, alignment);
1896  LV.getQuals().addCVRQualifiers(cvr);
1897
1898  // __weak attribute on a field is ignored.
1899  if (LV.getQuals().getObjCGCAttr() == Qualifiers::Weak)
1900    LV.getQuals().removeObjCGCAttr();
1901
1902  // Fields of may_alias structs act like 'char' for TBAA purposes.
1903  // FIXME: this should get propagated down through anonymous structs
1904  // and unions.
1905  if (mayAlias && LV.getTBAAInfo())
1906    LV.setTBAAInfo(CGM.getTBAAInfo(getContext().CharTy));
1907
1908  return LV;
1909}
1910
1911LValue
1912CodeGenFunction::EmitLValueForFieldInitialization(llvm::Value *BaseValue,
1913                                                  const FieldDecl *Field,
1914                                                  unsigned CVRQualifiers) {
1915  QualType FieldType = Field->getType();
1916
1917  if (!FieldType->isReferenceType())
1918    return EmitLValueForField(BaseValue, Field, CVRQualifiers);
1919
1920  const CGRecordLayout &RL =
1921    CGM.getTypes().getCGRecordLayout(Field->getParent());
1922  unsigned idx = RL.getLLVMFieldNo(Field);
1923  llvm::Value *V = Builder.CreateStructGEP(BaseValue, idx);
1924  assert(!FieldType.getObjCGCAttr() && "fields cannot have GC attrs");
1925
1926
1927  // Make sure that the address is pointing to the right type.  This is critical
1928  // for both unions and structs.  A union needs a bitcast, a struct element
1929  // will need a bitcast if the LLVM type laid out doesn't match the desired
1930  // type.
1931  llvm::Type *llvmType = ConvertTypeForMem(FieldType);
1932  unsigned AS = cast<llvm::PointerType>(V->getType())->getAddressSpace();
1933  V = Builder.CreateBitCast(V, llvmType->getPointerTo(AS));
1934
1935  unsigned Alignment = getContext().getDeclAlign(Field).getQuantity();
1936  return MakeAddrLValue(V, FieldType, Alignment);
1937}
1938
1939LValue CodeGenFunction::EmitCompoundLiteralLValue(const CompoundLiteralExpr *E){
1940  llvm::Value *DeclPtr = CreateMemTemp(E->getType(), ".compoundliteral");
1941  const Expr *InitExpr = E->getInitializer();
1942  LValue Result = MakeAddrLValue(DeclPtr, E->getType());
1943
1944  EmitAnyExprToMem(InitExpr, DeclPtr, E->getType().getQualifiers(),
1945                   /*Init*/ true);
1946
1947  return Result;
1948}
1949
1950LValue CodeGenFunction::
1951EmitConditionalOperatorLValue(const AbstractConditionalOperator *expr) {
1952  if (!expr->isGLValue()) {
1953    // ?: here should be an aggregate.
1954    assert((hasAggregateLLVMType(expr->getType()) &&
1955            !expr->getType()->isAnyComplexType()) &&
1956           "Unexpected conditional operator!");
1957    return EmitAggExprToLValue(expr);
1958  }
1959
1960  const Expr *condExpr = expr->getCond();
1961  bool CondExprBool;
1962  if (ConstantFoldsToSimpleInteger(condExpr, CondExprBool)) {
1963    const Expr *live = expr->getTrueExpr(), *dead = expr->getFalseExpr();
1964    if (!CondExprBool) std::swap(live, dead);
1965
1966    if (!ContainsLabel(dead))
1967      return EmitLValue(live);
1968  }
1969
1970  OpaqueValueMapping binding(*this, expr);
1971
1972  llvm::BasicBlock *lhsBlock = createBasicBlock("cond.true");
1973  llvm::BasicBlock *rhsBlock = createBasicBlock("cond.false");
1974  llvm::BasicBlock *contBlock = createBasicBlock("cond.end");
1975
1976  ConditionalEvaluation eval(*this);
1977  EmitBranchOnBoolExpr(condExpr, lhsBlock, rhsBlock);
1978
1979  // Any temporaries created here are conditional.
1980  EmitBlock(lhsBlock);
1981  eval.begin(*this);
1982  LValue lhs = EmitLValue(expr->getTrueExpr());
1983  eval.end(*this);
1984
1985  if (!lhs.isSimple())
1986    return EmitUnsupportedLValue(expr, "conditional operator");
1987
1988  lhsBlock = Builder.GetInsertBlock();
1989  Builder.CreateBr(contBlock);
1990
1991  // Any temporaries created here are conditional.
1992  EmitBlock(rhsBlock);
1993  eval.begin(*this);
1994  LValue rhs = EmitLValue(expr->getFalseExpr());
1995  eval.end(*this);
1996  if (!rhs.isSimple())
1997    return EmitUnsupportedLValue(expr, "conditional operator");
1998  rhsBlock = Builder.GetInsertBlock();
1999
2000  EmitBlock(contBlock);
2001
2002  llvm::PHINode *phi = Builder.CreatePHI(lhs.getAddress()->getType(), 2,
2003                                         "cond-lvalue");
2004  phi->addIncoming(lhs.getAddress(), lhsBlock);
2005  phi->addIncoming(rhs.getAddress(), rhsBlock);
2006  return MakeAddrLValue(phi, expr->getType());
2007}
2008
2009/// EmitCastLValue - Casts are never lvalues unless that cast is a dynamic_cast.
2010/// If the cast is a dynamic_cast, we can have the usual lvalue result,
2011/// otherwise if a cast is needed by the code generator in an lvalue context,
2012/// then it must mean that we need the address of an aggregate in order to
2013/// access one of its fields.  This can happen for all the reasons that casts
2014/// are permitted with aggregate result, including noop aggregate casts, and
2015/// cast from scalar to union.
2016LValue CodeGenFunction::EmitCastLValue(const CastExpr *E) {
2017  switch (E->getCastKind()) {
2018  case CK_ToVoid:
2019    return EmitUnsupportedLValue(E, "unexpected cast lvalue");
2020
2021  case CK_Dependent:
2022    llvm_unreachable("dependent cast kind in IR gen!");
2023
2024  case CK_GetObjCProperty: {
2025    LValue LV = EmitLValue(E->getSubExpr());
2026    assert(LV.isPropertyRef());
2027    RValue RV = EmitLoadOfPropertyRefLValue(LV);
2028
2029    // Property is an aggregate r-value.
2030    if (RV.isAggregate()) {
2031      return MakeAddrLValue(RV.getAggregateAddr(), E->getType());
2032    }
2033
2034    // Implicit property returns an l-value.
2035    assert(RV.isScalar());
2036    return MakeAddrLValue(RV.getScalarVal(), E->getSubExpr()->getType());
2037  }
2038
2039  case CK_NoOp:
2040  case CK_LValueToRValue:
2041    if (!E->getSubExpr()->Classify(getContext()).isPRValue()
2042        || E->getType()->isRecordType())
2043      return EmitLValue(E->getSubExpr());
2044    // Fall through to synthesize a temporary.
2045
2046  case CK_BitCast:
2047  case CK_ArrayToPointerDecay:
2048  case CK_FunctionToPointerDecay:
2049  case CK_NullToMemberPointer:
2050  case CK_NullToPointer:
2051  case CK_IntegralToPointer:
2052  case CK_PointerToIntegral:
2053  case CK_PointerToBoolean:
2054  case CK_VectorSplat:
2055  case CK_IntegralCast:
2056  case CK_IntegralToBoolean:
2057  case CK_IntegralToFloating:
2058  case CK_FloatingToIntegral:
2059  case CK_FloatingToBoolean:
2060  case CK_FloatingCast:
2061  case CK_FloatingRealToComplex:
2062  case CK_FloatingComplexToReal:
2063  case CK_FloatingComplexToBoolean:
2064  case CK_FloatingComplexCast:
2065  case CK_FloatingComplexToIntegralComplex:
2066  case CK_IntegralRealToComplex:
2067  case CK_IntegralComplexToReal:
2068  case CK_IntegralComplexToBoolean:
2069  case CK_IntegralComplexCast:
2070  case CK_IntegralComplexToFloatingComplex:
2071  case CK_DerivedToBaseMemberPointer:
2072  case CK_BaseToDerivedMemberPointer:
2073  case CK_MemberPointerToBoolean:
2074  case CK_AnyPointerToBlockPointerCast:
2075  case CK_ARCProduceObject:
2076  case CK_ARCConsumeObject:
2077  case CK_ARCReclaimReturnedObject:
2078  case CK_ARCExtendBlockObject: {
2079    // These casts only produce lvalues when we're binding a reference to a
2080    // temporary realized from a (converted) pure rvalue. Emit the expression
2081    // as a value, copy it into a temporary, and return an lvalue referring to
2082    // that temporary.
2083    llvm::Value *V = CreateMemTemp(E->getType(), "ref.temp");
2084    EmitAnyExprToMem(E, V, E->getType().getQualifiers(), false);
2085    return MakeAddrLValue(V, E->getType());
2086  }
2087
2088  case CK_Dynamic: {
2089    LValue LV = EmitLValue(E->getSubExpr());
2090    llvm::Value *V = LV.getAddress();
2091    const CXXDynamicCastExpr *DCE = cast<CXXDynamicCastExpr>(E);
2092    return MakeAddrLValue(EmitDynamicCast(V, DCE), E->getType());
2093  }
2094
2095  case CK_ConstructorConversion:
2096  case CK_UserDefinedConversion:
2097  case CK_CPointerToObjCPointerCast:
2098  case CK_BlockPointerToObjCPointerCast:
2099    return EmitLValue(E->getSubExpr());
2100
2101  case CK_UncheckedDerivedToBase:
2102  case CK_DerivedToBase: {
2103    const RecordType *DerivedClassTy =
2104      E->getSubExpr()->getType()->getAs<RecordType>();
2105    CXXRecordDecl *DerivedClassDecl =
2106      cast<CXXRecordDecl>(DerivedClassTy->getDecl());
2107
2108    LValue LV = EmitLValue(E->getSubExpr());
2109    llvm::Value *This = LV.getAddress();
2110
2111    // Perform the derived-to-base conversion
2112    llvm::Value *Base =
2113      GetAddressOfBaseClass(This, DerivedClassDecl,
2114                            E->path_begin(), E->path_end(),
2115                            /*NullCheckValue=*/false);
2116
2117    return MakeAddrLValue(Base, E->getType());
2118  }
2119  case CK_ToUnion:
2120    return EmitAggExprToLValue(E);
2121  case CK_BaseToDerived: {
2122    const RecordType *DerivedClassTy = E->getType()->getAs<RecordType>();
2123    CXXRecordDecl *DerivedClassDecl =
2124      cast<CXXRecordDecl>(DerivedClassTy->getDecl());
2125
2126    LValue LV = EmitLValue(E->getSubExpr());
2127
2128    // Perform the base-to-derived conversion
2129    llvm::Value *Derived =
2130      GetAddressOfDerivedClass(LV.getAddress(), DerivedClassDecl,
2131                               E->path_begin(), E->path_end(),
2132                               /*NullCheckValue=*/false);
2133
2134    return MakeAddrLValue(Derived, E->getType());
2135  }
2136  case CK_LValueBitCast: {
2137    // This must be a reinterpret_cast (or c-style equivalent).
2138    const ExplicitCastExpr *CE = cast<ExplicitCastExpr>(E);
2139
2140    LValue LV = EmitLValue(E->getSubExpr());
2141    llvm::Value *V = Builder.CreateBitCast(LV.getAddress(),
2142                                           ConvertType(CE->getTypeAsWritten()));
2143    return MakeAddrLValue(V, E->getType());
2144  }
2145  case CK_ObjCObjectLValueCast: {
2146    LValue LV = EmitLValue(E->getSubExpr());
2147    QualType ToType = getContext().getLValueReferenceType(E->getType());
2148    llvm::Value *V = Builder.CreateBitCast(LV.getAddress(),
2149                                           ConvertType(ToType));
2150    return MakeAddrLValue(V, E->getType());
2151  }
2152  }
2153
2154  llvm_unreachable("Unhandled lvalue cast kind?");
2155}
2156
2157LValue CodeGenFunction::EmitNullInitializationLValue(
2158                                              const CXXScalarValueInitExpr *E) {
2159  QualType Ty = E->getType();
2160  LValue LV = MakeAddrLValue(CreateMemTemp(Ty), Ty);
2161  EmitNullInitialization(LV.getAddress(), Ty);
2162  return LV;
2163}
2164
2165LValue CodeGenFunction::EmitOpaqueValueLValue(const OpaqueValueExpr *e) {
2166  assert(e->isGLValue() || e->getType()->isRecordType());
2167  return getOpaqueLValueMapping(e);
2168}
2169
2170LValue CodeGenFunction::EmitMaterializeTemporaryExpr(
2171                                           const MaterializeTemporaryExpr *E) {
2172  RValue RV = EmitReferenceBindingToExpr(E, /*InitializedDecl=*/0);
2173  return MakeAddrLValue(RV.getScalarVal(), E->getType());
2174}
2175
2176
2177//===--------------------------------------------------------------------===//
2178//                             Expression Emission
2179//===--------------------------------------------------------------------===//
2180
2181RValue CodeGenFunction::EmitCallExpr(const CallExpr *E,
2182                                     ReturnValueSlot ReturnValue) {
2183  if (CGDebugInfo *DI = getDebugInfo()) {
2184    DI->setLocation(E->getLocStart());
2185    DI->EmitLocation(Builder);
2186  }
2187
2188  // Builtins never have block type.
2189  if (E->getCallee()->getType()->isBlockPointerType())
2190    return EmitBlockCallExpr(E, ReturnValue);
2191
2192  if (const CXXMemberCallExpr *CE = dyn_cast<CXXMemberCallExpr>(E))
2193    return EmitCXXMemberCallExpr(CE, ReturnValue);
2194
2195  if (const CUDAKernelCallExpr *CE = dyn_cast<CUDAKernelCallExpr>(E))
2196    return EmitCUDAKernelCallExpr(CE, ReturnValue);
2197
2198  const Decl *TargetDecl = E->getCalleeDecl();
2199  if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(TargetDecl)) {
2200    if (unsigned builtinID = FD->getBuiltinID())
2201      return EmitBuiltinExpr(FD, builtinID, E);
2202  }
2203
2204  if (const CXXOperatorCallExpr *CE = dyn_cast<CXXOperatorCallExpr>(E))
2205    if (const CXXMethodDecl *MD = dyn_cast_or_null<CXXMethodDecl>(TargetDecl))
2206      return EmitCXXOperatorMemberCallExpr(CE, MD, ReturnValue);
2207
2208  if (const CXXPseudoDestructorExpr *PseudoDtor
2209          = dyn_cast<CXXPseudoDestructorExpr>(E->getCallee()->IgnoreParens())) {
2210    QualType DestroyedType = PseudoDtor->getDestroyedType();
2211    if (getContext().getLangOptions().ObjCAutoRefCount &&
2212        DestroyedType->isObjCLifetimeType() &&
2213        (DestroyedType.getObjCLifetime() == Qualifiers::OCL_Strong ||
2214         DestroyedType.getObjCLifetime() == Qualifiers::OCL_Weak)) {
2215      // Automatic Reference Counting:
2216      //   If the pseudo-expression names a retainable object with weak or
2217      //   strong lifetime, the object shall be released.
2218      Expr *BaseExpr = PseudoDtor->getBase();
2219      llvm::Value *BaseValue = NULL;
2220      Qualifiers BaseQuals;
2221
2222      // If this is s.x, emit s as an lvalue. If it is s->x, emit s as a scalar.
2223      if (PseudoDtor->isArrow()) {
2224        BaseValue = EmitScalarExpr(BaseExpr);
2225        const PointerType *PTy = BaseExpr->getType()->getAs<PointerType>();
2226        BaseQuals = PTy->getPointeeType().getQualifiers();
2227      } else {
2228        LValue BaseLV = EmitLValue(BaseExpr);
2229        BaseValue = BaseLV.getAddress();
2230        QualType BaseTy = BaseExpr->getType();
2231        BaseQuals = BaseTy.getQualifiers();
2232      }
2233
2234      switch (PseudoDtor->getDestroyedType().getObjCLifetime()) {
2235      case Qualifiers::OCL_None:
2236      case Qualifiers::OCL_ExplicitNone:
2237      case Qualifiers::OCL_Autoreleasing:
2238        break;
2239
2240      case Qualifiers::OCL_Strong:
2241        EmitARCRelease(Builder.CreateLoad(BaseValue,
2242                          PseudoDtor->getDestroyedType().isVolatileQualified()),
2243                       /*precise*/ true);
2244        break;
2245
2246      case Qualifiers::OCL_Weak:
2247        EmitARCDestroyWeak(BaseValue);
2248        break;
2249      }
2250    } else {
2251      // C++ [expr.pseudo]p1:
2252      //   The result shall only be used as the operand for the function call
2253      //   operator (), and the result of such a call has type void. The only
2254      //   effect is the evaluation of the postfix-expression before the dot or
2255      //   arrow.
2256      EmitScalarExpr(E->getCallee());
2257    }
2258
2259    return RValue::get(0);
2260  }
2261
2262  llvm::Value *Callee = EmitScalarExpr(E->getCallee());
2263  return EmitCall(E->getCallee()->getType(), Callee, ReturnValue,
2264                  E->arg_begin(), E->arg_end(), TargetDecl);
2265}
2266
2267LValue CodeGenFunction::EmitBinaryOperatorLValue(const BinaryOperator *E) {
2268  // Comma expressions just emit their LHS then their RHS as an l-value.
2269  if (E->getOpcode() == BO_Comma) {
2270    EmitIgnoredExpr(E->getLHS());
2271    EnsureInsertPoint();
2272    return EmitLValue(E->getRHS());
2273  }
2274
2275  if (E->getOpcode() == BO_PtrMemD ||
2276      E->getOpcode() == BO_PtrMemI)
2277    return EmitPointerToDataMemberBinaryExpr(E);
2278
2279  assert(E->getOpcode() == BO_Assign && "unexpected binary l-value");
2280
2281  // Note that in all of these cases, __block variables need the RHS
2282  // evaluated first just in case the variable gets moved by the RHS.
2283
2284  if (!hasAggregateLLVMType(E->getType())) {
2285    switch (E->getLHS()->getType().getObjCLifetime()) {
2286    case Qualifiers::OCL_Strong:
2287      return EmitARCStoreStrong(E, /*ignored*/ false).first;
2288
2289    case Qualifiers::OCL_Autoreleasing:
2290      return EmitARCStoreAutoreleasing(E).first;
2291
2292    // No reason to do any of these differently.
2293    case Qualifiers::OCL_None:
2294    case Qualifiers::OCL_ExplicitNone:
2295    case Qualifiers::OCL_Weak:
2296      break;
2297    }
2298
2299    RValue RV = EmitAnyExpr(E->getRHS());
2300    LValue LV = EmitLValue(E->getLHS());
2301    EmitStoreThroughLValue(RV, LV);
2302    return LV;
2303  }
2304
2305  if (E->getType()->isAnyComplexType())
2306    return EmitComplexAssignmentLValue(E);
2307
2308  return EmitAggExprToLValue(E);
2309}
2310
2311LValue CodeGenFunction::EmitCallExprLValue(const CallExpr *E) {
2312  RValue RV = EmitCallExpr(E);
2313
2314  if (!RV.isScalar())
2315    return MakeAddrLValue(RV.getAggregateAddr(), E->getType());
2316
2317  assert(E->getCallReturnType()->isReferenceType() &&
2318         "Can't have a scalar return unless the return type is a "
2319         "reference type!");
2320
2321  return MakeAddrLValue(RV.getScalarVal(), E->getType());
2322}
2323
2324LValue CodeGenFunction::EmitVAArgExprLValue(const VAArgExpr *E) {
2325  // FIXME: This shouldn't require another copy.
2326  return EmitAggExprToLValue(E);
2327}
2328
2329LValue CodeGenFunction::EmitCXXConstructLValue(const CXXConstructExpr *E) {
2330  assert(E->getType()->getAsCXXRecordDecl()->hasTrivialDestructor()
2331         && "binding l-value to type which needs a temporary");
2332  AggValueSlot Slot = CreateAggTemp(E->getType());
2333  EmitCXXConstructExpr(E, Slot);
2334  return MakeAddrLValue(Slot.getAddr(), E->getType());
2335}
2336
2337LValue
2338CodeGenFunction::EmitCXXTypeidLValue(const CXXTypeidExpr *E) {
2339  return MakeAddrLValue(EmitCXXTypeidExpr(E), E->getType());
2340}
2341
2342LValue
2343CodeGenFunction::EmitCXXBindTemporaryLValue(const CXXBindTemporaryExpr *E) {
2344  AggValueSlot Slot = CreateAggTemp(E->getType(), "temp.lvalue");
2345  Slot.setExternallyDestructed();
2346  EmitAggExpr(E->getSubExpr(), Slot);
2347  EmitCXXTemporary(E->getTemporary(), Slot.getAddr());
2348  return MakeAddrLValue(Slot.getAddr(), E->getType());
2349}
2350
2351LValue CodeGenFunction::EmitObjCMessageExprLValue(const ObjCMessageExpr *E) {
2352  RValue RV = EmitObjCMessageExpr(E);
2353
2354  if (!RV.isScalar())
2355    return MakeAddrLValue(RV.getAggregateAddr(), E->getType());
2356
2357  assert(E->getMethodDecl()->getResultType()->isReferenceType() &&
2358         "Can't have a scalar return unless the return type is a "
2359         "reference type!");
2360
2361  return MakeAddrLValue(RV.getScalarVal(), E->getType());
2362}
2363
2364LValue CodeGenFunction::EmitObjCSelectorLValue(const ObjCSelectorExpr *E) {
2365  llvm::Value *V =
2366    CGM.getObjCRuntime().GetSelector(Builder, E->getSelector(), true);
2367  return MakeAddrLValue(V, E->getType());
2368}
2369
2370llvm::Value *CodeGenFunction::EmitIvarOffset(const ObjCInterfaceDecl *Interface,
2371                                             const ObjCIvarDecl *Ivar) {
2372  return CGM.getObjCRuntime().EmitIvarOffset(*this, Interface, Ivar);
2373}
2374
2375LValue CodeGenFunction::EmitLValueForIvar(QualType ObjectTy,
2376                                          llvm::Value *BaseValue,
2377                                          const ObjCIvarDecl *Ivar,
2378                                          unsigned CVRQualifiers) {
2379  return CGM.getObjCRuntime().EmitObjCValueForIvar(*this, ObjectTy, BaseValue,
2380                                                   Ivar, CVRQualifiers);
2381}
2382
2383LValue CodeGenFunction::EmitObjCIvarRefLValue(const ObjCIvarRefExpr *E) {
2384  // FIXME: A lot of the code below could be shared with EmitMemberExpr.
2385  llvm::Value *BaseValue = 0;
2386  const Expr *BaseExpr = E->getBase();
2387  Qualifiers BaseQuals;
2388  QualType ObjectTy;
2389  if (E->isArrow()) {
2390    BaseValue = EmitScalarExpr(BaseExpr);
2391    ObjectTy = BaseExpr->getType()->getPointeeType();
2392    BaseQuals = ObjectTy.getQualifiers();
2393  } else {
2394    LValue BaseLV = EmitLValue(BaseExpr);
2395    // FIXME: this isn't right for bitfields.
2396    BaseValue = BaseLV.getAddress();
2397    ObjectTy = BaseExpr->getType();
2398    BaseQuals = ObjectTy.getQualifiers();
2399  }
2400
2401  LValue LV =
2402    EmitLValueForIvar(ObjectTy, BaseValue, E->getDecl(),
2403                      BaseQuals.getCVRQualifiers());
2404  setObjCGCLValueClass(getContext(), E, LV);
2405  return LV;
2406}
2407
2408LValue CodeGenFunction::EmitStmtExprLValue(const StmtExpr *E) {
2409  // Can only get l-value for message expression returning aggregate type
2410  RValue RV = EmitAnyExprToTemp(E);
2411  return MakeAddrLValue(RV.getAggregateAddr(), E->getType());
2412}
2413
2414RValue CodeGenFunction::EmitCall(QualType CalleeType, llvm::Value *Callee,
2415                                 ReturnValueSlot ReturnValue,
2416                                 CallExpr::const_arg_iterator ArgBeg,
2417                                 CallExpr::const_arg_iterator ArgEnd,
2418                                 const Decl *TargetDecl) {
2419  // Get the actual function type. The callee type will always be a pointer to
2420  // function type or a block pointer type.
2421  assert(CalleeType->isFunctionPointerType() &&
2422         "Call must have function pointer type!");
2423
2424  CalleeType = getContext().getCanonicalType(CalleeType);
2425
2426  const FunctionType *FnType
2427    = cast<FunctionType>(cast<PointerType>(CalleeType)->getPointeeType());
2428
2429  CallArgList Args;
2430  EmitCallArgs(Args, dyn_cast<FunctionProtoType>(FnType), ArgBeg, ArgEnd);
2431
2432  const CGFunctionInfo &FnInfo = CGM.getTypes().getFunctionInfo(Args, FnType);
2433
2434  // C99 6.5.2.2p6:
2435  //   If the expression that denotes the called function has a type
2436  //   that does not include a prototype, [the default argument
2437  //   promotions are performed]. If the number of arguments does not
2438  //   equal the number of parameters, the behavior is undefined. If
2439  //   the function is defined with a type that includes a prototype,
2440  //   and either the prototype ends with an ellipsis (, ...) or the
2441  //   types of the arguments after promotion are not compatible with
2442  //   the types of the parameters, the behavior is undefined. If the
2443  //   function is defined with a type that does not include a
2444  //   prototype, and the types of the arguments after promotion are
2445  //   not compatible with those of the parameters after promotion,
2446  //   the behavior is undefined [except in some trivial cases].
2447  // That is, in the general case, we should assume that a call
2448  // through an unprototyped function type works like a *non-variadic*
2449  // call.  The way we make this work is to cast to the exact type
2450  // of the promoted arguments.
2451  if (isa<FunctionNoProtoType>(FnType) &&
2452      !getTargetHooks().isNoProtoCallVariadic(FnType->getCallConv())) {
2453    assert(cast<llvm::FunctionType>(Callee->getType()->getContainedType(0))
2454             ->isVarArg());
2455    llvm::Type *CalleeTy = getTypes().GetFunctionType(FnInfo, false);
2456    CalleeTy = CalleeTy->getPointerTo();
2457    Callee = Builder.CreateBitCast(Callee, CalleeTy, "callee.knr.cast");
2458  }
2459
2460  return EmitCall(FnInfo, Callee, ReturnValue, Args, TargetDecl);
2461}
2462
2463LValue CodeGenFunction::
2464EmitPointerToDataMemberBinaryExpr(const BinaryOperator *E) {
2465  llvm::Value *BaseV;
2466  if (E->getOpcode() == BO_PtrMemI)
2467    BaseV = EmitScalarExpr(E->getLHS());
2468  else
2469    BaseV = EmitLValue(E->getLHS()).getAddress();
2470
2471  llvm::Value *OffsetV = EmitScalarExpr(E->getRHS());
2472
2473  const MemberPointerType *MPT
2474    = E->getRHS()->getType()->getAs<MemberPointerType>();
2475
2476  llvm::Value *AddV =
2477    CGM.getCXXABI().EmitMemberDataPointerAddress(*this, BaseV, OffsetV, MPT);
2478
2479  return MakeAddrLValue(AddV, MPT->getPointeeType());
2480}
2481
2482static void
2483EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *E, llvm::Value *Dest,
2484             llvm::Value *Ptr, llvm::Value *Val1, llvm::Value *Val2,
2485             uint64_t Size, unsigned Align, llvm::AtomicOrdering Order) {
2486  if (E->isCmpXChg()) {
2487    // Note that cmpxchg only supports specifying one ordering and
2488    // doesn't support weak cmpxchg, at least at the moment.
2489    llvm::LoadInst *LoadVal1 = CGF.Builder.CreateLoad(Val1);
2490    LoadVal1->setAlignment(Align);
2491    llvm::LoadInst *LoadVal2 = CGF.Builder.CreateLoad(Val2);
2492    LoadVal2->setAlignment(Align);
2493    llvm::AtomicCmpXchgInst *CXI =
2494        CGF.Builder.CreateAtomicCmpXchg(Ptr, LoadVal1, LoadVal2, Order);
2495    CXI->setVolatile(E->isVolatile());
2496    llvm::StoreInst *StoreVal1 = CGF.Builder.CreateStore(CXI, Val1);
2497    StoreVal1->setAlignment(Align);
2498    llvm::Value *Cmp = CGF.Builder.CreateICmpEQ(CXI, LoadVal1);
2499    CGF.EmitStoreOfScalar(Cmp, CGF.MakeAddrLValue(Dest, E->getType()));
2500    return;
2501  }
2502
2503  if (E->getOp() == AtomicExpr::Load) {
2504    llvm::LoadInst *Load = CGF.Builder.CreateLoad(Ptr);
2505    Load->setAtomic(Order);
2506    Load->setAlignment(Size);
2507    Load->setVolatile(E->isVolatile());
2508    llvm::StoreInst *StoreDest = CGF.Builder.CreateStore(Load, Dest);
2509    StoreDest->setAlignment(Align);
2510    return;
2511  }
2512
2513  if (E->getOp() == AtomicExpr::Store) {
2514    assert(!Dest && "Store does not return a value");
2515    llvm::LoadInst *LoadVal1 = CGF.Builder.CreateLoad(Val1);
2516    LoadVal1->setAlignment(Align);
2517    llvm::StoreInst *Store = CGF.Builder.CreateStore(LoadVal1, Ptr);
2518    Store->setAtomic(Order);
2519    Store->setAlignment(Size);
2520    Store->setVolatile(E->isVolatile());
2521    return;
2522  }
2523
2524  llvm::AtomicRMWInst::BinOp Op = llvm::AtomicRMWInst::Add;
2525  switch (E->getOp()) {
2526    case AtomicExpr::CmpXchgWeak:
2527    case AtomicExpr::CmpXchgStrong:
2528    case AtomicExpr::Store:
2529    case AtomicExpr::Load:  assert(0 && "Already handled!");
2530    case AtomicExpr::Add:   Op = llvm::AtomicRMWInst::Add;  break;
2531    case AtomicExpr::Sub:   Op = llvm::AtomicRMWInst::Sub;  break;
2532    case AtomicExpr::And:   Op = llvm::AtomicRMWInst::And;  break;
2533    case AtomicExpr::Or:    Op = llvm::AtomicRMWInst::Or;   break;
2534    case AtomicExpr::Xor:   Op = llvm::AtomicRMWInst::Xor;  break;
2535    case AtomicExpr::Xchg:  Op = llvm::AtomicRMWInst::Xchg; break;
2536  }
2537  llvm::LoadInst *LoadVal1 = CGF.Builder.CreateLoad(Val1);
2538  LoadVal1->setAlignment(Align);
2539  llvm::AtomicRMWInst *RMWI =
2540      CGF.Builder.CreateAtomicRMW(Op, Ptr, LoadVal1, Order);
2541  RMWI->setVolatile(E->isVolatile());
2542  llvm::StoreInst *StoreDest = CGF.Builder.CreateStore(RMWI, Dest);
2543  StoreDest->setAlignment(Align);
2544}
2545
2546// This function emits any expression (scalar, complex, or aggregate)
2547// into a temporary alloca.
2548static llvm::Value *
2549EmitValToTemp(CodeGenFunction &CGF, Expr *E) {
2550  llvm::Value *DeclPtr = CGF.CreateMemTemp(E->getType(), ".atomictmp");
2551  CGF.EmitAnyExprToMem(E, DeclPtr, E->getType().getQualifiers(),
2552                       /*Init*/ true);
2553  return DeclPtr;
2554}
2555
2556static RValue ConvertTempToRValue(CodeGenFunction &CGF, QualType Ty,
2557                                  llvm::Value *Dest) {
2558  if (Ty->isAnyComplexType())
2559    return RValue::getComplex(CGF.LoadComplexFromAddr(Dest, false));
2560  if (CGF.hasAggregateLLVMType(Ty))
2561    return RValue::getAggregate(Dest);
2562  return RValue::get(CGF.EmitLoadOfScalar(CGF.MakeAddrLValue(Dest, Ty)));
2563}
2564
2565RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E, llvm::Value *Dest) {
2566  QualType AtomicTy = E->getPtr()->getType()->getPointeeType();
2567  QualType MemTy = AtomicTy->getAs<AtomicType>()->getValueType();
2568  CharUnits sizeChars = getContext().getTypeSizeInChars(AtomicTy);
2569  uint64_t Size = sizeChars.getQuantity();
2570  CharUnits alignChars = getContext().getTypeAlignInChars(AtomicTy);
2571  unsigned Align = alignChars.getQuantity();
2572  // FIXME: Bound on Size should not be hardcoded.
2573  bool UseLibcall = (sizeChars != alignChars || !llvm::isPowerOf2_64(Size) ||
2574                     Size > 8);
2575
2576  llvm::Value *Ptr, *Order, *OrderFail = 0, *Val1 = 0, *Val2 = 0;
2577  Ptr = EmitScalarExpr(E->getPtr());
2578  Order = EmitScalarExpr(E->getOrder());
2579  if (E->isCmpXChg()) {
2580    Val1 = EmitScalarExpr(E->getVal1());
2581    Val2 = EmitValToTemp(*this, E->getVal2());
2582    OrderFail = EmitScalarExpr(E->getOrderFail());
2583    (void)OrderFail; // OrderFail is unused at the moment
2584  } else if ((E->getOp() == AtomicExpr::Add || E->getOp() == AtomicExpr::Sub) &&
2585             MemTy->isPointerType()) {
2586    // For pointers, we're required to do a bit of math: adding 1 to an int*
2587    // is not the same as adding 1 to a uintptr_t.
2588    QualType Val1Ty = E->getVal1()->getType();
2589    llvm::Value *Val1Scalar = EmitScalarExpr(E->getVal1());
2590    uint64_t PointeeIncAmt =
2591        getContext().getTypeSizeInChars(MemTy->getPointeeType()).getQuantity();
2592    llvm::Value *PointeeIncAmtVal =
2593        llvm::ConstantInt::get(Val1Scalar->getType(), PointeeIncAmt);
2594    Val1Scalar = Builder.CreateMul(Val1Scalar, PointeeIncAmtVal);
2595    Val1 = CreateMemTemp(Val1Ty, ".atomictmp");
2596    EmitStoreOfScalar(Val1Scalar, MakeAddrLValue(Val1, Val1Ty));
2597  } else if (E->getOp() != AtomicExpr::Load) {
2598    Val1 = EmitValToTemp(*this, E->getVal1());
2599  }
2600
2601  if (E->getOp() != AtomicExpr::Store && !Dest)
2602    Dest = CreateMemTemp(E->getType(), ".atomicdst");
2603
2604  if (UseLibcall) {
2605    // FIXME: Finalize what the libcalls are actually supposed to look like.
2606    // See also http://gcc.gnu.org/wiki/Atomic/GCCMM/LIbrary .
2607    return EmitUnsupportedRValue(E, "atomic library call");
2608  }
2609#if 0
2610  if (UseLibcall) {
2611    const char* LibCallName;
2612    switch (E->getOp()) {
2613    case AtomicExpr::CmpXchgWeak:
2614      LibCallName = "__atomic_compare_exchange_generic"; break;
2615    case AtomicExpr::CmpXchgStrong:
2616      LibCallName = "__atomic_compare_exchange_generic"; break;
2617    case AtomicExpr::Add:   LibCallName = "__atomic_fetch_add_generic"; break;
2618    case AtomicExpr::Sub:   LibCallName = "__atomic_fetch_sub_generic"; break;
2619    case AtomicExpr::And:   LibCallName = "__atomic_fetch_and_generic"; break;
2620    case AtomicExpr::Or:    LibCallName = "__atomic_fetch_or_generic"; break;
2621    case AtomicExpr::Xor:   LibCallName = "__atomic_fetch_xor_generic"; break;
2622    case AtomicExpr::Xchg:  LibCallName = "__atomic_exchange_generic"; break;
2623    case AtomicExpr::Store: LibCallName = "__atomic_store_generic"; break;
2624    case AtomicExpr::Load:  LibCallName = "__atomic_load_generic"; break;
2625    }
2626    llvm::SmallVector<QualType, 4> Params;
2627    CallArgList Args;
2628    QualType RetTy = getContext().VoidTy;
2629    if (E->getOp() != AtomicExpr::Store && !E->isCmpXChg())
2630      Args.add(RValue::get(EmitCastToVoidPtr(Dest)),
2631               getContext().VoidPtrTy);
2632    Args.add(RValue::get(EmitCastToVoidPtr(Ptr)),
2633             getContext().VoidPtrTy);
2634    if (E->getOp() != AtomicExpr::Load)
2635      Args.add(RValue::get(EmitCastToVoidPtr(Val1)),
2636               getContext().VoidPtrTy);
2637    if (E->isCmpXChg()) {
2638      Args.add(RValue::get(EmitCastToVoidPtr(Val2)),
2639               getContext().VoidPtrTy);
2640      RetTy = getContext().IntTy;
2641    }
2642    Args.add(RValue::get(llvm::ConstantInt::get(SizeTy, Size)),
2643             getContext().getSizeType());
2644    const CGFunctionInfo &FuncInfo =
2645        CGM.getTypes().getFunctionInfo(RetTy, Args, FunctionType::ExtInfo());
2646    llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(FuncInfo, false);
2647    llvm::Constant *Func = CGM.CreateRuntimeFunction(FTy, LibCallName);
2648    RValue Res = EmitCall(FuncInfo, Func, ReturnValueSlot(), Args);
2649    if (E->isCmpXChg())
2650      return Res;
2651    if (E->getOp() == AtomicExpr::Store)
2652      return RValue::get(0);
2653    return ConvertTempToRValue(*this, E->getType(), Dest);
2654  }
2655#endif
2656  llvm::Type *IPtrTy =
2657      llvm::IntegerType::get(getLLVMContext(), Size * 8)->getPointerTo();
2658  llvm::Value *OrigDest = Dest;
2659  Ptr = Builder.CreateBitCast(Ptr, IPtrTy);
2660  if (Val1) Val1 = Builder.CreateBitCast(Val1, IPtrTy);
2661  if (Val2) Val2 = Builder.CreateBitCast(Val2, IPtrTy);
2662  if (Dest && !E->isCmpXChg()) Dest = Builder.CreateBitCast(Dest, IPtrTy);
2663
2664  if (isa<llvm::ConstantInt>(Order)) {
2665    int ord = cast<llvm::ConstantInt>(Order)->getZExtValue();
2666    switch (ord) {
2667    case 0:  // memory_order_relaxed
2668      EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, Size, Align,
2669                   llvm::Monotonic);
2670      break;
2671    case 1:  // memory_order_consume
2672    case 2:  // memory_order_acquire
2673      EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, Size, Align,
2674                   llvm::Acquire);
2675      break;
2676    case 3:  // memory_order_release
2677      EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, Size, Align,
2678                   llvm::Release);
2679      break;
2680    case 4:  // memory_order_acq_rel
2681      EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, Size, Align,
2682                   llvm::AcquireRelease);
2683      break;
2684    case 5:  // memory_order_seq_cst
2685      EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, Size, Align,
2686                   llvm::SequentiallyConsistent);
2687      break;
2688    default: // invalid order
2689      // We should not ever get here normally, but it's hard to
2690      // enforce that in general.
2691      break;
2692    }
2693    if (E->getOp() == AtomicExpr::Store)
2694      return RValue::get(0);
2695    return ConvertTempToRValue(*this, E->getType(), OrigDest);
2696  }
2697
2698  // Long case, when Order isn't obviously constant.
2699
2700  // Create all the relevant BB's
2701  llvm::BasicBlock *MonotonicBB = 0, *AcquireBB = 0, *ReleaseBB = 0,
2702                   *AcqRelBB = 0, *SeqCstBB = 0;
2703  MonotonicBB = createBasicBlock("monotonic", CurFn);
2704  if (E->getOp() != AtomicExpr::Store)
2705    AcquireBB = createBasicBlock("acquire", CurFn);
2706  if (E->getOp() != AtomicExpr::Load)
2707    ReleaseBB = createBasicBlock("release", CurFn);
2708  if (E->getOp() != AtomicExpr::Load && E->getOp() != AtomicExpr::Store)
2709    AcqRelBB = createBasicBlock("acqrel", CurFn);
2710  SeqCstBB = createBasicBlock("seqcst", CurFn);
2711  llvm::BasicBlock *ContBB = createBasicBlock("atomic.continue", CurFn);
2712
2713  // Create the switch for the split
2714  // MonotonicBB is arbitrarily chosen as the default case; in practice, this
2715  // doesn't matter unless someone is crazy enough to use something that
2716  // doesn't fold to a constant for the ordering.
2717  Order = Builder.CreateIntCast(Order, Builder.getInt32Ty(), false);
2718  llvm::SwitchInst *SI = Builder.CreateSwitch(Order, MonotonicBB);
2719
2720  // Emit all the different atomics
2721  Builder.SetInsertPoint(MonotonicBB);
2722  EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, Size, Align,
2723               llvm::Monotonic);
2724  Builder.CreateBr(ContBB);
2725  if (E->getOp() != AtomicExpr::Store) {
2726    Builder.SetInsertPoint(AcquireBB);
2727    EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, Size, Align,
2728                 llvm::Acquire);
2729    Builder.CreateBr(ContBB);
2730    SI->addCase(Builder.getInt32(1), AcquireBB);
2731    SI->addCase(Builder.getInt32(2), AcquireBB);
2732  }
2733  if (E->getOp() != AtomicExpr::Load) {
2734    Builder.SetInsertPoint(ReleaseBB);
2735    EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, Size, Align,
2736                 llvm::Release);
2737    Builder.CreateBr(ContBB);
2738    SI->addCase(Builder.getInt32(3), ReleaseBB);
2739  }
2740  if (E->getOp() != AtomicExpr::Load && E->getOp() != AtomicExpr::Store) {
2741    Builder.SetInsertPoint(AcqRelBB);
2742    EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, Size, Align,
2743                 llvm::AcquireRelease);
2744    Builder.CreateBr(ContBB);
2745    SI->addCase(Builder.getInt32(4), AcqRelBB);
2746  }
2747  Builder.SetInsertPoint(SeqCstBB);
2748  EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, Size, Align,
2749               llvm::SequentiallyConsistent);
2750  Builder.CreateBr(ContBB);
2751  SI->addCase(Builder.getInt32(5), SeqCstBB);
2752
2753  // Cleanup and return
2754  Builder.SetInsertPoint(ContBB);
2755  if (E->getOp() == AtomicExpr::Store)
2756    return RValue::get(0);
2757  return ConvertTempToRValue(*this, E->getType(), OrigDest);
2758}
2759