CGCall.cpp revision 9cbe4f0ba01ec304e1e3d071c071f7bca33631c0 
1//===--- CGCall.cpp - Encapsulate calling convention details ----*- C++ -*-===//
2//
3//                     The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10// These classes wrap the information about a call or function
11// definition used to handle ABI compliancy.
12//
13//===----------------------------------------------------------------------===//
14
15#include "CGCall.h"
16#include "CGCXXABI.h"
17#include "ABIInfo.h"
18#include "CodeGenFunction.h"
19#include "CodeGenModule.h"
20#include "clang/Basic/TargetInfo.h"
21#include "clang/AST/Decl.h"
22#include "clang/AST/DeclCXX.h"
23#include "clang/AST/DeclObjC.h"
24#include "clang/Frontend/CodeGenOptions.h"
25#include "llvm/Attributes.h"
26#include "llvm/Support/CallSite.h"
27#include "llvm/Target/TargetData.h"
28#include "llvm/InlineAsm.h"
29#include "llvm/Transforms/Utils/Local.h"
30using namespace clang;
31using namespace CodeGen;
32
33/***/
34
35static unsigned ClangCallConvToLLVMCallConv(CallingConv CC) {
36  switch (CC) {
37  default: return llvm::CallingConv::C;
38  case CC_X86StdCall: return llvm::CallingConv::X86_StdCall;
39  case CC_X86FastCall: return llvm::CallingConv::X86_FastCall;
40  case CC_X86ThisCall: return llvm::CallingConv::X86_ThisCall;
41  case CC_AAPCS: return llvm::CallingConv::ARM_AAPCS;
42  case CC_AAPCS_VFP: return llvm::CallingConv::ARM_AAPCS_VFP;
43  // TODO: add support for CC_X86Pascal to llvm
44  }
45}
46
47/// Derives the 'this' type for codegen purposes, i.e. ignoring method
48/// qualification.
49/// FIXME: address space qualification?
50static CanQualType GetThisType(ASTContext &Context, const CXXRecordDecl *RD) {
51  QualType RecTy = Context.getTagDeclType(RD)->getCanonicalTypeInternal();
52  return Context.getPointerType(CanQualType::CreateUnsafe(RecTy));
53}
54
55/// Returns the canonical formal type of the given C++ method.
56static CanQual<FunctionProtoType> GetFormalType(const CXXMethodDecl *MD) {
57  return MD->getType()->getCanonicalTypeUnqualified()
58           .getAs<FunctionProtoType>();
59}
60
61/// Returns the "extra-canonicalized" return type, which discards
62/// qualifiers on the return type.  Codegen doesn't care about them,
63/// and it makes ABI code a little easier to be able to assume that
64/// all parameter and return types are top-level unqualified.
65static CanQualType GetReturnType(QualType RetTy) {
66  return RetTy->getCanonicalTypeUnqualified().getUnqualifiedType();
67}
68
69const CGFunctionInfo &
70CodeGenTypes::getFunctionInfo(CanQual<FunctionNoProtoType> FTNP) {
71  return getFunctionInfo(FTNP->getResultType().getUnqualifiedType(),
72                         llvm::SmallVector<CanQualType, 16>(),
73                         FTNP->getExtInfo());
74}
75
76/// \param Args - contains any initial parameters besides those
77///   in the formal type
78static const CGFunctionInfo &getFunctionInfo(CodeGenTypes &CGT,
79                                  llvm::SmallVectorImpl<CanQualType> &ArgTys,
80                                             CanQual<FunctionProtoType> FTP) {
81  // FIXME: Kill copy.
82  for (unsigned i = 0, e = FTP->getNumArgs(); i != e; ++i)
83    ArgTys.push_back(FTP->getArgType(i));
84  CanQualType ResTy = FTP->getResultType().getUnqualifiedType();
85  return CGT.getFunctionInfo(ResTy, ArgTys, FTP->getExtInfo());
86}
87
88const CGFunctionInfo &
89CodeGenTypes::getFunctionInfo(CanQual<FunctionProtoType> FTP) {
90  llvm::SmallVector<CanQualType, 16> ArgTys;
91  return ::getFunctionInfo(*this, ArgTys, FTP);
92}
93
94static CallingConv getCallingConventionForDecl(const Decl *D) {
95  // Set the appropriate calling convention for the Function.
96  if (D->hasAttr<StdCallAttr>())
97    return CC_X86StdCall;
98
99  if (D->hasAttr<FastCallAttr>())
100    return CC_X86FastCall;
101
102  if (D->hasAttr<ThisCallAttr>())
103    return CC_X86ThisCall;
104
105  if (D->hasAttr<PascalAttr>())
106    return CC_X86Pascal;
107
108  if (PcsAttr *PCS = D->getAttr<PcsAttr>())
109    return (PCS->getPCS() == PcsAttr::AAPCS ? CC_AAPCS : CC_AAPCS_VFP);
110
111  return CC_C;
112}
113
114const CGFunctionInfo &CodeGenTypes::getFunctionInfo(const CXXRecordDecl *RD,
115                                                 const FunctionProtoType *FTP) {
116  llvm::SmallVector<CanQualType, 16> ArgTys;
117
118  // Add the 'this' pointer.
119  ArgTys.push_back(GetThisType(Context, RD));
120
121  return ::getFunctionInfo(*this, ArgTys,
122              FTP->getCanonicalTypeUnqualified().getAs<FunctionProtoType>());
123}
124
125const CGFunctionInfo &CodeGenTypes::getFunctionInfo(const CXXMethodDecl *MD) {
126  llvm::SmallVector<CanQualType, 16> ArgTys;
127
128  assert(!isa<CXXConstructorDecl>(MD) && "wrong method for contructors!");
129  assert(!isa<CXXDestructorDecl>(MD) && "wrong method for destructors!");
130
131  // Add the 'this' pointer unless this is a static method.
132  if (MD->isInstance())
133    ArgTys.push_back(GetThisType(Context, MD->getParent()));
134
135  return ::getFunctionInfo(*this, ArgTys, GetFormalType(MD));
136}
137
138const CGFunctionInfo &CodeGenTypes::getFunctionInfo(const CXXConstructorDecl *D,
139                                                    CXXCtorType Type) {
140  llvm::SmallVector<CanQualType, 16> ArgTys;
141  ArgTys.push_back(GetThisType(Context, D->getParent()));
142  CanQualType ResTy = Context.VoidTy;
143
144  TheCXXABI.BuildConstructorSignature(D, Type, ResTy, ArgTys);
145
146  CanQual<FunctionProtoType> FTP = GetFormalType(D);
147
148  // Add the formal parameters.
149  for (unsigned i = 0, e = FTP->getNumArgs(); i != e; ++i)
150    ArgTys.push_back(FTP->getArgType(i));
151
152  return getFunctionInfo(ResTy, ArgTys, FTP->getExtInfo());
153}
154
155const CGFunctionInfo &CodeGenTypes::getFunctionInfo(const CXXDestructorDecl *D,
156                                                    CXXDtorType Type) {
157  llvm::SmallVector<CanQualType, 2> ArgTys;
158  ArgTys.push_back(GetThisType(Context, D->getParent()));
159  CanQualType ResTy = Context.VoidTy;
160
161  TheCXXABI.BuildDestructorSignature(D, Type, ResTy, ArgTys);
162
163  CanQual<FunctionProtoType> FTP = GetFormalType(D);
164  assert(FTP->getNumArgs() == 0 && "dtor with formal parameters");
165
166  return getFunctionInfo(ResTy, ArgTys, FTP->getExtInfo());
167}
168
169const CGFunctionInfo &CodeGenTypes::getFunctionInfo(const FunctionDecl *FD) {
170  if (const CXXMethodDecl *MD = dyn_cast<CXXMethodDecl>(FD))
171    if (MD->isInstance())
172      return getFunctionInfo(MD);
173
174  CanQualType FTy = FD->getType()->getCanonicalTypeUnqualified();
175  assert(isa<FunctionType>(FTy));
176  if (isa<FunctionNoProtoType>(FTy))
177    return getFunctionInfo(FTy.getAs<FunctionNoProtoType>());
178  assert(isa<FunctionProtoType>(FTy));
179  return getFunctionInfo(FTy.getAs<FunctionProtoType>());
180}
181
182const CGFunctionInfo &CodeGenTypes::getFunctionInfo(const ObjCMethodDecl *MD) {
183  llvm::SmallVector<CanQualType, 16> ArgTys;
184  ArgTys.push_back(Context.getCanonicalParamType(MD->getSelfDecl()->getType()));
185  ArgTys.push_back(Context.getCanonicalParamType(Context.getObjCSelType()));
186  // FIXME: Kill copy?
187  for (ObjCMethodDecl::param_iterator i = MD->param_begin(),
188         e = MD->param_end(); i != e; ++i) {
189    ArgTys.push_back(Context.getCanonicalParamType((*i)->getType()));
190  }
191
192  FunctionType::ExtInfo einfo;
193  einfo = einfo.withCallingConv(getCallingConventionForDecl(MD));
194
195  if (getContext().getLangOptions().ObjCAutoRefCount &&
196      MD->hasAttr<NSReturnsRetainedAttr>())
197    einfo = einfo.withProducesResult(true);
198
199  return getFunctionInfo(GetReturnType(MD->getResultType()), ArgTys, einfo);
200}
201
202const CGFunctionInfo &CodeGenTypes::getFunctionInfo(GlobalDecl GD) {
203  // FIXME: Do we need to handle ObjCMethodDecl?
204  const FunctionDecl *FD = cast<FunctionDecl>(GD.getDecl());
205
206  if (const CXXConstructorDecl *CD = dyn_cast<CXXConstructorDecl>(FD))
207    return getFunctionInfo(CD, GD.getCtorType());
208
209  if (const CXXDestructorDecl *DD = dyn_cast<CXXDestructorDecl>(FD))
210    return getFunctionInfo(DD, GD.getDtorType());
211
212  return getFunctionInfo(FD);
213}
214
215const CGFunctionInfo &CodeGenTypes::getFunctionInfo(QualType ResTy,
216                                                    const CallArgList &Args,
217                                            const FunctionType::ExtInfo &Info) {
218  // FIXME: Kill copy.
219  llvm::SmallVector<CanQualType, 16> ArgTys;
220  for (CallArgList::const_iterator i = Args.begin(), e = Args.end();
221       i != e; ++i)
222    ArgTys.push_back(Context.getCanonicalParamType(i->Ty));
223  return getFunctionInfo(GetReturnType(ResTy), ArgTys, Info);
224}
225
226const CGFunctionInfo &CodeGenTypes::getFunctionInfo(QualType ResTy,
227                                                    const FunctionArgList &Args,
228                                            const FunctionType::ExtInfo &Info) {
229  // FIXME: Kill copy.
230  llvm::SmallVector<CanQualType, 16> ArgTys;
231  for (FunctionArgList::const_iterator i = Args.begin(), e = Args.end();
232       i != e; ++i)
233    ArgTys.push_back(Context.getCanonicalParamType((*i)->getType()));
234  return getFunctionInfo(GetReturnType(ResTy), ArgTys, Info);
235}
236
237const CGFunctionInfo &CodeGenTypes::getNullaryFunctionInfo() {
238  llvm::SmallVector<CanQualType, 1> args;
239  return getFunctionInfo(getContext().VoidTy, args, FunctionType::ExtInfo());
240}
241
242const CGFunctionInfo &CodeGenTypes::getFunctionInfo(CanQualType ResTy,
243                           const llvm::SmallVectorImpl<CanQualType> &ArgTys,
244                                            const FunctionType::ExtInfo &Info) {
245#ifndef NDEBUG
246  for (llvm::SmallVectorImpl<CanQualType>::const_iterator
247         I = ArgTys.begin(), E = ArgTys.end(); I != E; ++I)
248    assert(I->isCanonicalAsParam());
249#endif
250
251  unsigned CC = ClangCallConvToLLVMCallConv(Info.getCC());
252
253  // Lookup or create unique function info.
254  llvm::FoldingSetNodeID ID;
255  CGFunctionInfo::Profile(ID, Info, ResTy,
256                          ArgTys.begin(), ArgTys.end());
257
258  void *InsertPos = 0;
259  CGFunctionInfo *FI = FunctionInfos.FindNodeOrInsertPos(ID, InsertPos);
260  if (FI)
261    return *FI;
262
263  // Construct the function info.
264  FI = new CGFunctionInfo(CC, Info.getNoReturn(), Info.getProducesResult(),
265                          Info.getHasRegParm(), Info.getRegParm(), ResTy,
266                          ArgTys.data(), ArgTys.size());
267  FunctionInfos.InsertNode(FI, InsertPos);
268
269  // Compute ABI information.
270  getABIInfo().computeInfo(*FI);
271
272  // Loop over all of the computed argument and return value info.  If any of
273  // them are direct or extend without a specified coerce type, specify the
274  // default now.
275  ABIArgInfo &RetInfo = FI->getReturnInfo();
276  if (RetInfo.canHaveCoerceToType() && RetInfo.getCoerceToType() == 0)
277    RetInfo.setCoerceToType(ConvertType(FI->getReturnType()));
278
279  for (CGFunctionInfo::arg_iterator I = FI->arg_begin(), E = FI->arg_end();
280       I != E; ++I)
281    if (I->info.canHaveCoerceToType() && I->info.getCoerceToType() == 0)
282      I->info.setCoerceToType(ConvertType(I->type));
283
284  return *FI;
285}
286
287CGFunctionInfo::CGFunctionInfo(unsigned _CallingConvention,
288                               bool _NoReturn, bool returnsRetained,
289                               bool _HasRegParm, unsigned _RegParm,
290                               CanQualType ResTy,
291                               const CanQualType *ArgTys,
292                               unsigned NumArgTys)
293  : CallingConvention(_CallingConvention),
294    EffectiveCallingConvention(_CallingConvention),
295    NoReturn(_NoReturn), ReturnsRetained(returnsRetained),
296    HasRegParm(_HasRegParm), RegParm(_RegParm)
297{
298  NumArgs = NumArgTys;
299
300  // FIXME: Coallocate with the CGFunctionInfo object.
301  Args = new ArgInfo[1 + NumArgTys];
302  Args[0].type = ResTy;
303  for (unsigned i = 0; i != NumArgTys; ++i)
304    Args[1 + i].type = ArgTys[i];
305}
306
307/***/
308
309void CodeGenTypes::GetExpandedTypes(QualType type,
310                     llvm::SmallVectorImpl<llvm::Type*> &expandedTypes) {
311  const RecordType *RT = type->getAsStructureType();
312  assert(RT && "Can only expand structure types.");
313  const RecordDecl *RD = RT->getDecl();
314  assert(!RD->hasFlexibleArrayMember() &&
315         "Cannot expand structure with flexible array.");
316
317  for (RecordDecl::field_iterator i = RD->field_begin(), e = RD->field_end();
318         i != e; ++i) {
319    const FieldDecl *FD = *i;
320    assert(!FD->isBitField() &&
321           "Cannot expand structure with bit-field members.");
322
323    QualType fieldType = FD->getType();
324    if (fieldType->isRecordType())
325      GetExpandedTypes(fieldType, expandedTypes);
326    else
327      expandedTypes.push_back(ConvertType(fieldType));
328  }
329}
330
331llvm::Function::arg_iterator
332CodeGenFunction::ExpandTypeFromArgs(QualType Ty, LValue LV,
333                                    llvm::Function::arg_iterator AI) {
334  const RecordType *RT = Ty->getAsStructureType();
335  assert(RT && "Can only expand structure types.");
336
337  RecordDecl *RD = RT->getDecl();
338  assert(LV.isSimple() &&
339         "Unexpected non-simple lvalue during struct expansion.");
340  llvm::Value *Addr = LV.getAddress();
341  for (RecordDecl::field_iterator i = RD->field_begin(), e = RD->field_end();
342         i != e; ++i) {
343    FieldDecl *FD = *i;
344    QualType FT = FD->getType();
345
346    // FIXME: What are the right qualifiers here?
347    LValue LV = EmitLValueForField(Addr, FD, 0);
348    if (CodeGenFunction::hasAggregateLLVMType(FT)) {
349      AI = ExpandTypeFromArgs(FT, LV, AI);
350    } else {
351      EmitStoreThroughLValue(RValue::get(AI), LV);
352      ++AI;
353    }
354  }
355
356  return AI;
357}
358
359void
360CodeGenFunction::ExpandTypeToArgs(QualType Ty, RValue RV,
361                                  llvm::SmallVector<llvm::Value*, 16> &Args) {
362  const RecordType *RT = Ty->getAsStructureType();
363  assert(RT && "Can only expand structure types.");
364
365  RecordDecl *RD = RT->getDecl();
366  assert(RV.isAggregate() && "Unexpected rvalue during struct expansion");
367  llvm::Value *Addr = RV.getAggregateAddr();
368  for (RecordDecl::field_iterator i = RD->field_begin(), e = RD->field_end();
369         i != e; ++i) {
370    FieldDecl *FD = *i;
371    QualType FT = FD->getType();
372
373    // FIXME: What are the right qualifiers here?
374    LValue LV = EmitLValueForField(Addr, FD, 0);
375    if (CodeGenFunction::hasAggregateLLVMType(FT)) {
376      ExpandTypeToArgs(FT, RValue::getAggregate(LV.getAddress()), Args);
377    } else {
378      RValue RV = EmitLoadOfLValue(LV);
379      assert(RV.isScalar() &&
380             "Unexpected non-scalar rvalue during struct expansion.");
381      Args.push_back(RV.getScalarVal());
382    }
383  }
384}
385
386/// EnterStructPointerForCoercedAccess - Given a struct pointer that we are
387/// accessing some number of bytes out of it, try to gep into the struct to get
388/// at its inner goodness.  Dive as deep as possible without entering an element
389/// with an in-memory size smaller than DstSize.
390static llvm::Value *
391EnterStructPointerForCoercedAccess(llvm::Value *SrcPtr,
392                                   const llvm::StructType *SrcSTy,
393                                   uint64_t DstSize, CodeGenFunction &CGF) {
394  // We can't dive into a zero-element struct.
395  if (SrcSTy->getNumElements() == 0) return SrcPtr;
396
397  const llvm::Type *FirstElt = SrcSTy->getElementType(0);
398
399  // If the first elt is at least as large as what we're looking for, or if the
400  // first element is the same size as the whole struct, we can enter it.
401  uint64_t FirstEltSize =
402    CGF.CGM.getTargetData().getTypeAllocSize(FirstElt);
403  if (FirstEltSize < DstSize &&
404      FirstEltSize < CGF.CGM.getTargetData().getTypeAllocSize(SrcSTy))
405    return SrcPtr;
406
407  // GEP into the first element.
408  SrcPtr = CGF.Builder.CreateConstGEP2_32(SrcPtr, 0, 0, "coerce.dive");
409
410  // If the first element is a struct, recurse.
411  const llvm::Type *SrcTy =
412    cast<llvm::PointerType>(SrcPtr->getType())->getElementType();
413  if (const llvm::StructType *SrcSTy = dyn_cast<llvm::StructType>(SrcTy))
414    return EnterStructPointerForCoercedAccess(SrcPtr, SrcSTy, DstSize, CGF);
415
416  return SrcPtr;
417}
418
419/// CoerceIntOrPtrToIntOrPtr - Convert a value Val to the specific Ty where both
420/// are either integers or pointers.  This does a truncation of the value if it
421/// is too large or a zero extension if it is too small.
422static llvm::Value *CoerceIntOrPtrToIntOrPtr(llvm::Value *Val,
423                                             const llvm::Type *Ty,
424                                             CodeGenFunction &CGF) {
425  if (Val->getType() == Ty)
426    return Val;
427
428  if (isa<llvm::PointerType>(Val->getType())) {
429    // If this is Pointer->Pointer avoid conversion to and from int.
430    if (isa<llvm::PointerType>(Ty))
431      return CGF.Builder.CreateBitCast(Val, Ty, "coerce.val");
432
433    // Convert the pointer to an integer so we can play with its width.
434    Val = CGF.Builder.CreatePtrToInt(Val, CGF.IntPtrTy, "coerce.val.pi");
435  }
436
437  const llvm::Type *DestIntTy = Ty;
438  if (isa<llvm::PointerType>(DestIntTy))
439    DestIntTy = CGF.IntPtrTy;
440
441  if (Val->getType() != DestIntTy)
442    Val = CGF.Builder.CreateIntCast(Val, DestIntTy, false, "coerce.val.ii");
443
444  if (isa<llvm::PointerType>(Ty))
445    Val = CGF.Builder.CreateIntToPtr(Val, Ty, "coerce.val.ip");
446  return Val;
447}
448
449
450
451/// CreateCoercedLoad - Create a load from \arg SrcPtr interpreted as
452/// a pointer to an object of type \arg Ty.
453///
454/// This safely handles the case when the src type is smaller than the
455/// destination type; in this situation the values of bits which not
456/// present in the src are undefined.
457static llvm::Value *CreateCoercedLoad(llvm::Value *SrcPtr,
458                                      const llvm::Type *Ty,
459                                      CodeGenFunction &CGF) {
460  const llvm::Type *SrcTy =
461    cast<llvm::PointerType>(SrcPtr->getType())->getElementType();
462
463  // If SrcTy and Ty are the same, just do a load.
464  if (SrcTy == Ty)
465    return CGF.Builder.CreateLoad(SrcPtr);
466
467  uint64_t DstSize = CGF.CGM.getTargetData().getTypeAllocSize(Ty);
468
469  if (const llvm::StructType *SrcSTy = dyn_cast<llvm::StructType>(SrcTy)) {
470    SrcPtr = EnterStructPointerForCoercedAccess(SrcPtr, SrcSTy, DstSize, CGF);
471    SrcTy = cast<llvm::PointerType>(SrcPtr->getType())->getElementType();
472  }
473
474  uint64_t SrcSize = CGF.CGM.getTargetData().getTypeAllocSize(SrcTy);
475
476  // If the source and destination are integer or pointer types, just do an
477  // extension or truncation to the desired type.
478  if ((isa<llvm::IntegerType>(Ty) || isa<llvm::PointerType>(Ty)) &&
479      (isa<llvm::IntegerType>(SrcTy) || isa<llvm::PointerType>(SrcTy))) {
480    llvm::LoadInst *Load = CGF.Builder.CreateLoad(SrcPtr);
481    return CoerceIntOrPtrToIntOrPtr(Load, Ty, CGF);
482  }
483
484  // If load is legal, just bitcast the src pointer.
485  if (SrcSize >= DstSize) {
486    // Generally SrcSize is never greater than DstSize, since this means we are
487    // losing bits. However, this can happen in cases where the structure has
488    // additional padding, for example due to a user specified alignment.
489    //
490    // FIXME: Assert that we aren't truncating non-padding bits when have access
491    // to that information.
492    llvm::Value *Casted =
493      CGF.Builder.CreateBitCast(SrcPtr, llvm::PointerType::getUnqual(Ty));
494    llvm::LoadInst *Load = CGF.Builder.CreateLoad(Casted);
495    // FIXME: Use better alignment / avoid requiring aligned load.
496    Load->setAlignment(1);
497    return Load;
498  }
499
500  // Otherwise do coercion through memory. This is stupid, but
501  // simple.
502  llvm::Value *Tmp = CGF.CreateTempAlloca(Ty);
503  llvm::Value *Casted =
504    CGF.Builder.CreateBitCast(Tmp, llvm::PointerType::getUnqual(SrcTy));
505  llvm::StoreInst *Store =
506    CGF.Builder.CreateStore(CGF.Builder.CreateLoad(SrcPtr), Casted);
507  // FIXME: Use better alignment / avoid requiring aligned store.
508  Store->setAlignment(1);
509  return CGF.Builder.CreateLoad(Tmp);
510}
511
512// Function to store a first-class aggregate into memory.  We prefer to
513// store the elements rather than the aggregate to be more friendly to
514// fast-isel.
515// FIXME: Do we need to recurse here?
516static void BuildAggStore(CodeGenFunction &CGF, llvm::Value *Val,
517                          llvm::Value *DestPtr, bool DestIsVolatile,
518                          bool LowAlignment) {
519  // Prefer scalar stores to first-class aggregate stores.
520  if (const llvm::StructType *STy =
521        dyn_cast<llvm::StructType>(Val->getType())) {
522    for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) {
523      llvm::Value *EltPtr = CGF.Builder.CreateConstGEP2_32(DestPtr, 0, i);
524      llvm::Value *Elt = CGF.Builder.CreateExtractValue(Val, i);
525      llvm::StoreInst *SI = CGF.Builder.CreateStore(Elt, EltPtr,
526                                                    DestIsVolatile);
527      if (LowAlignment)
528        SI->setAlignment(1);
529    }
530  } else {
531    CGF.Builder.CreateStore(Val, DestPtr, DestIsVolatile);
532  }
533}
534
535/// CreateCoercedStore - Create a store to \arg DstPtr from \arg Src,
536/// where the source and destination may have different types.
537///
538/// This safely handles the case when the src type is larger than the
539/// destination type; the upper bits of the src will be lost.
540static void CreateCoercedStore(llvm::Value *Src,
541                               llvm::Value *DstPtr,
542                               bool DstIsVolatile,
543                               CodeGenFunction &CGF) {
544  const llvm::Type *SrcTy = Src->getType();
545  const llvm::Type *DstTy =
546    cast<llvm::PointerType>(DstPtr->getType())->getElementType();
547  if (SrcTy == DstTy) {
548    CGF.Builder.CreateStore(Src, DstPtr, DstIsVolatile);
549    return;
550  }
551
552  uint64_t SrcSize = CGF.CGM.getTargetData().getTypeAllocSize(SrcTy);
553
554  if (const llvm::StructType *DstSTy = dyn_cast<llvm::StructType>(DstTy)) {
555    DstPtr = EnterStructPointerForCoercedAccess(DstPtr, DstSTy, SrcSize, CGF);
556    DstTy = cast<llvm::PointerType>(DstPtr->getType())->getElementType();
557  }
558
559  // If the source and destination are integer or pointer types, just do an
560  // extension or truncation to the desired type.
561  if ((isa<llvm::IntegerType>(SrcTy) || isa<llvm::PointerType>(SrcTy)) &&
562      (isa<llvm::IntegerType>(DstTy) || isa<llvm::PointerType>(DstTy))) {
563    Src = CoerceIntOrPtrToIntOrPtr(Src, DstTy, CGF);
564    CGF.Builder.CreateStore(Src, DstPtr, DstIsVolatile);
565    return;
566  }
567
568  uint64_t DstSize = CGF.CGM.getTargetData().getTypeAllocSize(DstTy);
569
570  // If store is legal, just bitcast the src pointer.
571  if (SrcSize <= DstSize) {
572    llvm::Value *Casted =
573      CGF.Builder.CreateBitCast(DstPtr, llvm::PointerType::getUnqual(SrcTy));
574    // FIXME: Use better alignment / avoid requiring aligned store.
575    BuildAggStore(CGF, Src, Casted, DstIsVolatile, true);
576  } else {
577    // Otherwise do coercion through memory. This is stupid, but
578    // simple.
579
580    // Generally SrcSize is never greater than DstSize, since this means we are
581    // losing bits. However, this can happen in cases where the structure has
582    // additional padding, for example due to a user specified alignment.
583    //
584    // FIXME: Assert that we aren't truncating non-padding bits when have access
585    // to that information.
586    llvm::Value *Tmp = CGF.CreateTempAlloca(SrcTy);
587    CGF.Builder.CreateStore(Src, Tmp);
588    llvm::Value *Casted =
589      CGF.Builder.CreateBitCast(Tmp, llvm::PointerType::getUnqual(DstTy));
590    llvm::LoadInst *Load = CGF.Builder.CreateLoad(Casted);
591    // FIXME: Use better alignment / avoid requiring aligned load.
592    Load->setAlignment(1);
593    CGF.Builder.CreateStore(Load, DstPtr, DstIsVolatile);
594  }
595}
596
597/***/
598
599bool CodeGenModule::ReturnTypeUsesSRet(const CGFunctionInfo &FI) {
600  return FI.getReturnInfo().isIndirect();
601}
602
603bool CodeGenModule::ReturnTypeUsesFPRet(QualType ResultType) {
604  if (const BuiltinType *BT = ResultType->getAs<BuiltinType>()) {
605    switch (BT->getKind()) {
606    default:
607      return false;
608    case BuiltinType::Float:
609      return getContext().Target.useObjCFPRetForRealType(TargetInfo::Float);
610    case BuiltinType::Double:
611      return getContext().Target.useObjCFPRetForRealType(TargetInfo::Double);
612    case BuiltinType::LongDouble:
613      return getContext().Target.useObjCFPRetForRealType(
614        TargetInfo::LongDouble);
615    }
616  }
617
618  return false;
619}
620
621llvm::FunctionType *CodeGenTypes::GetFunctionType(GlobalDecl GD) {
622  const CGFunctionInfo &FI = getFunctionInfo(GD);
623
624  // For definition purposes, don't consider a K&R function variadic.
625  bool Variadic = false;
626  if (const FunctionProtoType *FPT =
627        cast<FunctionDecl>(GD.getDecl())->getType()->getAs<FunctionProtoType>())
628    Variadic = FPT->isVariadic();
629
630  return GetFunctionType(FI, Variadic);
631}
632
633llvm::FunctionType *
634CodeGenTypes::GetFunctionType(const CGFunctionInfo &FI, bool isVariadic) {
635  llvm::SmallVector<llvm::Type*, 8> argTypes;
636  const llvm::Type *resultType = 0;
637
638  const ABIArgInfo &retAI = FI.getReturnInfo();
639  switch (retAI.getKind()) {
640  case ABIArgInfo::Expand:
641    llvm_unreachable("Invalid ABI kind for return argument");
642
643  case ABIArgInfo::Extend:
644  case ABIArgInfo::Direct:
645    resultType = retAI.getCoerceToType();
646    break;
647
648  case ABIArgInfo::Indirect: {
649    assert(!retAI.getIndirectAlign() && "Align unused on indirect return.");
650    resultType = llvm::Type::getVoidTy(getLLVMContext());
651
652    QualType ret = FI.getReturnType();
653    const llvm::Type *ty = ConvertType(ret);
654    unsigned addressSpace = Context.getTargetAddressSpace(ret);
655    argTypes.push_back(llvm::PointerType::get(ty, addressSpace));
656    break;
657  }
658
659  case ABIArgInfo::Ignore:
660    resultType = llvm::Type::getVoidTy(getLLVMContext());
661    break;
662  }
663
664  for (CGFunctionInfo::const_arg_iterator it = FI.arg_begin(),
665         ie = FI.arg_end(); it != ie; ++it) {
666    const ABIArgInfo &argAI = it->info;
667
668    switch (argAI.getKind()) {
669    case ABIArgInfo::Ignore:
670      break;
671
672    case ABIArgInfo::Indirect: {
673      // indirect arguments are always on the stack, which is addr space #0.
674      const llvm::Type *LTy = ConvertTypeForMem(it->type);
675      argTypes.push_back(LTy->getPointerTo());
676      break;
677    }
678
679    case ABIArgInfo::Extend:
680    case ABIArgInfo::Direct: {
681      // If the coerce-to type is a first class aggregate, flatten it.  Either
682      // way is semantically identical, but fast-isel and the optimizer
683      // generally likes scalar values better than FCAs.
684      llvm::Type *argType = argAI.getCoerceToType();
685      if (const llvm::StructType *st = dyn_cast<llvm::StructType>(argType)) {
686        for (unsigned i = 0, e = st->getNumElements(); i != e; ++i)
687          argTypes.push_back(st->getElementType(i));
688      } else {
689        argTypes.push_back(argType);
690      }
691      break;
692    }
693
694    case ABIArgInfo::Expand:
695      GetExpandedTypes(it->type, argTypes);
696      break;
697    }
698  }
699
700  return llvm::FunctionType::get(resultType, argTypes, isVariadic);
701}
702
703const llvm::Type *CodeGenTypes::GetFunctionTypeForVTable(GlobalDecl GD) {
704  const CXXMethodDecl *MD = cast<CXXMethodDecl>(GD.getDecl());
705  const FunctionProtoType *FPT = MD->getType()->getAs<FunctionProtoType>();
706
707  if (!VerifyFuncTypeComplete(FPT)) {
708    const CGFunctionInfo *Info;
709    if (isa<CXXDestructorDecl>(MD))
710      Info = &getFunctionInfo(cast<CXXDestructorDecl>(MD), GD.getDtorType());
711    else
712      Info = &getFunctionInfo(MD);
713    return GetFunctionType(*Info, FPT->isVariadic());
714  }
715
716  return llvm::StructType::get(getLLVMContext());
717}
718
719void CodeGenModule::ConstructAttributeList(const CGFunctionInfo &FI,
720                                           const Decl *TargetDecl,
721                                           AttributeListType &PAL,
722                                           unsigned &CallingConv) {
723  unsigned FuncAttrs = 0;
724  unsigned RetAttrs = 0;
725
726  CallingConv = FI.getEffectiveCallingConvention();
727
728  if (FI.isNoReturn())
729    FuncAttrs |= llvm::Attribute::NoReturn;
730
731  // FIXME: handle sseregparm someday...
732  if (TargetDecl) {
733    if (TargetDecl->hasAttr<NoThrowAttr>())
734      FuncAttrs |= llvm::Attribute::NoUnwind;
735    else if (const FunctionDecl *Fn = dyn_cast<FunctionDecl>(TargetDecl)) {
736      const FunctionProtoType *FPT = Fn->getType()->getAs<FunctionProtoType>();
737      if (FPT && FPT->isNothrow(getContext()))
738        FuncAttrs |= llvm::Attribute::NoUnwind;
739    }
740
741    if (TargetDecl->hasAttr<NoReturnAttr>())
742      FuncAttrs |= llvm::Attribute::NoReturn;
743    if (TargetDecl->hasAttr<ConstAttr>())
744      FuncAttrs |= llvm::Attribute::ReadNone;
745    else if (TargetDecl->hasAttr<PureAttr>())
746      FuncAttrs |= llvm::Attribute::ReadOnly;
747    if (TargetDecl->hasAttr<MallocAttr>())
748      RetAttrs |= llvm::Attribute::NoAlias;
749  }
750
751  if (CodeGenOpts.OptimizeSize)
752    FuncAttrs |= llvm::Attribute::OptimizeForSize;
753  if (CodeGenOpts.DisableRedZone)
754    FuncAttrs |= llvm::Attribute::NoRedZone;
755  if (CodeGenOpts.NoImplicitFloat)
756    FuncAttrs |= llvm::Attribute::NoImplicitFloat;
757
758  QualType RetTy = FI.getReturnType();
759  unsigned Index = 1;
760  const ABIArgInfo &RetAI = FI.getReturnInfo();
761  switch (RetAI.getKind()) {
762  case ABIArgInfo::Extend:
763   if (RetTy->hasSignedIntegerRepresentation())
764     RetAttrs |= llvm::Attribute::SExt;
765   else if (RetTy->hasUnsignedIntegerRepresentation())
766     RetAttrs |= llvm::Attribute::ZExt;
767    break;
768  case ABIArgInfo::Direct:
769  case ABIArgInfo::Ignore:
770    break;
771
772  case ABIArgInfo::Indirect:
773    PAL.push_back(llvm::AttributeWithIndex::get(Index,
774                                                llvm::Attribute::StructRet));
775    ++Index;
776    // sret disables readnone and readonly
777    FuncAttrs &= ~(llvm::Attribute::ReadOnly |
778                   llvm::Attribute::ReadNone);
779    break;
780
781  case ABIArgInfo::Expand:
782    assert(0 && "Invalid ABI kind for return argument");
783  }
784
785  if (RetAttrs)
786    PAL.push_back(llvm::AttributeWithIndex::get(0, RetAttrs));
787
788  // FIXME: RegParm should be reduced in case of global register variable.
789  signed RegParm;
790  if (FI.getHasRegParm())
791    RegParm = FI.getRegParm();
792  else
793    RegParm = CodeGenOpts.NumRegisterParameters;
794
795  unsigned PointerWidth = getContext().Target.getPointerWidth(0);
796  for (CGFunctionInfo::const_arg_iterator it = FI.arg_begin(),
797         ie = FI.arg_end(); it != ie; ++it) {
798    QualType ParamType = it->type;
799    const ABIArgInfo &AI = it->info;
800    unsigned Attributes = 0;
801
802    // 'restrict' -> 'noalias' is done in EmitFunctionProlog when we
803    // have the corresponding parameter variable.  It doesn't make
804    // sense to do it here because parameters are so messed up.
805    switch (AI.getKind()) {
806    case ABIArgInfo::Extend:
807      if (ParamType->isSignedIntegerOrEnumerationType())
808        Attributes |= llvm::Attribute::SExt;
809      else if (ParamType->isUnsignedIntegerOrEnumerationType())
810        Attributes |= llvm::Attribute::ZExt;
811      // FALL THROUGH
812    case ABIArgInfo::Direct:
813      if (RegParm > 0 &&
814          (ParamType->isIntegerType() || ParamType->isPointerType())) {
815        RegParm -=
816        (Context.getTypeSize(ParamType) + PointerWidth - 1) / PointerWidth;
817        if (RegParm >= 0)
818          Attributes |= llvm::Attribute::InReg;
819      }
820      // FIXME: handle sseregparm someday...
821
822      if (const llvm::StructType *STy =
823            dyn_cast<llvm::StructType>(AI.getCoerceToType()))
824        Index += STy->getNumElements()-1;  // 1 will be added below.
825      break;
826
827    case ABIArgInfo::Indirect:
828      if (AI.getIndirectByVal())
829        Attributes |= llvm::Attribute::ByVal;
830
831      Attributes |=
832        llvm::Attribute::constructAlignmentFromInt(AI.getIndirectAlign());
833      // byval disables readnone and readonly.
834      FuncAttrs &= ~(llvm::Attribute::ReadOnly |
835                     llvm::Attribute::ReadNone);
836      break;
837
838    case ABIArgInfo::Ignore:
839      // Skip increment, no matching LLVM parameter.
840      continue;
841
842    case ABIArgInfo::Expand: {
843      llvm::SmallVector<llvm::Type*, 8> types;
844      // FIXME: This is rather inefficient. Do we ever actually need to do
845      // anything here? The result should be just reconstructed on the other
846      // side, so extension should be a non-issue.
847      getTypes().GetExpandedTypes(ParamType, types);
848      Index += types.size();
849      continue;
850    }
851    }
852
853    if (Attributes)
854      PAL.push_back(llvm::AttributeWithIndex::get(Index, Attributes));
855    ++Index;
856  }
857  if (FuncAttrs)
858    PAL.push_back(llvm::AttributeWithIndex::get(~0, FuncAttrs));
859}
860
861/// An argument came in as a promoted argument; demote it back to its
862/// declared type.
863static llvm::Value *emitArgumentDemotion(CodeGenFunction &CGF,
864                                         const VarDecl *var,
865                                         llvm::Value *value) {
866  const llvm::Type *varType = CGF.ConvertType(var->getType());
867
868  // This can happen with promotions that actually don't change the
869  // underlying type, like the enum promotions.
870  if (value->getType() == varType) return value;
871
872  assert((varType->isIntegerTy() || varType->isFloatingPointTy())
873         && "unexpected promotion type");
874
875  if (isa<llvm::IntegerType>(varType))
876    return CGF.Builder.CreateTrunc(value, varType, "arg.unpromote");
877
878  return CGF.Builder.CreateFPCast(value, varType, "arg.unpromote");
879}
880
881void CodeGenFunction::EmitFunctionProlog(const CGFunctionInfo &FI,
882                                         llvm::Function *Fn,
883                                         const FunctionArgList &Args) {
884  // If this is an implicit-return-zero function, go ahead and
885  // initialize the return value.  TODO: it might be nice to have
886  // a more general mechanism for this that didn't require synthesized
887  // return statements.
888  if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(CurFuncDecl)) {
889    if (FD->hasImplicitReturnZero()) {
890      QualType RetTy = FD->getResultType().getUnqualifiedType();
891      const llvm::Type* LLVMTy = CGM.getTypes().ConvertType(RetTy);
892      llvm::Constant* Zero = llvm::Constant::getNullValue(LLVMTy);
893      Builder.CreateStore(Zero, ReturnValue);
894    }
895  }
896
897  // FIXME: We no longer need the types from FunctionArgList; lift up and
898  // simplify.
899
900  // Emit allocs for param decls.  Give the LLVM Argument nodes names.
901  llvm::Function::arg_iterator AI = Fn->arg_begin();
902
903  // Name the struct return argument.
904  if (CGM.ReturnTypeUsesSRet(FI)) {
905    AI->setName("agg.result");
906    ++AI;
907  }
908
909  assert(FI.arg_size() == Args.size() &&
910         "Mismatch between function signature & arguments.");
911  unsigned ArgNo = 1;
912  CGFunctionInfo::const_arg_iterator info_it = FI.arg_begin();
913  for (FunctionArgList::const_iterator i = Args.begin(), e = Args.end();
914       i != e; ++i, ++info_it, ++ArgNo) {
915    const VarDecl *Arg = *i;
916    QualType Ty = info_it->type;
917    const ABIArgInfo &ArgI = info_it->info;
918
919    bool isPromoted =
920      isa<ParmVarDecl>(Arg) && cast<ParmVarDecl>(Arg)->isKNRPromoted();
921
922    switch (ArgI.getKind()) {
923    case ABIArgInfo::Indirect: {
924      llvm::Value *V = AI;
925
926      if (hasAggregateLLVMType(Ty)) {
927        // Aggregates and complex variables are accessed by reference.  All we
928        // need to do is realign the value, if requested
929        if (ArgI.getIndirectRealign()) {
930          llvm::Value *AlignedTemp = CreateMemTemp(Ty, "coerce");
931
932          // Copy from the incoming argument pointer to the temporary with the
933          // appropriate alignment.
934          //
935          // FIXME: We should have a common utility for generating an aggregate
936          // copy.
937          const llvm::Type *I8PtrTy = Builder.getInt8PtrTy();
938          CharUnits Size = getContext().getTypeSizeInChars(Ty);
939          llvm::Value *Dst = Builder.CreateBitCast(AlignedTemp, I8PtrTy);
940          llvm::Value *Src = Builder.CreateBitCast(V, I8PtrTy);
941          Builder.CreateMemCpy(Dst,
942                               Src,
943                               llvm::ConstantInt::get(IntPtrTy,
944                                                      Size.getQuantity()),
945                               ArgI.getIndirectAlign(),
946                               false);
947          V = AlignedTemp;
948        }
949      } else {
950        // Load scalar value from indirect argument.
951        CharUnits Alignment = getContext().getTypeAlignInChars(Ty);
952        V = EmitLoadOfScalar(V, false, Alignment.getQuantity(), Ty);
953
954        if (isPromoted)
955          V = emitArgumentDemotion(*this, Arg, V);
956      }
957      EmitParmDecl(*Arg, V, ArgNo);
958      break;
959    }
960
961    case ABIArgInfo::Extend:
962    case ABIArgInfo::Direct: {
963      // If we have the trivial case, handle it with no muss and fuss.
964      if (!isa<llvm::StructType>(ArgI.getCoerceToType()) &&
965          ArgI.getCoerceToType() == ConvertType(Ty) &&
966          ArgI.getDirectOffset() == 0) {
967        assert(AI != Fn->arg_end() && "Argument mismatch!");
968        llvm::Value *V = AI;
969
970        if (Arg->getType().isRestrictQualified())
971          AI->addAttr(llvm::Attribute::NoAlias);
972
973        if (isPromoted)
974          V = emitArgumentDemotion(*this, Arg, V);
975
976        EmitParmDecl(*Arg, V, ArgNo);
977        break;
978      }
979
980      llvm::AllocaInst *Alloca = CreateMemTemp(Ty, "coerce");
981
982      // The alignment we need to use is the max of the requested alignment for
983      // the argument plus the alignment required by our access code below.
984      unsigned AlignmentToUse =
985        CGM.getTargetData().getABITypeAlignment(ArgI.getCoerceToType());
986      AlignmentToUse = std::max(AlignmentToUse,
987                        (unsigned)getContext().getDeclAlign(Arg).getQuantity());
988
989      Alloca->setAlignment(AlignmentToUse);
990      llvm::Value *V = Alloca;
991      llvm::Value *Ptr = V;    // Pointer to store into.
992
993      // If the value is offset in memory, apply the offset now.
994      if (unsigned Offs = ArgI.getDirectOffset()) {
995        Ptr = Builder.CreateBitCast(Ptr, Builder.getInt8PtrTy());
996        Ptr = Builder.CreateConstGEP1_32(Ptr, Offs);
997        Ptr = Builder.CreateBitCast(Ptr,
998                          llvm::PointerType::getUnqual(ArgI.getCoerceToType()));
999      }
1000
1001      // If the coerce-to type is a first class aggregate, we flatten it and
1002      // pass the elements. Either way is semantically identical, but fast-isel
1003      // and the optimizer generally likes scalar values better than FCAs.
1004      if (const llvm::StructType *STy =
1005            dyn_cast<llvm::StructType>(ArgI.getCoerceToType())) {
1006        Ptr = Builder.CreateBitCast(Ptr, llvm::PointerType::getUnqual(STy));
1007
1008        for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) {
1009          assert(AI != Fn->arg_end() && "Argument mismatch!");
1010          AI->setName(Arg->getName() + ".coerce" + llvm::Twine(i));
1011          llvm::Value *EltPtr = Builder.CreateConstGEP2_32(Ptr, 0, i);
1012          Builder.CreateStore(AI++, EltPtr);
1013        }
1014      } else {
1015        // Simple case, just do a coerced store of the argument into the alloca.
1016        assert(AI != Fn->arg_end() && "Argument mismatch!");
1017        AI->setName(Arg->getName() + ".coerce");
1018        CreateCoercedStore(AI++, Ptr, /*DestIsVolatile=*/false, *this);
1019      }
1020
1021
1022      // Match to what EmitParmDecl is expecting for this type.
1023      if (!CodeGenFunction::hasAggregateLLVMType(Ty)) {
1024        V = EmitLoadOfScalar(V, false, AlignmentToUse, Ty);
1025        if (isPromoted)
1026          V = emitArgumentDemotion(*this, Arg, V);
1027      }
1028      EmitParmDecl(*Arg, V, ArgNo);
1029      continue;  // Skip ++AI increment, already done.
1030    }
1031
1032    case ABIArgInfo::Expand: {
1033      // If this structure was expanded into multiple arguments then
1034      // we need to create a temporary and reconstruct it from the
1035      // arguments.
1036      llvm::Value *Temp = CreateMemTemp(Ty, Arg->getName() + ".addr");
1037      llvm::Function::arg_iterator End =
1038        ExpandTypeFromArgs(Ty, MakeAddrLValue(Temp, Ty), AI);
1039      EmitParmDecl(*Arg, Temp, ArgNo);
1040
1041      // Name the arguments used in expansion and increment AI.
1042      unsigned Index = 0;
1043      for (; AI != End; ++AI, ++Index)
1044        AI->setName(Arg->getName() + "." + llvm::Twine(Index));
1045      continue;
1046    }
1047
1048    case ABIArgInfo::Ignore:
1049      // Initialize the local variable appropriately.
1050      if (hasAggregateLLVMType(Ty))
1051        EmitParmDecl(*Arg, CreateMemTemp(Ty), ArgNo);
1052      else
1053        EmitParmDecl(*Arg, llvm::UndefValue::get(ConvertType(Arg->getType())),
1054                     ArgNo);
1055
1056      // Skip increment, no matching LLVM parameter.
1057      continue;
1058    }
1059
1060    ++AI;
1061  }
1062  assert(AI == Fn->arg_end() && "Argument mismatch!");
1063}
1064
1065/// Try to emit a fused autorelease of a return result.
1066static llvm::Value *tryEmitFusedAutoreleaseOfResult(CodeGenFunction &CGF,
1067                                                    llvm::Value *result) {
1068  // We must be immediately followed the cast.
1069  llvm::BasicBlock *BB = CGF.Builder.GetInsertBlock();
1070  if (BB->empty()) return 0;
1071  if (&BB->back() != result) return 0;
1072
1073  const llvm::Type *resultType = result->getType();
1074
1075  // result is in a BasicBlock and is therefore an Instruction.
1076  llvm::Instruction *generator = cast<llvm::Instruction>(result);
1077
1078  llvm::SmallVector<llvm::Instruction*,4> insnsToKill;
1079
1080  // Look for:
1081  //  %generator = bitcast %type1* %generator2 to %type2*
1082  while (llvm::BitCastInst *bitcast = dyn_cast<llvm::BitCastInst>(generator)) {
1083    // We would have emitted this as a constant if the operand weren't
1084    // an Instruction.
1085    generator = cast<llvm::Instruction>(bitcast->getOperand(0));
1086
1087    // Require the generator to be immediately followed by the cast.
1088    if (generator->getNextNode() != bitcast)
1089      return 0;
1090
1091    insnsToKill.push_back(bitcast);
1092  }
1093
1094  // Look for:
1095  //   %generator = call i8* @objc_retain(i8* %originalResult)
1096  // or
1097  //   %generator = call i8* @objc_retainAutoreleasedReturnValue(i8* %originalResult)
1098  llvm::CallInst *call = dyn_cast<llvm::CallInst>(generator);
1099  if (!call) return 0;
1100
1101  bool doRetainAutorelease;
1102
1103  if (call->getCalledValue() == CGF.CGM.getARCEntrypoints().objc_retain) {
1104    doRetainAutorelease = true;
1105  } else if (call->getCalledValue() == CGF.CGM.getARCEntrypoints()
1106                                          .objc_retainAutoreleasedReturnValue) {
1107    doRetainAutorelease = false;
1108
1109    // Look for an inline asm immediately preceding the call and kill it, too.
1110    llvm::Instruction *prev = call->getPrevNode();
1111    if (llvm::CallInst *asmCall = dyn_cast_or_null<llvm::CallInst>(prev))
1112      if (asmCall->getCalledValue()
1113            == CGF.CGM.getARCEntrypoints().retainAutoreleasedReturnValueMarker)
1114        insnsToKill.push_back(prev);
1115  } else {
1116    return 0;
1117  }
1118
1119  result = call->getArgOperand(0);
1120  insnsToKill.push_back(call);
1121
1122  // Keep killing bitcasts, for sanity.  Note that we no longer care
1123  // about precise ordering as long as there's exactly one use.
1124  while (llvm::BitCastInst *bitcast = dyn_cast<llvm::BitCastInst>(result)) {
1125    if (!bitcast->hasOneUse()) break;
1126    insnsToKill.push_back(bitcast);
1127    result = bitcast->getOperand(0);
1128  }
1129
1130  // Delete all the unnecessary instructions, from latest to earliest.
1131  for (llvm::SmallVectorImpl<llvm::Instruction*>::iterator
1132         i = insnsToKill.begin(), e = insnsToKill.end(); i != e; ++i)
1133    (*i)->eraseFromParent();
1134
1135  // Do the fused retain/autorelease if we were asked to.
1136  if (doRetainAutorelease)
1137    result = CGF.EmitARCRetainAutoreleaseReturnValue(result);
1138
1139  // Cast back to the result type.
1140  return CGF.Builder.CreateBitCast(result, resultType);
1141}
1142
1143/// Emit an ARC autorelease of the result of a function.
1144static llvm::Value *emitAutoreleaseOfResult(CodeGenFunction &CGF,
1145                                            llvm::Value *result) {
1146  // At -O0, try to emit a fused retain/autorelease.
1147  if (CGF.shouldUseFusedARCCalls())
1148    if (llvm::Value *fused = tryEmitFusedAutoreleaseOfResult(CGF, result))
1149      return fused;
1150
1151  return CGF.EmitARCAutoreleaseReturnValue(result);
1152}
1153
1154void CodeGenFunction::EmitFunctionEpilog(const CGFunctionInfo &FI) {
1155  // Functions with no result always return void.
1156  if (ReturnValue == 0) {
1157    Builder.CreateRetVoid();
1158    return;
1159  }
1160
1161  llvm::DebugLoc RetDbgLoc;
1162  llvm::Value *RV = 0;
1163  QualType RetTy = FI.getReturnType();
1164  const ABIArgInfo &RetAI = FI.getReturnInfo();
1165
1166  switch (RetAI.getKind()) {
1167  case ABIArgInfo::Indirect: {
1168    unsigned Alignment = getContext().getTypeAlignInChars(RetTy).getQuantity();
1169    if (RetTy->isAnyComplexType()) {
1170      ComplexPairTy RT = LoadComplexFromAddr(ReturnValue, false);
1171      StoreComplexToAddr(RT, CurFn->arg_begin(), false);
1172    } else if (CodeGenFunction::hasAggregateLLVMType(RetTy)) {
1173      // Do nothing; aggregrates get evaluated directly into the destination.
1174    } else {
1175      EmitStoreOfScalar(Builder.CreateLoad(ReturnValue), CurFn->arg_begin(),
1176                        false, Alignment, RetTy);
1177    }
1178    break;
1179  }
1180
1181  case ABIArgInfo::Extend:
1182  case ABIArgInfo::Direct:
1183    if (RetAI.getCoerceToType() == ConvertType(RetTy) &&
1184        RetAI.getDirectOffset() == 0) {
1185      // The internal return value temp always will have pointer-to-return-type
1186      // type, just do a load.
1187
1188      // If the instruction right before the insertion point is a store to the
1189      // return value, we can elide the load, zap the store, and usually zap the
1190      // alloca.
1191      llvm::BasicBlock *InsertBB = Builder.GetInsertBlock();
1192      llvm::StoreInst *SI = 0;
1193      if (InsertBB->empty() ||
1194          !(SI = dyn_cast<llvm::StoreInst>(&InsertBB->back())) ||
1195          SI->getPointerOperand() != ReturnValue || SI->isVolatile()) {
1196        RV = Builder.CreateLoad(ReturnValue);
1197      } else {
1198        // Get the stored value and nuke the now-dead store.
1199        RetDbgLoc = SI->getDebugLoc();
1200        RV = SI->getValueOperand();
1201        SI->eraseFromParent();
1202
1203        // If that was the only use of the return value, nuke it as well now.
1204        if (ReturnValue->use_empty() && isa<llvm::AllocaInst>(ReturnValue)) {
1205          cast<llvm::AllocaInst>(ReturnValue)->eraseFromParent();
1206          ReturnValue = 0;
1207        }
1208      }
1209    } else {
1210      llvm::Value *V = ReturnValue;
1211      // If the value is offset in memory, apply the offset now.
1212      if (unsigned Offs = RetAI.getDirectOffset()) {
1213        V = Builder.CreateBitCast(V, Builder.getInt8PtrTy());
1214        V = Builder.CreateConstGEP1_32(V, Offs);
1215        V = Builder.CreateBitCast(V,
1216                         llvm::PointerType::getUnqual(RetAI.getCoerceToType()));
1217      }
1218
1219      RV = CreateCoercedLoad(V, RetAI.getCoerceToType(), *this);
1220    }
1221
1222    // In ARC, end functions that return a retainable type with a call
1223    // to objc_autoreleaseReturnValue.
1224    if (AutoreleaseResult) {
1225      assert(getLangOptions().ObjCAutoRefCount &&
1226             !FI.isReturnsRetained() &&
1227             RetTy->isObjCRetainableType());
1228      RV = emitAutoreleaseOfResult(*this, RV);
1229    }
1230
1231    break;
1232
1233  case ABIArgInfo::Ignore:
1234    break;
1235
1236  case ABIArgInfo::Expand:
1237    assert(0 && "Invalid ABI kind for return argument");
1238  }
1239
1240  llvm::Instruction *Ret = RV ? Builder.CreateRet(RV) : Builder.CreateRetVoid();
1241  if (!RetDbgLoc.isUnknown())
1242    Ret->setDebugLoc(RetDbgLoc);
1243}
1244
1245void CodeGenFunction::EmitDelegateCallArg(CallArgList &args,
1246                                          const VarDecl *param) {
1247  // StartFunction converted the ABI-lowered parameter(s) into a
1248  // local alloca.  We need to turn that into an r-value suitable
1249  // for EmitCall.
1250  llvm::Value *local = GetAddrOfLocalVar(param);
1251
1252  QualType type = param->getType();
1253
1254  // For the most part, we just need to load the alloca, except:
1255  // 1) aggregate r-values are actually pointers to temporaries, and
1256  // 2) references to aggregates are pointers directly to the aggregate.
1257  // I don't know why references to non-aggregates are different here.
1258  if (const ReferenceType *ref = type->getAs<ReferenceType>()) {
1259    if (hasAggregateLLVMType(ref->getPointeeType()))
1260      return args.add(RValue::getAggregate(local), type);
1261
1262    // Locals which are references to scalars are represented
1263    // with allocas holding the pointer.
1264    return args.add(RValue::get(Builder.CreateLoad(local)), type);
1265  }
1266
1267  if (type->isAnyComplexType()) {
1268    ComplexPairTy complex = LoadComplexFromAddr(local, /*volatile*/ false);
1269    return args.add(RValue::getComplex(complex), type);
1270  }
1271
1272  if (hasAggregateLLVMType(type))
1273    return args.add(RValue::getAggregate(local), type);
1274
1275  unsigned alignment = getContext().getDeclAlign(param).getQuantity();
1276  llvm::Value *value = EmitLoadOfScalar(local, false, alignment, type);
1277  return args.add(RValue::get(value), type);
1278}
1279
1280static bool isProvablyNull(llvm::Value *addr) {
1281  return isa<llvm::ConstantPointerNull>(addr);
1282}
1283
1284static bool isProvablyNonNull(llvm::Value *addr) {
1285  return isa<llvm::AllocaInst>(addr);
1286}
1287
1288/// Emit the actual writing-back of a writeback.
1289static void emitWriteback(CodeGenFunction &CGF,
1290                          const CallArgList::Writeback &writeback) {
1291  llvm::Value *srcAddr = writeback.Address;
1292  assert(!isProvablyNull(srcAddr) &&
1293         "shouldn't have writeback for provably null argument");
1294
1295  llvm::BasicBlock *contBB = 0;
1296
1297  // If the argument wasn't provably non-null, we need to null check
1298  // before doing the store.
1299  bool provablyNonNull = isProvablyNonNull(srcAddr);
1300  if (!provablyNonNull) {
1301    llvm::BasicBlock *writebackBB = CGF.createBasicBlock("icr.writeback");
1302    contBB = CGF.createBasicBlock("icr.done");
1303
1304    llvm::Value *isNull = CGF.Builder.CreateIsNull(srcAddr, "icr.isnull");
1305    CGF.Builder.CreateCondBr(isNull, contBB, writebackBB);
1306    CGF.EmitBlock(writebackBB);
1307  }
1308
1309  // Load the value to writeback.
1310  llvm::Value *value = CGF.Builder.CreateLoad(writeback.Temporary);
1311
1312  // Cast it back, in case we're writing an id to a Foo* or something.
1313  value = CGF.Builder.CreateBitCast(value,
1314               cast<llvm::PointerType>(srcAddr->getType())->getElementType(),
1315                            "icr.writeback-cast");
1316
1317  // Perform the writeback.
1318  QualType srcAddrType = writeback.AddressType;
1319  CGF.EmitStoreThroughLValue(RValue::get(value),
1320                             CGF.MakeAddrLValue(srcAddr, srcAddrType));
1321
1322  // Jump to the continuation block.
1323  if (!provablyNonNull)
1324    CGF.EmitBlock(contBB);
1325}
1326
1327static void emitWritebacks(CodeGenFunction &CGF,
1328                           const CallArgList &args) {
1329  for (CallArgList::writeback_iterator
1330         i = args.writeback_begin(), e = args.writeback_end(); i != e; ++i)
1331    emitWriteback(CGF, *i);
1332}
1333
1334/// Emit an argument that's being passed call-by-writeback.  That is,
1335/// we are passing the address of
1336static void emitWritebackArg(CodeGenFunction &CGF, CallArgList &args,
1337                             const ObjCIndirectCopyRestoreExpr *CRE) {
1338  llvm::Value *srcAddr = CGF.EmitScalarExpr(CRE->getSubExpr());
1339
1340  // The dest and src types don't necessarily match in LLVM terms
1341  // because of the crazy ObjC compatibility rules.
1342
1343  const llvm::PointerType *destType =
1344    cast<llvm::PointerType>(CGF.ConvertType(CRE->getType()));
1345
1346  // If the address is a constant null, just pass the appropriate null.
1347  if (isProvablyNull(srcAddr)) {
1348    args.add(RValue::get(llvm::ConstantPointerNull::get(destType)),
1349             CRE->getType());
1350    return;
1351  }
1352
1353  QualType srcAddrType =
1354    CRE->getSubExpr()->getType()->castAs<PointerType>()->getPointeeType();
1355
1356  // Create the temporary.
1357  llvm::Value *temp = CGF.CreateTempAlloca(destType->getElementType(),
1358                                           "icr.temp");
1359
1360  // Zero-initialize it if we're not doing a copy-initialization.
1361  bool shouldCopy = CRE->shouldCopy();
1362  if (!shouldCopy) {
1363    llvm::Value *null =
1364      llvm::ConstantPointerNull::get(
1365        cast<llvm::PointerType>(destType->getElementType()));
1366    CGF.Builder.CreateStore(null, temp);
1367  }
1368
1369  llvm::BasicBlock *contBB = 0;
1370
1371  // If the address is *not* known to be non-null, we need to switch.
1372  llvm::Value *finalArgument;
1373
1374  bool provablyNonNull = isProvablyNonNull(srcAddr);
1375  if (provablyNonNull) {
1376    finalArgument = temp;
1377  } else {
1378    llvm::Value *isNull = CGF.Builder.CreateIsNull(srcAddr, "icr.isnull");
1379
1380    finalArgument = CGF.Builder.CreateSelect(isNull,
1381                                   llvm::ConstantPointerNull::get(destType),
1382                                             temp, "icr.argument");
1383
1384    // If we need to copy, then the load has to be conditional, which
1385    // means we need control flow.
1386    if (shouldCopy) {
1387      contBB = CGF.createBasicBlock("icr.cont");
1388      llvm::BasicBlock *copyBB = CGF.createBasicBlock("icr.copy");
1389      CGF.Builder.CreateCondBr(isNull, contBB, copyBB);
1390      CGF.EmitBlock(copyBB);
1391    }
1392  }
1393
1394  // Perform a copy if necessary.
1395  if (shouldCopy) {
1396    LValue srcLV = CGF.MakeAddrLValue(srcAddr, srcAddrType);
1397    RValue srcRV = CGF.EmitLoadOfLValue(srcLV);
1398    assert(srcRV.isScalar());
1399
1400    llvm::Value *src = srcRV.getScalarVal();
1401    src = CGF.Builder.CreateBitCast(src, destType->getElementType(),
1402                                    "icr.cast");
1403
1404    // Use an ordinary store, not a store-to-lvalue.
1405    CGF.Builder.CreateStore(src, temp);
1406  }
1407
1408  // Finish the control flow if we needed it.
1409  if (shouldCopy && !provablyNonNull)
1410    CGF.EmitBlock(contBB);
1411
1412  args.addWriteback(srcAddr, srcAddrType, temp);
1413  args.add(RValue::get(finalArgument), CRE->getType());
1414}
1415
1416void CodeGenFunction::EmitCallArg(CallArgList &args, const Expr *E,
1417                                  QualType type) {
1418  if (const ObjCIndirectCopyRestoreExpr *CRE
1419        = dyn_cast<ObjCIndirectCopyRestoreExpr>(E)) {
1420    assert(getContext().getLangOptions().ObjCAutoRefCount);
1421    assert(getContext().hasSameType(E->getType(), type));
1422    return emitWritebackArg(*this, args, CRE);
1423  }
1424
1425  if (type->isReferenceType())
1426    return args.add(EmitReferenceBindingToExpr(E, /*InitializedDecl=*/0),
1427                    type);
1428
1429  if (hasAggregateLLVMType(type) && !E->getType()->isAnyComplexType() &&
1430      isa<ImplicitCastExpr>(E) &&
1431      cast<CastExpr>(E)->getCastKind() == CK_LValueToRValue) {
1432    LValue L = EmitLValue(cast<CastExpr>(E)->getSubExpr());
1433    assert(L.isSimple());
1434    args.add(RValue::getAggregate(L.getAddress(), L.isVolatileQualified()),
1435             type, /*NeedsCopy*/true);
1436    return;
1437  }
1438
1439  args.add(EmitAnyExprToTemp(E), type);
1440}
1441
1442/// Emits a call or invoke instruction to the given function, depending
1443/// on the current state of the EH stack.
1444llvm::CallSite
1445CodeGenFunction::EmitCallOrInvoke(llvm::Value *Callee,
1446                                  llvm::Value * const *ArgBegin,
1447                                  llvm::Value * const *ArgEnd,
1448                                  const llvm::Twine &Name) {
1449  llvm::BasicBlock *InvokeDest = getInvokeDest();
1450  if (!InvokeDest)
1451    return Builder.CreateCall(Callee, ArgBegin, ArgEnd, Name);
1452
1453  llvm::BasicBlock *ContBB = createBasicBlock("invoke.cont");
1454  llvm::InvokeInst *Invoke = Builder.CreateInvoke(Callee, ContBB, InvokeDest,
1455                                                  ArgBegin, ArgEnd, Name);
1456  EmitBlock(ContBB);
1457  return Invoke;
1458}
1459
1460RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
1461                                 llvm::Value *Callee,
1462                                 ReturnValueSlot ReturnValue,
1463                                 const CallArgList &CallArgs,
1464                                 const Decl *TargetDecl,
1465                                 llvm::Instruction **callOrInvoke) {
1466  // FIXME: We no longer need the types from CallArgs; lift up and simplify.
1467  llvm::SmallVector<llvm::Value*, 16> Args;
1468
1469  // Handle struct-return functions by passing a pointer to the
1470  // location that we would like to return into.
1471  QualType RetTy = CallInfo.getReturnType();
1472  const ABIArgInfo &RetAI = CallInfo.getReturnInfo();
1473
1474
1475  // If the call returns a temporary with struct return, create a temporary
1476  // alloca to hold the result, unless one is given to us.
1477  if (CGM.ReturnTypeUsesSRet(CallInfo)) {
1478    llvm::Value *Value = ReturnValue.getValue();
1479    if (!Value)
1480      Value = CreateMemTemp(RetTy);
1481    Args.push_back(Value);
1482  }
1483
1484  assert(CallInfo.arg_size() == CallArgs.size() &&
1485         "Mismatch between function signature & arguments.");
1486  CGFunctionInfo::const_arg_iterator info_it = CallInfo.arg_begin();
1487  for (CallArgList::const_iterator I = CallArgs.begin(), E = CallArgs.end();
1488       I != E; ++I, ++info_it) {
1489    const ABIArgInfo &ArgInfo = info_it->info;
1490    RValue RV = I->RV;
1491
1492    unsigned TypeAlign =
1493      getContext().getTypeAlignInChars(I->Ty).getQuantity();
1494    switch (ArgInfo.getKind()) {
1495    case ABIArgInfo::Indirect: {
1496      if (RV.isScalar() || RV.isComplex()) {
1497        // Make a temporary alloca to pass the argument.
1498        llvm::AllocaInst *AI = CreateMemTemp(I->Ty);
1499        if (ArgInfo.getIndirectAlign() > AI->getAlignment())
1500          AI->setAlignment(ArgInfo.getIndirectAlign());
1501        Args.push_back(AI);
1502        if (RV.isScalar())
1503          EmitStoreOfScalar(RV.getScalarVal(), Args.back(), false,
1504                            TypeAlign, I->Ty);
1505        else
1506          StoreComplexToAddr(RV.getComplexVal(), Args.back(), false);
1507      } else {
1508        // We want to avoid creating an unnecessary temporary+copy here;
1509        // however, we need one in two cases:
1510        // 1. If the argument is not byval, and we are required to copy the
1511        //    source.  (This case doesn't occur on any common architecture.)
1512        // 2. If the argument is byval, RV is not sufficiently aligned, and
1513        //    we cannot force it to be sufficiently aligned.
1514        llvm::Value *Addr = RV.getAggregateAddr();
1515        unsigned Align = ArgInfo.getIndirectAlign();
1516        const llvm::TargetData *TD = &CGM.getTargetData();
1517        if ((!ArgInfo.getIndirectByVal() && I->NeedsCopy) ||
1518            (ArgInfo.getIndirectByVal() && TypeAlign < Align &&
1519             llvm::getOrEnforceKnownAlignment(Addr, Align, TD) < Align)) {
1520          // Create an aligned temporary, and copy to it.
1521          llvm::AllocaInst *AI = CreateMemTemp(I->Ty);
1522          if (Align > AI->getAlignment())
1523            AI->setAlignment(Align);
1524          Args.push_back(AI);
1525          EmitAggregateCopy(AI, Addr, I->Ty, RV.isVolatileQualified());
1526        } else {
1527          // Skip the extra memcpy call.
1528          Args.push_back(Addr);
1529        }
1530      }
1531      break;
1532    }
1533
1534    case ABIArgInfo::Ignore:
1535      break;
1536
1537    case ABIArgInfo::Extend:
1538    case ABIArgInfo::Direct: {
1539      if (!isa<llvm::StructType>(ArgInfo.getCoerceToType()) &&
1540          ArgInfo.getCoerceToType() == ConvertType(info_it->type) &&
1541          ArgInfo.getDirectOffset() == 0) {
1542        if (RV.isScalar())
1543          Args.push_back(RV.getScalarVal());
1544        else
1545          Args.push_back(Builder.CreateLoad(RV.getAggregateAddr()));
1546        break;
1547      }
1548
1549      // FIXME: Avoid the conversion through memory if possible.
1550      llvm::Value *SrcPtr;
1551      if (RV.isScalar()) {
1552        SrcPtr = CreateMemTemp(I->Ty, "coerce");
1553        EmitStoreOfScalar(RV.getScalarVal(), SrcPtr, false, TypeAlign, I->Ty);
1554      } else if (RV.isComplex()) {
1555        SrcPtr = CreateMemTemp(I->Ty, "coerce");
1556        StoreComplexToAddr(RV.getComplexVal(), SrcPtr, false);
1557      } else
1558        SrcPtr = RV.getAggregateAddr();
1559
1560      // If the value is offset in memory, apply the offset now.
1561      if (unsigned Offs = ArgInfo.getDirectOffset()) {
1562        SrcPtr = Builder.CreateBitCast(SrcPtr, Builder.getInt8PtrTy());
1563        SrcPtr = Builder.CreateConstGEP1_32(SrcPtr, Offs);
1564        SrcPtr = Builder.CreateBitCast(SrcPtr,
1565                       llvm::PointerType::getUnqual(ArgInfo.getCoerceToType()));
1566
1567      }
1568
1569      // If the coerce-to type is a first class aggregate, we flatten it and
1570      // pass the elements. Either way is semantically identical, but fast-isel
1571      // and the optimizer generally likes scalar values better than FCAs.
1572      if (const llvm::StructType *STy =
1573            dyn_cast<llvm::StructType>(ArgInfo.getCoerceToType())) {
1574        SrcPtr = Builder.CreateBitCast(SrcPtr,
1575                                       llvm::PointerType::getUnqual(STy));
1576        for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) {
1577          llvm::Value *EltPtr = Builder.CreateConstGEP2_32(SrcPtr, 0, i);
1578          llvm::LoadInst *LI = Builder.CreateLoad(EltPtr);
1579          // We don't know what we're loading from.
1580          LI->setAlignment(1);
1581          Args.push_back(LI);
1582        }
1583      } else {
1584        // In the simple case, just pass the coerced loaded value.
1585        Args.push_back(CreateCoercedLoad(SrcPtr, ArgInfo.getCoerceToType(),
1586                                         *this));
1587      }
1588
1589      break;
1590    }
1591
1592    case ABIArgInfo::Expand:
1593      ExpandTypeToArgs(I->Ty, RV, Args);
1594      break;
1595    }
1596  }
1597
1598  // If the callee is a bitcast of a function to a varargs pointer to function
1599  // type, check to see if we can remove the bitcast.  This handles some cases
1600  // with unprototyped functions.
1601  if (llvm::ConstantExpr *CE = dyn_cast<llvm::ConstantExpr>(Callee))
1602    if (llvm::Function *CalleeF = dyn_cast<llvm::Function>(CE->getOperand(0))) {
1603      const llvm::PointerType *CurPT=cast<llvm::PointerType>(Callee->getType());
1604      const llvm::FunctionType *CurFT =
1605        cast<llvm::FunctionType>(CurPT->getElementType());
1606      const llvm::FunctionType *ActualFT = CalleeF->getFunctionType();
1607
1608      if (CE->getOpcode() == llvm::Instruction::BitCast &&
1609          ActualFT->getReturnType() == CurFT->getReturnType() &&
1610          ActualFT->getNumParams() == CurFT->getNumParams() &&
1611          ActualFT->getNumParams() == Args.size() &&
1612          (CurFT->isVarArg() || !ActualFT->isVarArg())) {
1613        bool ArgsMatch = true;
1614        for (unsigned i = 0, e = ActualFT->getNumParams(); i != e; ++i)
1615          if (ActualFT->getParamType(i) != CurFT->getParamType(i)) {
1616            ArgsMatch = false;
1617            break;
1618          }
1619
1620        // Strip the cast if we can get away with it.  This is a nice cleanup,
1621        // but also allows us to inline the function at -O0 if it is marked
1622        // always_inline.
1623        if (ArgsMatch)
1624          Callee = CalleeF;
1625      }
1626    }
1627
1628
1629  unsigned CallingConv;
1630  CodeGen::AttributeListType AttributeList;
1631  CGM.ConstructAttributeList(CallInfo, TargetDecl, AttributeList, CallingConv);
1632  llvm::AttrListPtr Attrs = llvm::AttrListPtr::get(AttributeList.begin(),
1633                                                   AttributeList.end());
1634
1635  llvm::BasicBlock *InvokeDest = 0;
1636  if (!(Attrs.getFnAttributes() & llvm::Attribute::NoUnwind))
1637    InvokeDest = getInvokeDest();
1638
1639  llvm::CallSite CS;
1640  if (!InvokeDest) {
1641    CS = Builder.CreateCall(Callee, Args.data(), Args.data()+Args.size());
1642  } else {
1643    llvm::BasicBlock *Cont = createBasicBlock("invoke.cont");
1644    CS = Builder.CreateInvoke(Callee, Cont, InvokeDest,
1645                              Args.data(), Args.data()+Args.size());
1646    EmitBlock(Cont);
1647  }
1648  if (callOrInvoke)
1649    *callOrInvoke = CS.getInstruction();
1650
1651  CS.setAttributes(Attrs);
1652  CS.setCallingConv(static_cast<llvm::CallingConv::ID>(CallingConv));
1653
1654  // If the call doesn't return, finish the basic block and clear the
1655  // insertion point; this allows the rest of IRgen to discard
1656  // unreachable code.
1657  if (CS.doesNotReturn()) {
1658    Builder.CreateUnreachable();
1659    Builder.ClearInsertionPoint();
1660
1661    // FIXME: For now, emit a dummy basic block because expr emitters in
1662    // generally are not ready to handle emitting expressions at unreachable
1663    // points.
1664    EnsureInsertPoint();
1665
1666    // Return a reasonable RValue.
1667    return GetUndefRValue(RetTy);
1668  }
1669
1670  llvm::Instruction *CI = CS.getInstruction();
1671  if (Builder.isNamePreserving() && !CI->getType()->isVoidTy())
1672    CI->setName("call");
1673
1674  // Emit any writebacks immediately.  Arguably this should happen
1675  // after any return-value munging.
1676  if (CallArgs.hasWritebacks())
1677    emitWritebacks(*this, CallArgs);
1678
1679  switch (RetAI.getKind()) {
1680  case ABIArgInfo::Indirect: {
1681    unsigned Alignment = getContext().getTypeAlignInChars(RetTy).getQuantity();
1682    if (RetTy->isAnyComplexType())
1683      return RValue::getComplex(LoadComplexFromAddr(Args[0], false));
1684    if (CodeGenFunction::hasAggregateLLVMType(RetTy))
1685      return RValue::getAggregate(Args[0]);
1686    return RValue::get(EmitLoadOfScalar(Args[0], false, Alignment, RetTy));
1687  }
1688
1689  case ABIArgInfo::Ignore:
1690    // If we are ignoring an argument that had a result, make sure to
1691    // construct the appropriate return value for our caller.
1692    return GetUndefRValue(RetTy);
1693
1694  case ABIArgInfo::Extend:
1695  case ABIArgInfo::Direct: {
1696    if (RetAI.getCoerceToType() == ConvertType(RetTy) &&
1697        RetAI.getDirectOffset() == 0) {
1698      if (RetTy->isAnyComplexType()) {
1699        llvm::Value *Real = Builder.CreateExtractValue(CI, 0);
1700        llvm::Value *Imag = Builder.CreateExtractValue(CI, 1);
1701        return RValue::getComplex(std::make_pair(Real, Imag));
1702      }
1703      if (CodeGenFunction::hasAggregateLLVMType(RetTy)) {
1704        llvm::Value *DestPtr = ReturnValue.getValue();
1705        bool DestIsVolatile = ReturnValue.isVolatile();
1706
1707        if (!DestPtr) {
1708          DestPtr = CreateMemTemp(RetTy, "agg.tmp");
1709          DestIsVolatile = false;
1710        }
1711        BuildAggStore(*this, CI, DestPtr, DestIsVolatile, false);
1712        return RValue::getAggregate(DestPtr);
1713      }
1714      return RValue::get(CI);
1715    }
1716
1717    llvm::Value *DestPtr = ReturnValue.getValue();
1718    bool DestIsVolatile = ReturnValue.isVolatile();
1719
1720    if (!DestPtr) {
1721      DestPtr = CreateMemTemp(RetTy, "coerce");
1722      DestIsVolatile = false;
1723    }
1724
1725    // If the value is offset in memory, apply the offset now.
1726    llvm::Value *StorePtr = DestPtr;
1727    if (unsigned Offs = RetAI.getDirectOffset()) {
1728      StorePtr = Builder.CreateBitCast(StorePtr, Builder.getInt8PtrTy());
1729      StorePtr = Builder.CreateConstGEP1_32(StorePtr, Offs);
1730      StorePtr = Builder.CreateBitCast(StorePtr,
1731                         llvm::PointerType::getUnqual(RetAI.getCoerceToType()));
1732    }
1733    CreateCoercedStore(CI, StorePtr, DestIsVolatile, *this);
1734
1735    unsigned Alignment = getContext().getTypeAlignInChars(RetTy).getQuantity();
1736    if (RetTy->isAnyComplexType())
1737      return RValue::getComplex(LoadComplexFromAddr(DestPtr, false));
1738    if (CodeGenFunction::hasAggregateLLVMType(RetTy))
1739      return RValue::getAggregate(DestPtr);
1740    return RValue::get(EmitLoadOfScalar(DestPtr, false, Alignment, RetTy));
1741  }
1742
1743  case ABIArgInfo::Expand:
1744    assert(0 && "Invalid ABI kind for return argument");
1745  }
1746
1747  assert(0 && "Unhandled ABIArgInfo::Kind");
1748  return RValue::get(0);
1749}
1750
1751/* VarArg handling */
1752
1753llvm::Value *CodeGenFunction::EmitVAArg(llvm::Value *VAListAddr, QualType Ty) {
1754  return CGM.getTypes().getABIInfo().EmitVAArg(VAListAddr, Ty, *this);
1755}
1756