CGCall.cpp revision 71305cc81bd379ddb8aa0d49e268267383202ca9 
1//===--- CGCall.cpp - Encapsulate calling convention details ----*- C++ -*-===//
2//
3//                     The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10// These classes wrap the information about a call or function
11// definition used to handle ABI compliancy.
12//
13//===----------------------------------------------------------------------===//
14
15#include "CGCall.h"
16#include "CGCXXABI.h"
17#include "ABIInfo.h"
18#include "CodeGenFunction.h"
19#include "CodeGenModule.h"
20#include "clang/Basic/TargetInfo.h"
21#include "clang/AST/Decl.h"
22#include "clang/AST/DeclCXX.h"
23#include "clang/AST/DeclObjC.h"
24#include "clang/Frontend/CodeGenOptions.h"
25#include "llvm/Attributes.h"
26#include "llvm/Support/CallSite.h"
27#include "llvm/Target/TargetData.h"
28#include "llvm/InlineAsm.h"
29#include "llvm/Transforms/Utils/Local.h"
30using namespace clang;
31using namespace CodeGen;
32
33/***/
34
35static unsigned ClangCallConvToLLVMCallConv(CallingConv CC) {
36  switch (CC) {
37  default: return llvm::CallingConv::C;
38  case CC_X86StdCall: return llvm::CallingConv::X86_StdCall;
39  case CC_X86FastCall: return llvm::CallingConv::X86_FastCall;
40  case CC_X86ThisCall: return llvm::CallingConv::X86_ThisCall;
41  case CC_AAPCS: return llvm::CallingConv::ARM_AAPCS;
42  case CC_AAPCS_VFP: return llvm::CallingConv::ARM_AAPCS_VFP;
43  // TODO: add support for CC_X86Pascal to llvm
44  }
45}
46
47/// Derives the 'this' type for codegen purposes, i.e. ignoring method
48/// qualification.
49/// FIXME: address space qualification?
50static CanQualType GetThisType(ASTContext &Context, const CXXRecordDecl *RD) {
51  QualType RecTy = Context.getTagDeclType(RD)->getCanonicalTypeInternal();
52  return Context.getPointerType(CanQualType::CreateUnsafe(RecTy));
53}
54
55/// Returns the canonical formal type of the given C++ method.
56static CanQual<FunctionProtoType> GetFormalType(const CXXMethodDecl *MD) {
57  return MD->getType()->getCanonicalTypeUnqualified()
58           .getAs<FunctionProtoType>();
59}
60
61/// Returns the "extra-canonicalized" return type, which discards
62/// qualifiers on the return type.  Codegen doesn't care about them,
63/// and it makes ABI code a little easier to be able to assume that
64/// all parameter and return types are top-level unqualified.
65static CanQualType GetReturnType(QualType RetTy) {
66  return RetTy->getCanonicalTypeUnqualified().getUnqualifiedType();
67}
68
69const CGFunctionInfo &
70CodeGenTypes::getFunctionInfo(CanQual<FunctionNoProtoType> FTNP) {
71  return getFunctionInfo(FTNP->getResultType().getUnqualifiedType(),
72                         llvm::SmallVector<CanQualType, 16>(),
73                         FTNP->getExtInfo());
74}
75
76/// \param Args - contains any initial parameters besides those
77///   in the formal type
78static const CGFunctionInfo &getFunctionInfo(CodeGenTypes &CGT,
79                                  llvm::SmallVectorImpl<CanQualType> &ArgTys,
80                                             CanQual<FunctionProtoType> FTP) {
81  // FIXME: Kill copy.
82  for (unsigned i = 0, e = FTP->getNumArgs(); i != e; ++i)
83    ArgTys.push_back(FTP->getArgType(i));
84  CanQualType ResTy = FTP->getResultType().getUnqualifiedType();
85  return CGT.getFunctionInfo(ResTy, ArgTys, FTP->getExtInfo());
86}
87
88const CGFunctionInfo &
89CodeGenTypes::getFunctionInfo(CanQual<FunctionProtoType> FTP) {
90  llvm::SmallVector<CanQualType, 16> ArgTys;
91  return ::getFunctionInfo(*this, ArgTys, FTP);
92}
93
94static CallingConv getCallingConventionForDecl(const Decl *D) {
95  // Set the appropriate calling convention for the Function.
96  if (D->hasAttr<StdCallAttr>())
97    return CC_X86StdCall;
98
99  if (D->hasAttr<FastCallAttr>())
100    return CC_X86FastCall;
101
102  if (D->hasAttr<ThisCallAttr>())
103    return CC_X86ThisCall;
104
105  if (D->hasAttr<PascalAttr>())
106    return CC_X86Pascal;
107
108  if (PcsAttr *PCS = D->getAttr<PcsAttr>())
109    return (PCS->getPCS() == PcsAttr::AAPCS ? CC_AAPCS : CC_AAPCS_VFP);
110
111  return CC_C;
112}
113
114const CGFunctionInfo &CodeGenTypes::getFunctionInfo(const CXXRecordDecl *RD,
115                                                 const FunctionProtoType *FTP) {
116  llvm::SmallVector<CanQualType, 16> ArgTys;
117
118  // Add the 'this' pointer.
119  ArgTys.push_back(GetThisType(Context, RD));
120
121  return ::getFunctionInfo(*this, ArgTys,
122              FTP->getCanonicalTypeUnqualified().getAs<FunctionProtoType>());
123}
124
125const CGFunctionInfo &CodeGenTypes::getFunctionInfo(const CXXMethodDecl *MD) {
126  llvm::SmallVector<CanQualType, 16> ArgTys;
127
128  assert(!isa<CXXConstructorDecl>(MD) && "wrong method for contructors!");
129  assert(!isa<CXXDestructorDecl>(MD) && "wrong method for destructors!");
130
131  // Add the 'this' pointer unless this is a static method.
132  if (MD->isInstance())
133    ArgTys.push_back(GetThisType(Context, MD->getParent()));
134
135  return ::getFunctionInfo(*this, ArgTys, GetFormalType(MD));
136}
137
138const CGFunctionInfo &CodeGenTypes::getFunctionInfo(const CXXConstructorDecl *D,
139                                                    CXXCtorType Type) {
140  llvm::SmallVector<CanQualType, 16> ArgTys;
141  ArgTys.push_back(GetThisType(Context, D->getParent()));
142  CanQualType ResTy = Context.VoidTy;
143
144  TheCXXABI.BuildConstructorSignature(D, Type, ResTy, ArgTys);
145
146  CanQual<FunctionProtoType> FTP = GetFormalType(D);
147
148  // Add the formal parameters.
149  for (unsigned i = 0, e = FTP->getNumArgs(); i != e; ++i)
150    ArgTys.push_back(FTP->getArgType(i));
151
152  return getFunctionInfo(ResTy, ArgTys, FTP->getExtInfo());
153}
154
155const CGFunctionInfo &CodeGenTypes::getFunctionInfo(const CXXDestructorDecl *D,
156                                                    CXXDtorType Type) {
157  llvm::SmallVector<CanQualType, 2> ArgTys;
158  ArgTys.push_back(GetThisType(Context, D->getParent()));
159  CanQualType ResTy = Context.VoidTy;
160
161  TheCXXABI.BuildDestructorSignature(D, Type, ResTy, ArgTys);
162
163  CanQual<FunctionProtoType> FTP = GetFormalType(D);
164  assert(FTP->getNumArgs() == 0 && "dtor with formal parameters");
165
166  return getFunctionInfo(ResTy, ArgTys, FTP->getExtInfo());
167}
168
169const CGFunctionInfo &CodeGenTypes::getFunctionInfo(const FunctionDecl *FD) {
170  if (const CXXMethodDecl *MD = dyn_cast<CXXMethodDecl>(FD))
171    if (MD->isInstance())
172      return getFunctionInfo(MD);
173
174  CanQualType FTy = FD->getType()->getCanonicalTypeUnqualified();
175  assert(isa<FunctionType>(FTy));
176  if (isa<FunctionNoProtoType>(FTy))
177    return getFunctionInfo(FTy.getAs<FunctionNoProtoType>());
178  assert(isa<FunctionProtoType>(FTy));
179  return getFunctionInfo(FTy.getAs<FunctionProtoType>());
180}
181
182const CGFunctionInfo &CodeGenTypes::getFunctionInfo(const ObjCMethodDecl *MD) {
183  llvm::SmallVector<CanQualType, 16> ArgTys;
184  ArgTys.push_back(Context.getCanonicalParamType(MD->getSelfDecl()->getType()));
185  ArgTys.push_back(Context.getCanonicalParamType(Context.getObjCSelType()));
186  // FIXME: Kill copy?
187  for (ObjCMethodDecl::param_iterator i = MD->param_begin(),
188         e = MD->param_end(); i != e; ++i) {
189    ArgTys.push_back(Context.getCanonicalParamType((*i)->getType()));
190  }
191
192  FunctionType::ExtInfo einfo;
193  einfo = einfo.withCallingConv(getCallingConventionForDecl(MD));
194
195  if (getContext().getLangOptions().ObjCAutoRefCount &&
196      MD->hasAttr<NSReturnsRetainedAttr>())
197    einfo = einfo.withProducesResult(true);
198
199  return getFunctionInfo(GetReturnType(MD->getResultType()), ArgTys, einfo);
200}
201
202const CGFunctionInfo &CodeGenTypes::getFunctionInfo(GlobalDecl GD) {
203  // FIXME: Do we need to handle ObjCMethodDecl?
204  const FunctionDecl *FD = cast<FunctionDecl>(GD.getDecl());
205
206  if (const CXXConstructorDecl *CD = dyn_cast<CXXConstructorDecl>(FD))
207    return getFunctionInfo(CD, GD.getCtorType());
208
209  if (const CXXDestructorDecl *DD = dyn_cast<CXXDestructorDecl>(FD))
210    return getFunctionInfo(DD, GD.getDtorType());
211
212  return getFunctionInfo(FD);
213}
214
215const CGFunctionInfo &CodeGenTypes::getFunctionInfo(QualType ResTy,
216                                                    const CallArgList &Args,
217                                            const FunctionType::ExtInfo &Info) {
218  // FIXME: Kill copy.
219  llvm::SmallVector<CanQualType, 16> ArgTys;
220  for (CallArgList::const_iterator i = Args.begin(), e = Args.end();
221       i != e; ++i)
222    ArgTys.push_back(Context.getCanonicalParamType(i->Ty));
223  return getFunctionInfo(GetReturnType(ResTy), ArgTys, Info);
224}
225
226const CGFunctionInfo &CodeGenTypes::getFunctionInfo(QualType ResTy,
227                                                    const FunctionArgList &Args,
228                                            const FunctionType::ExtInfo &Info) {
229  // FIXME: Kill copy.
230  llvm::SmallVector<CanQualType, 16> ArgTys;
231  for (FunctionArgList::const_iterator i = Args.begin(), e = Args.end();
232       i != e; ++i)
233    ArgTys.push_back(Context.getCanonicalParamType((*i)->getType()));
234  return getFunctionInfo(GetReturnType(ResTy), ArgTys, Info);
235}
236
237const CGFunctionInfo &CodeGenTypes::getNullaryFunctionInfo() {
238  llvm::SmallVector<CanQualType, 1> args;
239  return getFunctionInfo(getContext().VoidTy, args, FunctionType::ExtInfo());
240}
241
242const CGFunctionInfo &CodeGenTypes::getFunctionInfo(CanQualType ResTy,
243                           const llvm::SmallVectorImpl<CanQualType> &ArgTys,
244                                            const FunctionType::ExtInfo &Info) {
245#ifndef NDEBUG
246  for (llvm::SmallVectorImpl<CanQualType>::const_iterator
247         I = ArgTys.begin(), E = ArgTys.end(); I != E; ++I)
248    assert(I->isCanonicalAsParam());
249#endif
250
251  unsigned CC = ClangCallConvToLLVMCallConv(Info.getCC());
252
253  // Lookup or create unique function info.
254  llvm::FoldingSetNodeID ID;
255  CGFunctionInfo::Profile(ID, Info, ResTy, ArgTys.begin(), ArgTys.end());
256
257  void *InsertPos = 0;
258  CGFunctionInfo *FI = FunctionInfos.FindNodeOrInsertPos(ID, InsertPos);
259  if (FI)
260    return *FI;
261
262  // Construct the function info.
263  FI = new CGFunctionInfo(CC, Info.getNoReturn(), Info.getProducesResult(),
264                          Info.getHasRegParm(), Info.getRegParm(), ResTy,
265                          ArgTys.data(), ArgTys.size());
266  FunctionInfos.InsertNode(FI, InsertPos);
267
268  bool Inserted = FunctionsBeingProcessed.insert(FI); (void)Inserted;
269  assert(Inserted && "Recursively being processed?");
270
271  // Compute ABI information.
272  getABIInfo().computeInfo(*FI);
273
274  bool Erased = FunctionsBeingProcessed.erase(FI); (void)Erased;
275  assert(Erased && "Not in set?");
276
277  // Loop over all of the computed argument and return value info.  If any of
278  // them are direct or extend without a specified coerce type, specify the
279  // default now.
280  ABIArgInfo &RetInfo = FI->getReturnInfo();
281  if (RetInfo.canHaveCoerceToType() && RetInfo.getCoerceToType() == 0)
282    RetInfo.setCoerceToType(ConvertType(FI->getReturnType()));
283
284  for (CGFunctionInfo::arg_iterator I = FI->arg_begin(), E = FI->arg_end();
285       I != E; ++I)
286    if (I->info.canHaveCoerceToType() && I->info.getCoerceToType() == 0)
287      I->info.setCoerceToType(ConvertType(I->type));
288
289  return *FI;
290}
291
292CGFunctionInfo::CGFunctionInfo(unsigned _CallingConvention,
293                               bool _NoReturn, bool returnsRetained,
294                               bool _HasRegParm, unsigned _RegParm,
295                               CanQualType ResTy,
296                               const CanQualType *ArgTys,
297                               unsigned NumArgTys)
298  : CallingConvention(_CallingConvention),
299    EffectiveCallingConvention(_CallingConvention),
300    NoReturn(_NoReturn), ReturnsRetained(returnsRetained),
301    HasRegParm(_HasRegParm), RegParm(_RegParm)
302{
303  NumArgs = NumArgTys;
304
305  // FIXME: Coallocate with the CGFunctionInfo object.
306  Args = new ArgInfo[1 + NumArgTys];
307  Args[0].type = ResTy;
308  for (unsigned i = 0; i != NumArgTys; ++i)
309    Args[1 + i].type = ArgTys[i];
310}
311
312/***/
313
314void CodeGenTypes::GetExpandedTypes(QualType type,
315                     llvm::SmallVectorImpl<llvm::Type*> &expandedTypes) {
316  const RecordType *RT = type->getAsStructureType();
317  assert(RT && "Can only expand structure types.");
318  const RecordDecl *RD = RT->getDecl();
319  assert(!RD->hasFlexibleArrayMember() &&
320         "Cannot expand structure with flexible array.");
321
322  for (RecordDecl::field_iterator i = RD->field_begin(), e = RD->field_end();
323         i != e; ++i) {
324    const FieldDecl *FD = *i;
325    assert(!FD->isBitField() &&
326           "Cannot expand structure with bit-field members.");
327
328    QualType fieldType = FD->getType();
329    if (fieldType->isRecordType())
330      GetExpandedTypes(fieldType, expandedTypes);
331    else
332      expandedTypes.push_back(ConvertType(fieldType));
333  }
334}
335
336llvm::Function::arg_iterator
337CodeGenFunction::ExpandTypeFromArgs(QualType Ty, LValue LV,
338                                    llvm::Function::arg_iterator AI) {
339  const RecordType *RT = Ty->getAsStructureType();
340  assert(RT && "Can only expand structure types.");
341
342  RecordDecl *RD = RT->getDecl();
343  assert(LV.isSimple() &&
344         "Unexpected non-simple lvalue during struct expansion.");
345  llvm::Value *Addr = LV.getAddress();
346  for (RecordDecl::field_iterator i = RD->field_begin(), e = RD->field_end();
347         i != e; ++i) {
348    FieldDecl *FD = *i;
349    QualType FT = FD->getType();
350
351    // FIXME: What are the right qualifiers here?
352    LValue LV = EmitLValueForField(Addr, FD, 0);
353    if (CodeGenFunction::hasAggregateLLVMType(FT)) {
354      AI = ExpandTypeFromArgs(FT, LV, AI);
355    } else {
356      EmitStoreThroughLValue(RValue::get(AI), LV);
357      ++AI;
358    }
359  }
360
361  return AI;
362}
363
364/// EnterStructPointerForCoercedAccess - Given a struct pointer that we are
365/// accessing some number of bytes out of it, try to gep into the struct to get
366/// at its inner goodness.  Dive as deep as possible without entering an element
367/// with an in-memory size smaller than DstSize.
368static llvm::Value *
369EnterStructPointerForCoercedAccess(llvm::Value *SrcPtr,
370                                   const llvm::StructType *SrcSTy,
371                                   uint64_t DstSize, CodeGenFunction &CGF) {
372  // We can't dive into a zero-element struct.
373  if (SrcSTy->getNumElements() == 0) return SrcPtr;
374
375  const llvm::Type *FirstElt = SrcSTy->getElementType(0);
376
377  // If the first elt is at least as large as what we're looking for, or if the
378  // first element is the same size as the whole struct, we can enter it.
379  uint64_t FirstEltSize =
380    CGF.CGM.getTargetData().getTypeAllocSize(FirstElt);
381  if (FirstEltSize < DstSize &&
382      FirstEltSize < CGF.CGM.getTargetData().getTypeAllocSize(SrcSTy))
383    return SrcPtr;
384
385  // GEP into the first element.
386  SrcPtr = CGF.Builder.CreateConstGEP2_32(SrcPtr, 0, 0, "coerce.dive");
387
388  // If the first element is a struct, recurse.
389  const llvm::Type *SrcTy =
390    cast<llvm::PointerType>(SrcPtr->getType())->getElementType();
391  if (const llvm::StructType *SrcSTy = dyn_cast<llvm::StructType>(SrcTy))
392    return EnterStructPointerForCoercedAccess(SrcPtr, SrcSTy, DstSize, CGF);
393
394  return SrcPtr;
395}
396
397/// CoerceIntOrPtrToIntOrPtr - Convert a value Val to the specific Ty where both
398/// are either integers or pointers.  This does a truncation of the value if it
399/// is too large or a zero extension if it is too small.
400static llvm::Value *CoerceIntOrPtrToIntOrPtr(llvm::Value *Val,
401                                             const llvm::Type *Ty,
402                                             CodeGenFunction &CGF) {
403  if (Val->getType() == Ty)
404    return Val;
405
406  if (isa<llvm::PointerType>(Val->getType())) {
407    // If this is Pointer->Pointer avoid conversion to and from int.
408    if (isa<llvm::PointerType>(Ty))
409      return CGF.Builder.CreateBitCast(Val, Ty, "coerce.val");
410
411    // Convert the pointer to an integer so we can play with its width.
412    Val = CGF.Builder.CreatePtrToInt(Val, CGF.IntPtrTy, "coerce.val.pi");
413  }
414
415  const llvm::Type *DestIntTy = Ty;
416  if (isa<llvm::PointerType>(DestIntTy))
417    DestIntTy = CGF.IntPtrTy;
418
419  if (Val->getType() != DestIntTy)
420    Val = CGF.Builder.CreateIntCast(Val, DestIntTy, false, "coerce.val.ii");
421
422  if (isa<llvm::PointerType>(Ty))
423    Val = CGF.Builder.CreateIntToPtr(Val, Ty, "coerce.val.ip");
424  return Val;
425}
426
427
428
429/// CreateCoercedLoad - Create a load from \arg SrcPtr interpreted as
430/// a pointer to an object of type \arg Ty.
431///
432/// This safely handles the case when the src type is smaller than the
433/// destination type; in this situation the values of bits which not
434/// present in the src are undefined.
435static llvm::Value *CreateCoercedLoad(llvm::Value *SrcPtr,
436                                      const llvm::Type *Ty,
437                                      CodeGenFunction &CGF) {
438  const llvm::Type *SrcTy =
439    cast<llvm::PointerType>(SrcPtr->getType())->getElementType();
440
441  // If SrcTy and Ty are the same, just do a load.
442  if (SrcTy == Ty)
443    return CGF.Builder.CreateLoad(SrcPtr);
444
445  uint64_t DstSize = CGF.CGM.getTargetData().getTypeAllocSize(Ty);
446
447  if (const llvm::StructType *SrcSTy = dyn_cast<llvm::StructType>(SrcTy)) {
448    SrcPtr = EnterStructPointerForCoercedAccess(SrcPtr, SrcSTy, DstSize, CGF);
449    SrcTy = cast<llvm::PointerType>(SrcPtr->getType())->getElementType();
450  }
451
452  uint64_t SrcSize = CGF.CGM.getTargetData().getTypeAllocSize(SrcTy);
453
454  // If the source and destination are integer or pointer types, just do an
455  // extension or truncation to the desired type.
456  if ((isa<llvm::IntegerType>(Ty) || isa<llvm::PointerType>(Ty)) &&
457      (isa<llvm::IntegerType>(SrcTy) || isa<llvm::PointerType>(SrcTy))) {
458    llvm::LoadInst *Load = CGF.Builder.CreateLoad(SrcPtr);
459    return CoerceIntOrPtrToIntOrPtr(Load, Ty, CGF);
460  }
461
462  // If load is legal, just bitcast the src pointer.
463  if (SrcSize >= DstSize) {
464    // Generally SrcSize is never greater than DstSize, since this means we are
465    // losing bits. However, this can happen in cases where the structure has
466    // additional padding, for example due to a user specified alignment.
467    //
468    // FIXME: Assert that we aren't truncating non-padding bits when have access
469    // to that information.
470    llvm::Value *Casted =
471      CGF.Builder.CreateBitCast(SrcPtr, llvm::PointerType::getUnqual(Ty));
472    llvm::LoadInst *Load = CGF.Builder.CreateLoad(Casted);
473    // FIXME: Use better alignment / avoid requiring aligned load.
474    Load->setAlignment(1);
475    return Load;
476  }
477
478  // Otherwise do coercion through memory. This is stupid, but
479  // simple.
480  llvm::Value *Tmp = CGF.CreateTempAlloca(Ty);
481  llvm::Value *Casted =
482    CGF.Builder.CreateBitCast(Tmp, llvm::PointerType::getUnqual(SrcTy));
483  llvm::StoreInst *Store =
484    CGF.Builder.CreateStore(CGF.Builder.CreateLoad(SrcPtr), Casted);
485  // FIXME: Use better alignment / avoid requiring aligned store.
486  Store->setAlignment(1);
487  return CGF.Builder.CreateLoad(Tmp);
488}
489
490// Function to store a first-class aggregate into memory.  We prefer to
491// store the elements rather than the aggregate to be more friendly to
492// fast-isel.
493// FIXME: Do we need to recurse here?
494static void BuildAggStore(CodeGenFunction &CGF, llvm::Value *Val,
495                          llvm::Value *DestPtr, bool DestIsVolatile,
496                          bool LowAlignment) {
497  // Prefer scalar stores to first-class aggregate stores.
498  if (const llvm::StructType *STy =
499        dyn_cast<llvm::StructType>(Val->getType())) {
500    for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) {
501      llvm::Value *EltPtr = CGF.Builder.CreateConstGEP2_32(DestPtr, 0, i);
502      llvm::Value *Elt = CGF.Builder.CreateExtractValue(Val, i);
503      llvm::StoreInst *SI = CGF.Builder.CreateStore(Elt, EltPtr,
504                                                    DestIsVolatile);
505      if (LowAlignment)
506        SI->setAlignment(1);
507    }
508  } else {
509    CGF.Builder.CreateStore(Val, DestPtr, DestIsVolatile);
510  }
511}
512
513/// CreateCoercedStore - Create a store to \arg DstPtr from \arg Src,
514/// where the source and destination may have different types.
515///
516/// This safely handles the case when the src type is larger than the
517/// destination type; the upper bits of the src will be lost.
518static void CreateCoercedStore(llvm::Value *Src,
519                               llvm::Value *DstPtr,
520                               bool DstIsVolatile,
521                               CodeGenFunction &CGF) {
522  const llvm::Type *SrcTy = Src->getType();
523  const llvm::Type *DstTy =
524    cast<llvm::PointerType>(DstPtr->getType())->getElementType();
525  if (SrcTy == DstTy) {
526    CGF.Builder.CreateStore(Src, DstPtr, DstIsVolatile);
527    return;
528  }
529
530  uint64_t SrcSize = CGF.CGM.getTargetData().getTypeAllocSize(SrcTy);
531
532  if (const llvm::StructType *DstSTy = dyn_cast<llvm::StructType>(DstTy)) {
533    DstPtr = EnterStructPointerForCoercedAccess(DstPtr, DstSTy, SrcSize, CGF);
534    DstTy = cast<llvm::PointerType>(DstPtr->getType())->getElementType();
535  }
536
537  // If the source and destination are integer or pointer types, just do an
538  // extension or truncation to the desired type.
539  if ((isa<llvm::IntegerType>(SrcTy) || isa<llvm::PointerType>(SrcTy)) &&
540      (isa<llvm::IntegerType>(DstTy) || isa<llvm::PointerType>(DstTy))) {
541    Src = CoerceIntOrPtrToIntOrPtr(Src, DstTy, CGF);
542    CGF.Builder.CreateStore(Src, DstPtr, DstIsVolatile);
543    return;
544  }
545
546  uint64_t DstSize = CGF.CGM.getTargetData().getTypeAllocSize(DstTy);
547
548  // If store is legal, just bitcast the src pointer.
549  if (SrcSize <= DstSize) {
550    llvm::Value *Casted =
551      CGF.Builder.CreateBitCast(DstPtr, llvm::PointerType::getUnqual(SrcTy));
552    // FIXME: Use better alignment / avoid requiring aligned store.
553    BuildAggStore(CGF, Src, Casted, DstIsVolatile, true);
554  } else {
555    // Otherwise do coercion through memory. This is stupid, but
556    // simple.
557
558    // Generally SrcSize is never greater than DstSize, since this means we are
559    // losing bits. However, this can happen in cases where the structure has
560    // additional padding, for example due to a user specified alignment.
561    //
562    // FIXME: Assert that we aren't truncating non-padding bits when have access
563    // to that information.
564    llvm::Value *Tmp = CGF.CreateTempAlloca(SrcTy);
565    CGF.Builder.CreateStore(Src, Tmp);
566    llvm::Value *Casted =
567      CGF.Builder.CreateBitCast(Tmp, llvm::PointerType::getUnqual(DstTy));
568    llvm::LoadInst *Load = CGF.Builder.CreateLoad(Casted);
569    // FIXME: Use better alignment / avoid requiring aligned load.
570    Load->setAlignment(1);
571    CGF.Builder.CreateStore(Load, DstPtr, DstIsVolatile);
572  }
573}
574
575/***/
576
577bool CodeGenModule::ReturnTypeUsesSRet(const CGFunctionInfo &FI) {
578  return FI.getReturnInfo().isIndirect();
579}
580
581bool CodeGenModule::ReturnTypeUsesFPRet(QualType ResultType) {
582  if (const BuiltinType *BT = ResultType->getAs<BuiltinType>()) {
583    switch (BT->getKind()) {
584    default:
585      return false;
586    case BuiltinType::Float:
587      return getContext().Target.useObjCFPRetForRealType(TargetInfo::Float);
588    case BuiltinType::Double:
589      return getContext().Target.useObjCFPRetForRealType(TargetInfo::Double);
590    case BuiltinType::LongDouble:
591      return getContext().Target.useObjCFPRetForRealType(
592        TargetInfo::LongDouble);
593    }
594  }
595
596  return false;
597}
598
599llvm::FunctionType *CodeGenTypes::GetFunctionType(GlobalDecl GD) {
600  const CGFunctionInfo &FI = getFunctionInfo(GD);
601
602  // For definition purposes, don't consider a K&R function variadic.
603  bool Variadic = false;
604  if (const FunctionProtoType *FPT =
605        cast<FunctionDecl>(GD.getDecl())->getType()->getAs<FunctionProtoType>())
606    Variadic = FPT->isVariadic();
607
608  return GetFunctionType(FI, Variadic);
609}
610
611llvm::FunctionType *
612CodeGenTypes::GetFunctionType(const CGFunctionInfo &FI, bool isVariadic) {
613
614  bool Inserted = FunctionsBeingProcessed.insert(&FI); (void)Inserted;
615  assert(Inserted && "Recursively being processed?");
616
617  llvm::SmallVector<llvm::Type*, 8> argTypes;
618  const llvm::Type *resultType = 0;
619
620  const ABIArgInfo &retAI = FI.getReturnInfo();
621  switch (retAI.getKind()) {
622  case ABIArgInfo::Expand:
623    llvm_unreachable("Invalid ABI kind for return argument");
624
625  case ABIArgInfo::Extend:
626  case ABIArgInfo::Direct:
627    resultType = retAI.getCoerceToType();
628    break;
629
630  case ABIArgInfo::Indirect: {
631    assert(!retAI.getIndirectAlign() && "Align unused on indirect return.");
632    resultType = llvm::Type::getVoidTy(getLLVMContext());
633
634    QualType ret = FI.getReturnType();
635    const llvm::Type *ty = ConvertType(ret);
636    unsigned addressSpace = Context.getTargetAddressSpace(ret);
637    argTypes.push_back(llvm::PointerType::get(ty, addressSpace));
638    break;
639  }
640
641  case ABIArgInfo::Ignore:
642    resultType = llvm::Type::getVoidTy(getLLVMContext());
643    break;
644  }
645
646  for (CGFunctionInfo::const_arg_iterator it = FI.arg_begin(),
647         ie = FI.arg_end(); it != ie; ++it) {
648    const ABIArgInfo &argAI = it->info;
649
650    switch (argAI.getKind()) {
651    case ABIArgInfo::Ignore:
652      break;
653
654    case ABIArgInfo::Indirect: {
655      // indirect arguments are always on the stack, which is addr space #0.
656      const llvm::Type *LTy = ConvertTypeForMem(it->type);
657      argTypes.push_back(LTy->getPointerTo());
658      break;
659    }
660
661    case ABIArgInfo::Extend:
662    case ABIArgInfo::Direct: {
663      // If the coerce-to type is a first class aggregate, flatten it.  Either
664      // way is semantically identical, but fast-isel and the optimizer
665      // generally likes scalar values better than FCAs.
666      llvm::Type *argType = argAI.getCoerceToType();
667      if (const llvm::StructType *st = dyn_cast<llvm::StructType>(argType)) {
668        for (unsigned i = 0, e = st->getNumElements(); i != e; ++i)
669          argTypes.push_back(st->getElementType(i));
670      } else {
671        argTypes.push_back(argType);
672      }
673      break;
674    }
675
676    case ABIArgInfo::Expand:
677      GetExpandedTypes(it->type, argTypes);
678      break;
679    }
680  }
681
682  bool Erased = FunctionsBeingProcessed.erase(&FI); (void)Erased;
683  assert(Erased && "Not in set?");
684
685  return llvm::FunctionType::get(resultType, argTypes, isVariadic);
686}
687
688const llvm::Type *CodeGenTypes::GetFunctionTypeForVTable(GlobalDecl GD) {
689  const CXXMethodDecl *MD = cast<CXXMethodDecl>(GD.getDecl());
690  const FunctionProtoType *FPT = MD->getType()->getAs<FunctionProtoType>();
691
692  if (!isFuncTypeConvertible(FPT))
693    return llvm::StructType::get(getLLVMContext());
694
695  const CGFunctionInfo *Info;
696  if (isa<CXXDestructorDecl>(MD))
697    Info = &getFunctionInfo(cast<CXXDestructorDecl>(MD), GD.getDtorType());
698  else
699    Info = &getFunctionInfo(MD);
700  return GetFunctionType(*Info, FPT->isVariadic());
701}
702
703void CodeGenModule::ConstructAttributeList(const CGFunctionInfo &FI,
704                                           const Decl *TargetDecl,
705                                           AttributeListType &PAL,
706                                           unsigned &CallingConv) {
707  unsigned FuncAttrs = 0;
708  unsigned RetAttrs = 0;
709
710  CallingConv = FI.getEffectiveCallingConvention();
711
712  if (FI.isNoReturn())
713    FuncAttrs |= llvm::Attribute::NoReturn;
714
715  // FIXME: handle sseregparm someday...
716  if (TargetDecl) {
717    if (TargetDecl->hasAttr<NoThrowAttr>())
718      FuncAttrs |= llvm::Attribute::NoUnwind;
719    else if (const FunctionDecl *Fn = dyn_cast<FunctionDecl>(TargetDecl)) {
720      const FunctionProtoType *FPT = Fn->getType()->getAs<FunctionProtoType>();
721      if (FPT && FPT->isNothrow(getContext()))
722        FuncAttrs |= llvm::Attribute::NoUnwind;
723    }
724
725    if (TargetDecl->hasAttr<NoReturnAttr>())
726      FuncAttrs |= llvm::Attribute::NoReturn;
727    if (TargetDecl->hasAttr<ConstAttr>())
728      FuncAttrs |= llvm::Attribute::ReadNone;
729    else if (TargetDecl->hasAttr<PureAttr>())
730      FuncAttrs |= llvm::Attribute::ReadOnly;
731    if (TargetDecl->hasAttr<MallocAttr>())
732      RetAttrs |= llvm::Attribute::NoAlias;
733  }
734
735  if (CodeGenOpts.OptimizeSize)
736    FuncAttrs |= llvm::Attribute::OptimizeForSize;
737  if (CodeGenOpts.DisableRedZone)
738    FuncAttrs |= llvm::Attribute::NoRedZone;
739  if (CodeGenOpts.NoImplicitFloat)
740    FuncAttrs |= llvm::Attribute::NoImplicitFloat;
741
742  QualType RetTy = FI.getReturnType();
743  unsigned Index = 1;
744  const ABIArgInfo &RetAI = FI.getReturnInfo();
745  switch (RetAI.getKind()) {
746  case ABIArgInfo::Extend:
747   if (RetTy->hasSignedIntegerRepresentation())
748     RetAttrs |= llvm::Attribute::SExt;
749   else if (RetTy->hasUnsignedIntegerRepresentation())
750     RetAttrs |= llvm::Attribute::ZExt;
751    break;
752  case ABIArgInfo::Direct:
753  case ABIArgInfo::Ignore:
754    break;
755
756  case ABIArgInfo::Indirect:
757    PAL.push_back(llvm::AttributeWithIndex::get(Index,
758                                                llvm::Attribute::StructRet));
759    ++Index;
760    // sret disables readnone and readonly
761    FuncAttrs &= ~(llvm::Attribute::ReadOnly |
762                   llvm::Attribute::ReadNone);
763    break;
764
765  case ABIArgInfo::Expand:
766    assert(0 && "Invalid ABI kind for return argument");
767  }
768
769  if (RetAttrs)
770    PAL.push_back(llvm::AttributeWithIndex::get(0, RetAttrs));
771
772  // FIXME: RegParm should be reduced in case of global register variable.
773  signed RegParm;
774  if (FI.getHasRegParm())
775    RegParm = FI.getRegParm();
776  else
777    RegParm = CodeGenOpts.NumRegisterParameters;
778
779  unsigned PointerWidth = getContext().Target.getPointerWidth(0);
780  for (CGFunctionInfo::const_arg_iterator it = FI.arg_begin(),
781         ie = FI.arg_end(); it != ie; ++it) {
782    QualType ParamType = it->type;
783    const ABIArgInfo &AI = it->info;
784    unsigned Attributes = 0;
785
786    // 'restrict' -> 'noalias' is done in EmitFunctionProlog when we
787    // have the corresponding parameter variable.  It doesn't make
788    // sense to do it here because parameters are so messed up.
789    switch (AI.getKind()) {
790    case ABIArgInfo::Extend:
791      if (ParamType->isSignedIntegerOrEnumerationType())
792        Attributes |= llvm::Attribute::SExt;
793      else if (ParamType->isUnsignedIntegerOrEnumerationType())
794        Attributes |= llvm::Attribute::ZExt;
795      // FALL THROUGH
796    case ABIArgInfo::Direct:
797      if (RegParm > 0 &&
798          (ParamType->isIntegerType() || ParamType->isPointerType())) {
799        RegParm -=
800        (Context.getTypeSize(ParamType) + PointerWidth - 1) / PointerWidth;
801        if (RegParm >= 0)
802          Attributes |= llvm::Attribute::InReg;
803      }
804      // FIXME: handle sseregparm someday...
805
806      if (const llvm::StructType *STy =
807            dyn_cast<llvm::StructType>(AI.getCoerceToType()))
808        Index += STy->getNumElements()-1;  // 1 will be added below.
809      break;
810
811    case ABIArgInfo::Indirect:
812      if (AI.getIndirectByVal())
813        Attributes |= llvm::Attribute::ByVal;
814
815      Attributes |=
816        llvm::Attribute::constructAlignmentFromInt(AI.getIndirectAlign());
817      // byval disables readnone and readonly.
818      FuncAttrs &= ~(llvm::Attribute::ReadOnly |
819                     llvm::Attribute::ReadNone);
820      break;
821
822    case ABIArgInfo::Ignore:
823      // Skip increment, no matching LLVM parameter.
824      continue;
825
826    case ABIArgInfo::Expand: {
827      llvm::SmallVector<llvm::Type*, 8> types;
828      // FIXME: This is rather inefficient. Do we ever actually need to do
829      // anything here? The result should be just reconstructed on the other
830      // side, so extension should be a non-issue.
831      getTypes().GetExpandedTypes(ParamType, types);
832      Index += types.size();
833      continue;
834    }
835    }
836
837    if (Attributes)
838      PAL.push_back(llvm::AttributeWithIndex::get(Index, Attributes));
839    ++Index;
840  }
841  if (FuncAttrs)
842    PAL.push_back(llvm::AttributeWithIndex::get(~0, FuncAttrs));
843}
844
845/// An argument came in as a promoted argument; demote it back to its
846/// declared type.
847static llvm::Value *emitArgumentDemotion(CodeGenFunction &CGF,
848                                         const VarDecl *var,
849                                         llvm::Value *value) {
850  const llvm::Type *varType = CGF.ConvertType(var->getType());
851
852  // This can happen with promotions that actually don't change the
853  // underlying type, like the enum promotions.
854  if (value->getType() == varType) return value;
855
856  assert((varType->isIntegerTy() || varType->isFloatingPointTy())
857         && "unexpected promotion type");
858
859  if (isa<llvm::IntegerType>(varType))
860    return CGF.Builder.CreateTrunc(value, varType, "arg.unpromote");
861
862  return CGF.Builder.CreateFPCast(value, varType, "arg.unpromote");
863}
864
865void CodeGenFunction::EmitFunctionProlog(const CGFunctionInfo &FI,
866                                         llvm::Function *Fn,
867                                         const FunctionArgList &Args) {
868  // If this is an implicit-return-zero function, go ahead and
869  // initialize the return value.  TODO: it might be nice to have
870  // a more general mechanism for this that didn't require synthesized
871  // return statements.
872  if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(CurFuncDecl)) {
873    if (FD->hasImplicitReturnZero()) {
874      QualType RetTy = FD->getResultType().getUnqualifiedType();
875      const llvm::Type* LLVMTy = CGM.getTypes().ConvertType(RetTy);
876      llvm::Constant* Zero = llvm::Constant::getNullValue(LLVMTy);
877      Builder.CreateStore(Zero, ReturnValue);
878    }
879  }
880
881  // FIXME: We no longer need the types from FunctionArgList; lift up and
882  // simplify.
883
884  // Emit allocs for param decls.  Give the LLVM Argument nodes names.
885  llvm::Function::arg_iterator AI = Fn->arg_begin();
886
887  // Name the struct return argument.
888  if (CGM.ReturnTypeUsesSRet(FI)) {
889    AI->setName("agg.result");
890    ++AI;
891  }
892
893  assert(FI.arg_size() == Args.size() &&
894         "Mismatch between function signature & arguments.");
895  unsigned ArgNo = 1;
896  CGFunctionInfo::const_arg_iterator info_it = FI.arg_begin();
897  for (FunctionArgList::const_iterator i = Args.begin(), e = Args.end();
898       i != e; ++i, ++info_it, ++ArgNo) {
899    const VarDecl *Arg = *i;
900    QualType Ty = info_it->type;
901    const ABIArgInfo &ArgI = info_it->info;
902
903    bool isPromoted =
904      isa<ParmVarDecl>(Arg) && cast<ParmVarDecl>(Arg)->isKNRPromoted();
905
906    switch (ArgI.getKind()) {
907    case ABIArgInfo::Indirect: {
908      llvm::Value *V = AI;
909
910      if (hasAggregateLLVMType(Ty)) {
911        // Aggregates and complex variables are accessed by reference.  All we
912        // need to do is realign the value, if requested
913        if (ArgI.getIndirectRealign()) {
914          llvm::Value *AlignedTemp = CreateMemTemp(Ty, "coerce");
915
916          // Copy from the incoming argument pointer to the temporary with the
917          // appropriate alignment.
918          //
919          // FIXME: We should have a common utility for generating an aggregate
920          // copy.
921          const llvm::Type *I8PtrTy = Builder.getInt8PtrTy();
922          CharUnits Size = getContext().getTypeSizeInChars(Ty);
923          llvm::Value *Dst = Builder.CreateBitCast(AlignedTemp, I8PtrTy);
924          llvm::Value *Src = Builder.CreateBitCast(V, I8PtrTy);
925          Builder.CreateMemCpy(Dst,
926                               Src,
927                               llvm::ConstantInt::get(IntPtrTy,
928                                                      Size.getQuantity()),
929                               ArgI.getIndirectAlign(),
930                               false);
931          V = AlignedTemp;
932        }
933      } else {
934        // Load scalar value from indirect argument.
935        CharUnits Alignment = getContext().getTypeAlignInChars(Ty);
936        V = EmitLoadOfScalar(V, false, Alignment.getQuantity(), Ty);
937
938        if (isPromoted)
939          V = emitArgumentDemotion(*this, Arg, V);
940      }
941      EmitParmDecl(*Arg, V, ArgNo);
942      break;
943    }
944
945    case ABIArgInfo::Extend:
946    case ABIArgInfo::Direct: {
947      // If we have the trivial case, handle it with no muss and fuss.
948      if (!isa<llvm::StructType>(ArgI.getCoerceToType()) &&
949          ArgI.getCoerceToType() == ConvertType(Ty) &&
950          ArgI.getDirectOffset() == 0) {
951        assert(AI != Fn->arg_end() && "Argument mismatch!");
952        llvm::Value *V = AI;
953
954        if (Arg->getType().isRestrictQualified())
955          AI->addAttr(llvm::Attribute::NoAlias);
956
957        if (isPromoted)
958          V = emitArgumentDemotion(*this, Arg, V);
959
960        EmitParmDecl(*Arg, V, ArgNo);
961        break;
962      }
963
964      llvm::AllocaInst *Alloca = CreateMemTemp(Ty, "coerce");
965
966      // The alignment we need to use is the max of the requested alignment for
967      // the argument plus the alignment required by our access code below.
968      unsigned AlignmentToUse =
969        CGM.getTargetData().getABITypeAlignment(ArgI.getCoerceToType());
970      AlignmentToUse = std::max(AlignmentToUse,
971                        (unsigned)getContext().getDeclAlign(Arg).getQuantity());
972
973      Alloca->setAlignment(AlignmentToUse);
974      llvm::Value *V = Alloca;
975      llvm::Value *Ptr = V;    // Pointer to store into.
976
977      // If the value is offset in memory, apply the offset now.
978      if (unsigned Offs = ArgI.getDirectOffset()) {
979        Ptr = Builder.CreateBitCast(Ptr, Builder.getInt8PtrTy());
980        Ptr = Builder.CreateConstGEP1_32(Ptr, Offs);
981        Ptr = Builder.CreateBitCast(Ptr,
982                          llvm::PointerType::getUnqual(ArgI.getCoerceToType()));
983      }
984
985      // If the coerce-to type is a first class aggregate, we flatten it and
986      // pass the elements. Either way is semantically identical, but fast-isel
987      // and the optimizer generally likes scalar values better than FCAs.
988      if (const llvm::StructType *STy =
989            dyn_cast<llvm::StructType>(ArgI.getCoerceToType())) {
990        Ptr = Builder.CreateBitCast(Ptr, llvm::PointerType::getUnqual(STy));
991
992        for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) {
993          assert(AI != Fn->arg_end() && "Argument mismatch!");
994          AI->setName(Arg->getName() + ".coerce" + llvm::Twine(i));
995          llvm::Value *EltPtr = Builder.CreateConstGEP2_32(Ptr, 0, i);
996          Builder.CreateStore(AI++, EltPtr);
997        }
998      } else {
999        // Simple case, just do a coerced store of the argument into the alloca.
1000        assert(AI != Fn->arg_end() && "Argument mismatch!");
1001        AI->setName(Arg->getName() + ".coerce");
1002        CreateCoercedStore(AI++, Ptr, /*DestIsVolatile=*/false, *this);
1003      }
1004
1005
1006      // Match to what EmitParmDecl is expecting for this type.
1007      if (!CodeGenFunction::hasAggregateLLVMType(Ty)) {
1008        V = EmitLoadOfScalar(V, false, AlignmentToUse, Ty);
1009        if (isPromoted)
1010          V = emitArgumentDemotion(*this, Arg, V);
1011      }
1012      EmitParmDecl(*Arg, V, ArgNo);
1013      continue;  // Skip ++AI increment, already done.
1014    }
1015
1016    case ABIArgInfo::Expand: {
1017      // If this structure was expanded into multiple arguments then
1018      // we need to create a temporary and reconstruct it from the
1019      // arguments.
1020      llvm::Value *Temp = CreateMemTemp(Ty, Arg->getName() + ".addr");
1021      llvm::Function::arg_iterator End =
1022        ExpandTypeFromArgs(Ty, MakeAddrLValue(Temp, Ty), AI);
1023      EmitParmDecl(*Arg, Temp, ArgNo);
1024
1025      // Name the arguments used in expansion and increment AI.
1026      unsigned Index = 0;
1027      for (; AI != End; ++AI, ++Index)
1028        AI->setName(Arg->getName() + "." + llvm::Twine(Index));
1029      continue;
1030    }
1031
1032    case ABIArgInfo::Ignore:
1033      // Initialize the local variable appropriately.
1034      if (hasAggregateLLVMType(Ty))
1035        EmitParmDecl(*Arg, CreateMemTemp(Ty), ArgNo);
1036      else
1037        EmitParmDecl(*Arg, llvm::UndefValue::get(ConvertType(Arg->getType())),
1038                     ArgNo);
1039
1040      // Skip increment, no matching LLVM parameter.
1041      continue;
1042    }
1043
1044    ++AI;
1045  }
1046  assert(AI == Fn->arg_end() && "Argument mismatch!");
1047}
1048
1049/// Try to emit a fused autorelease of a return result.
1050static llvm::Value *tryEmitFusedAutoreleaseOfResult(CodeGenFunction &CGF,
1051                                                    llvm::Value *result) {
1052  // We must be immediately followed the cast.
1053  llvm::BasicBlock *BB = CGF.Builder.GetInsertBlock();
1054  if (BB->empty()) return 0;
1055  if (&BB->back() != result) return 0;
1056
1057  const llvm::Type *resultType = result->getType();
1058
1059  // result is in a BasicBlock and is therefore an Instruction.
1060  llvm::Instruction *generator = cast<llvm::Instruction>(result);
1061
1062  llvm::SmallVector<llvm::Instruction*,4> insnsToKill;
1063
1064  // Look for:
1065  //  %generator = bitcast %type1* %generator2 to %type2*
1066  while (llvm::BitCastInst *bitcast = dyn_cast<llvm::BitCastInst>(generator)) {
1067    // We would have emitted this as a constant if the operand weren't
1068    // an Instruction.
1069    generator = cast<llvm::Instruction>(bitcast->getOperand(0));
1070
1071    // Require the generator to be immediately followed by the cast.
1072    if (generator->getNextNode() != bitcast)
1073      return 0;
1074
1075    insnsToKill.push_back(bitcast);
1076  }
1077
1078  // Look for:
1079  //   %generator = call i8* @objc_retain(i8* %originalResult)
1080  // or
1081  //   %generator = call i8* @objc_retainAutoreleasedReturnValue(i8* %originalResult)
1082  llvm::CallInst *call = dyn_cast<llvm::CallInst>(generator);
1083  if (!call) return 0;
1084
1085  bool doRetainAutorelease;
1086
1087  if (call->getCalledValue() == CGF.CGM.getARCEntrypoints().objc_retain) {
1088    doRetainAutorelease = true;
1089  } else if (call->getCalledValue() == CGF.CGM.getARCEntrypoints()
1090                                          .objc_retainAutoreleasedReturnValue) {
1091    doRetainAutorelease = false;
1092
1093    // Look for an inline asm immediately preceding the call and kill it, too.
1094    llvm::Instruction *prev = call->getPrevNode();
1095    if (llvm::CallInst *asmCall = dyn_cast_or_null<llvm::CallInst>(prev))
1096      if (asmCall->getCalledValue()
1097            == CGF.CGM.getARCEntrypoints().retainAutoreleasedReturnValueMarker)
1098        insnsToKill.push_back(prev);
1099  } else {
1100    return 0;
1101  }
1102
1103  result = call->getArgOperand(0);
1104  insnsToKill.push_back(call);
1105
1106  // Keep killing bitcasts, for sanity.  Note that we no longer care
1107  // about precise ordering as long as there's exactly one use.
1108  while (llvm::BitCastInst *bitcast = dyn_cast<llvm::BitCastInst>(result)) {
1109    if (!bitcast->hasOneUse()) break;
1110    insnsToKill.push_back(bitcast);
1111    result = bitcast->getOperand(0);
1112  }
1113
1114  // Delete all the unnecessary instructions, from latest to earliest.
1115  for (llvm::SmallVectorImpl<llvm::Instruction*>::iterator
1116         i = insnsToKill.begin(), e = insnsToKill.end(); i != e; ++i)
1117    (*i)->eraseFromParent();
1118
1119  // Do the fused retain/autorelease if we were asked to.
1120  if (doRetainAutorelease)
1121    result = CGF.EmitARCRetainAutoreleaseReturnValue(result);
1122
1123  // Cast back to the result type.
1124  return CGF.Builder.CreateBitCast(result, resultType);
1125}
1126
1127/// Emit an ARC autorelease of the result of a function.
1128static llvm::Value *emitAutoreleaseOfResult(CodeGenFunction &CGF,
1129                                            llvm::Value *result) {
1130  // At -O0, try to emit a fused retain/autorelease.
1131  if (CGF.shouldUseFusedARCCalls())
1132    if (llvm::Value *fused = tryEmitFusedAutoreleaseOfResult(CGF, result))
1133      return fused;
1134
1135  return CGF.EmitARCAutoreleaseReturnValue(result);
1136}
1137
1138void CodeGenFunction::EmitFunctionEpilog(const CGFunctionInfo &FI) {
1139  // Functions with no result always return void.
1140  if (ReturnValue == 0) {
1141    Builder.CreateRetVoid();
1142    return;
1143  }
1144
1145  llvm::DebugLoc RetDbgLoc;
1146  llvm::Value *RV = 0;
1147  QualType RetTy = FI.getReturnType();
1148  const ABIArgInfo &RetAI = FI.getReturnInfo();
1149
1150  switch (RetAI.getKind()) {
1151  case ABIArgInfo::Indirect: {
1152    unsigned Alignment = getContext().getTypeAlignInChars(RetTy).getQuantity();
1153    if (RetTy->isAnyComplexType()) {
1154      ComplexPairTy RT = LoadComplexFromAddr(ReturnValue, false);
1155      StoreComplexToAddr(RT, CurFn->arg_begin(), false);
1156    } else if (CodeGenFunction::hasAggregateLLVMType(RetTy)) {
1157      // Do nothing; aggregrates get evaluated directly into the destination.
1158    } else {
1159      EmitStoreOfScalar(Builder.CreateLoad(ReturnValue), CurFn->arg_begin(),
1160                        false, Alignment, RetTy);
1161    }
1162    break;
1163  }
1164
1165  case ABIArgInfo::Extend:
1166  case ABIArgInfo::Direct:
1167    if (RetAI.getCoerceToType() == ConvertType(RetTy) &&
1168        RetAI.getDirectOffset() == 0) {
1169      // The internal return value temp always will have pointer-to-return-type
1170      // type, just do a load.
1171
1172      // If the instruction right before the insertion point is a store to the
1173      // return value, we can elide the load, zap the store, and usually zap the
1174      // alloca.
1175      llvm::BasicBlock *InsertBB = Builder.GetInsertBlock();
1176      llvm::StoreInst *SI = 0;
1177      if (InsertBB->empty() ||
1178          !(SI = dyn_cast<llvm::StoreInst>(&InsertBB->back())) ||
1179          SI->getPointerOperand() != ReturnValue || SI->isVolatile()) {
1180        RV = Builder.CreateLoad(ReturnValue);
1181      } else {
1182        // Get the stored value and nuke the now-dead store.
1183        RetDbgLoc = SI->getDebugLoc();
1184        RV = SI->getValueOperand();
1185        SI->eraseFromParent();
1186
1187        // If that was the only use of the return value, nuke it as well now.
1188        if (ReturnValue->use_empty() && isa<llvm::AllocaInst>(ReturnValue)) {
1189          cast<llvm::AllocaInst>(ReturnValue)->eraseFromParent();
1190          ReturnValue = 0;
1191        }
1192      }
1193    } else {
1194      llvm::Value *V = ReturnValue;
1195      // If the value is offset in memory, apply the offset now.
1196      if (unsigned Offs = RetAI.getDirectOffset()) {
1197        V = Builder.CreateBitCast(V, Builder.getInt8PtrTy());
1198        V = Builder.CreateConstGEP1_32(V, Offs);
1199        V = Builder.CreateBitCast(V,
1200                         llvm::PointerType::getUnqual(RetAI.getCoerceToType()));
1201      }
1202
1203      RV = CreateCoercedLoad(V, RetAI.getCoerceToType(), *this);
1204    }
1205
1206    // In ARC, end functions that return a retainable type with a call
1207    // to objc_autoreleaseReturnValue.
1208    if (AutoreleaseResult) {
1209      assert(getLangOptions().ObjCAutoRefCount &&
1210             !FI.isReturnsRetained() &&
1211             RetTy->isObjCRetainableType());
1212      RV = emitAutoreleaseOfResult(*this, RV);
1213    }
1214
1215    break;
1216
1217  case ABIArgInfo::Ignore:
1218    break;
1219
1220  case ABIArgInfo::Expand:
1221    assert(0 && "Invalid ABI kind for return argument");
1222  }
1223
1224  llvm::Instruction *Ret = RV ? Builder.CreateRet(RV) : Builder.CreateRetVoid();
1225  if (!RetDbgLoc.isUnknown())
1226    Ret->setDebugLoc(RetDbgLoc);
1227}
1228
1229void CodeGenFunction::EmitDelegateCallArg(CallArgList &args,
1230                                          const VarDecl *param) {
1231  // StartFunction converted the ABI-lowered parameter(s) into a
1232  // local alloca.  We need to turn that into an r-value suitable
1233  // for EmitCall.
1234  llvm::Value *local = GetAddrOfLocalVar(param);
1235
1236  QualType type = param->getType();
1237
1238  // For the most part, we just need to load the alloca, except:
1239  // 1) aggregate r-values are actually pointers to temporaries, and
1240  // 2) references to aggregates are pointers directly to the aggregate.
1241  // I don't know why references to non-aggregates are different here.
1242  if (const ReferenceType *ref = type->getAs<ReferenceType>()) {
1243    if (hasAggregateLLVMType(ref->getPointeeType()))
1244      return args.add(RValue::getAggregate(local), type);
1245
1246    // Locals which are references to scalars are represented
1247    // with allocas holding the pointer.
1248    return args.add(RValue::get(Builder.CreateLoad(local)), type);
1249  }
1250
1251  if (type->isAnyComplexType()) {
1252    ComplexPairTy complex = LoadComplexFromAddr(local, /*volatile*/ false);
1253    return args.add(RValue::getComplex(complex), type);
1254  }
1255
1256  if (hasAggregateLLVMType(type))
1257    return args.add(RValue::getAggregate(local), type);
1258
1259  unsigned alignment = getContext().getDeclAlign(param).getQuantity();
1260  llvm::Value *value = EmitLoadOfScalar(local, false, alignment, type);
1261  return args.add(RValue::get(value), type);
1262}
1263
1264static bool isProvablyNull(llvm::Value *addr) {
1265  return isa<llvm::ConstantPointerNull>(addr);
1266}
1267
1268static bool isProvablyNonNull(llvm::Value *addr) {
1269  return isa<llvm::AllocaInst>(addr);
1270}
1271
1272/// Emit the actual writing-back of a writeback.
1273static void emitWriteback(CodeGenFunction &CGF,
1274                          const CallArgList::Writeback &writeback) {
1275  llvm::Value *srcAddr = writeback.Address;
1276  assert(!isProvablyNull(srcAddr) &&
1277         "shouldn't have writeback for provably null argument");
1278
1279  llvm::BasicBlock *contBB = 0;
1280
1281  // If the argument wasn't provably non-null, we need to null check
1282  // before doing the store.
1283  bool provablyNonNull = isProvablyNonNull(srcAddr);
1284  if (!provablyNonNull) {
1285    llvm::BasicBlock *writebackBB = CGF.createBasicBlock("icr.writeback");
1286    contBB = CGF.createBasicBlock("icr.done");
1287
1288    llvm::Value *isNull = CGF.Builder.CreateIsNull(srcAddr, "icr.isnull");
1289    CGF.Builder.CreateCondBr(isNull, contBB, writebackBB);
1290    CGF.EmitBlock(writebackBB);
1291  }
1292
1293  // Load the value to writeback.
1294  llvm::Value *value = CGF.Builder.CreateLoad(writeback.Temporary);
1295
1296  // Cast it back, in case we're writing an id to a Foo* or something.
1297  value = CGF.Builder.CreateBitCast(value,
1298               cast<llvm::PointerType>(srcAddr->getType())->getElementType(),
1299                            "icr.writeback-cast");
1300
1301  // Perform the writeback.
1302  QualType srcAddrType = writeback.AddressType;
1303  CGF.EmitStoreThroughLValue(RValue::get(value),
1304                             CGF.MakeAddrLValue(srcAddr, srcAddrType));
1305
1306  // Jump to the continuation block.
1307  if (!provablyNonNull)
1308    CGF.EmitBlock(contBB);
1309}
1310
1311static void emitWritebacks(CodeGenFunction &CGF,
1312                           const CallArgList &args) {
1313  for (CallArgList::writeback_iterator
1314         i = args.writeback_begin(), e = args.writeback_end(); i != e; ++i)
1315    emitWriteback(CGF, *i);
1316}
1317
1318/// Emit an argument that's being passed call-by-writeback.  That is,
1319/// we are passing the address of
1320static void emitWritebackArg(CodeGenFunction &CGF, CallArgList &args,
1321                             const ObjCIndirectCopyRestoreExpr *CRE) {
1322  llvm::Value *srcAddr = CGF.EmitScalarExpr(CRE->getSubExpr());
1323
1324  // The dest and src types don't necessarily match in LLVM terms
1325  // because of the crazy ObjC compatibility rules.
1326
1327  const llvm::PointerType *destType =
1328    cast<llvm::PointerType>(CGF.ConvertType(CRE->getType()));
1329
1330  // If the address is a constant null, just pass the appropriate null.
1331  if (isProvablyNull(srcAddr)) {
1332    args.add(RValue::get(llvm::ConstantPointerNull::get(destType)),
1333             CRE->getType());
1334    return;
1335  }
1336
1337  QualType srcAddrType =
1338    CRE->getSubExpr()->getType()->castAs<PointerType>()->getPointeeType();
1339
1340  // Create the temporary.
1341  llvm::Value *temp = CGF.CreateTempAlloca(destType->getElementType(),
1342                                           "icr.temp");
1343
1344  // Zero-initialize it if we're not doing a copy-initialization.
1345  bool shouldCopy = CRE->shouldCopy();
1346  if (!shouldCopy) {
1347    llvm::Value *null =
1348      llvm::ConstantPointerNull::get(
1349        cast<llvm::PointerType>(destType->getElementType()));
1350    CGF.Builder.CreateStore(null, temp);
1351  }
1352
1353  llvm::BasicBlock *contBB = 0;
1354
1355  // If the address is *not* known to be non-null, we need to switch.
1356  llvm::Value *finalArgument;
1357
1358  bool provablyNonNull = isProvablyNonNull(srcAddr);
1359  if (provablyNonNull) {
1360    finalArgument = temp;
1361  } else {
1362    llvm::Value *isNull = CGF.Builder.CreateIsNull(srcAddr, "icr.isnull");
1363
1364    finalArgument = CGF.Builder.CreateSelect(isNull,
1365                                   llvm::ConstantPointerNull::get(destType),
1366                                             temp, "icr.argument");
1367
1368    // If we need to copy, then the load has to be conditional, which
1369    // means we need control flow.
1370    if (shouldCopy) {
1371      contBB = CGF.createBasicBlock("icr.cont");
1372      llvm::BasicBlock *copyBB = CGF.createBasicBlock("icr.copy");
1373      CGF.Builder.CreateCondBr(isNull, contBB, copyBB);
1374      CGF.EmitBlock(copyBB);
1375    }
1376  }
1377
1378  // Perform a copy if necessary.
1379  if (shouldCopy) {
1380    LValue srcLV = CGF.MakeAddrLValue(srcAddr, srcAddrType);
1381    RValue srcRV = CGF.EmitLoadOfLValue(srcLV);
1382    assert(srcRV.isScalar());
1383
1384    llvm::Value *src = srcRV.getScalarVal();
1385    src = CGF.Builder.CreateBitCast(src, destType->getElementType(),
1386                                    "icr.cast");
1387
1388    // Use an ordinary store, not a store-to-lvalue.
1389    CGF.Builder.CreateStore(src, temp);
1390  }
1391
1392  // Finish the control flow if we needed it.
1393  if (shouldCopy && !provablyNonNull)
1394    CGF.EmitBlock(contBB);
1395
1396  args.addWriteback(srcAddr, srcAddrType, temp);
1397  args.add(RValue::get(finalArgument), CRE->getType());
1398}
1399
1400void CodeGenFunction::EmitCallArg(CallArgList &args, const Expr *E,
1401                                  QualType type) {
1402  if (const ObjCIndirectCopyRestoreExpr *CRE
1403        = dyn_cast<ObjCIndirectCopyRestoreExpr>(E)) {
1404    assert(getContext().getLangOptions().ObjCAutoRefCount);
1405    assert(getContext().hasSameType(E->getType(), type));
1406    return emitWritebackArg(*this, args, CRE);
1407  }
1408
1409  if (type->isReferenceType())
1410    return args.add(EmitReferenceBindingToExpr(E, /*InitializedDecl=*/0),
1411                    type);
1412
1413  if (hasAggregateLLVMType(type) && !E->getType()->isAnyComplexType() &&
1414      isa<ImplicitCastExpr>(E) &&
1415      cast<CastExpr>(E)->getCastKind() == CK_LValueToRValue) {
1416    LValue L = EmitLValue(cast<CastExpr>(E)->getSubExpr());
1417    assert(L.isSimple());
1418    args.add(RValue::getAggregate(L.getAddress(), L.isVolatileQualified()),
1419             type, /*NeedsCopy*/true);
1420    return;
1421  }
1422
1423  args.add(EmitAnyExprToTemp(E), type);
1424}
1425
1426/// Emits a call or invoke instruction to the given function, depending
1427/// on the current state of the EH stack.
1428llvm::CallSite
1429CodeGenFunction::EmitCallOrInvoke(llvm::Value *Callee,
1430                                  llvm::Value * const *ArgBegin,
1431                                  llvm::Value * const *ArgEnd,
1432                                  const llvm::Twine &Name) {
1433  llvm::BasicBlock *InvokeDest = getInvokeDest();
1434  if (!InvokeDest)
1435    return Builder.CreateCall(Callee, ArgBegin, ArgEnd, Name);
1436
1437  llvm::BasicBlock *ContBB = createBasicBlock("invoke.cont");
1438  llvm::InvokeInst *Invoke = Builder.CreateInvoke(Callee, ContBB, InvokeDest,
1439                                                  ArgBegin, ArgEnd, Name);
1440  EmitBlock(ContBB);
1441  return Invoke;
1442}
1443
1444static void checkArgMatches(llvm::Value *Elt, unsigned &ArgNo,
1445                            llvm::FunctionType *FTy) {
1446  if (ArgNo < FTy->getNumParams())
1447    assert(Elt->getType() == FTy->getParamType(ArgNo));
1448  else
1449    assert(FTy->isVarArg());
1450  ++ArgNo;
1451}
1452
1453void CodeGenFunction::ExpandTypeToArgs(QualType Ty, RValue RV,
1454                                       llvm::SmallVector<llvm::Value*,16> &Args,
1455                                       llvm::FunctionType *IRFuncTy) {
1456  const RecordType *RT = Ty->getAsStructureType();
1457  assert(RT && "Can only expand structure types.");
1458
1459  RecordDecl *RD = RT->getDecl();
1460  assert(RV.isAggregate() && "Unexpected rvalue during struct expansion");
1461  llvm::Value *Addr = RV.getAggregateAddr();
1462  for (RecordDecl::field_iterator i = RD->field_begin(), e = RD->field_end();
1463       i != e; ++i) {
1464    FieldDecl *FD = *i;
1465    QualType FT = FD->getType();
1466
1467    // FIXME: What are the right qualifiers here?
1468    LValue LV = EmitLValueForField(Addr, FD, 0);
1469    if (CodeGenFunction::hasAggregateLLVMType(FT)) {
1470      ExpandTypeToArgs(FT, RValue::getAggregate(LV.getAddress()),
1471                       Args, IRFuncTy);
1472      continue;
1473    }
1474
1475    RValue RV = EmitLoadOfLValue(LV);
1476    assert(RV.isScalar() &&
1477           "Unexpected non-scalar rvalue during struct expansion.");
1478
1479    // Insert a bitcast as needed.
1480    llvm::Value *V = RV.getScalarVal();
1481    if (Args.size() < IRFuncTy->getNumParams() &&
1482        V->getType() != IRFuncTy->getParamType(Args.size()))
1483      V = Builder.CreateBitCast(V, IRFuncTy->getParamType(Args.size()));
1484
1485    Args.push_back(V);
1486  }
1487}
1488
1489
1490RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
1491                                 llvm::Value *Callee,
1492                                 ReturnValueSlot ReturnValue,
1493                                 const CallArgList &CallArgs,
1494                                 const Decl *TargetDecl,
1495                                 llvm::Instruction **callOrInvoke) {
1496  // FIXME: We no longer need the types from CallArgs; lift up and simplify.
1497  llvm::SmallVector<llvm::Value*, 16> Args;
1498
1499  // Handle struct-return functions by passing a pointer to the
1500  // location that we would like to return into.
1501  QualType RetTy = CallInfo.getReturnType();
1502  const ABIArgInfo &RetAI = CallInfo.getReturnInfo();
1503
1504  // IRArgNo - Keep track of the argument number in the callee we're looking at.
1505  unsigned IRArgNo = 0;
1506  llvm::FunctionType *IRFuncTy =
1507    cast<llvm::FunctionType>(
1508                  cast<llvm::PointerType>(Callee->getType())->getElementType());
1509
1510  // If the call returns a temporary with struct return, create a temporary
1511  // alloca to hold the result, unless one is given to us.
1512  if (CGM.ReturnTypeUsesSRet(CallInfo)) {
1513    llvm::Value *Value = ReturnValue.getValue();
1514    if (!Value)
1515      Value = CreateMemTemp(RetTy);
1516    Args.push_back(Value);
1517    checkArgMatches(Value, IRArgNo, IRFuncTy);
1518  }
1519
1520  assert(CallInfo.arg_size() == CallArgs.size() &&
1521         "Mismatch between function signature & arguments.");
1522  CGFunctionInfo::const_arg_iterator info_it = CallInfo.arg_begin();
1523  for (CallArgList::const_iterator I = CallArgs.begin(), E = CallArgs.end();
1524       I != E; ++I, ++info_it) {
1525    const ABIArgInfo &ArgInfo = info_it->info;
1526    RValue RV = I->RV;
1527
1528    unsigned TypeAlign =
1529      getContext().getTypeAlignInChars(I->Ty).getQuantity();
1530    switch (ArgInfo.getKind()) {
1531    case ABIArgInfo::Indirect: {
1532      if (RV.isScalar() || RV.isComplex()) {
1533        // Make a temporary alloca to pass the argument.
1534        llvm::AllocaInst *AI = CreateMemTemp(I->Ty);
1535        if (ArgInfo.getIndirectAlign() > AI->getAlignment())
1536          AI->setAlignment(ArgInfo.getIndirectAlign());
1537        Args.push_back(AI);
1538
1539        if (RV.isScalar())
1540          EmitStoreOfScalar(RV.getScalarVal(), Args.back(), false,
1541                            TypeAlign, I->Ty);
1542        else
1543          StoreComplexToAddr(RV.getComplexVal(), Args.back(), false);
1544
1545        // Validate argument match.
1546        checkArgMatches(AI, IRArgNo, IRFuncTy);
1547      } else {
1548        // We want to avoid creating an unnecessary temporary+copy here;
1549        // however, we need one in two cases:
1550        // 1. If the argument is not byval, and we are required to copy the
1551        //    source.  (This case doesn't occur on any common architecture.)
1552        // 2. If the argument is byval, RV is not sufficiently aligned, and
1553        //    we cannot force it to be sufficiently aligned.
1554        llvm::Value *Addr = RV.getAggregateAddr();
1555        unsigned Align = ArgInfo.getIndirectAlign();
1556        const llvm::TargetData *TD = &CGM.getTargetData();
1557        if ((!ArgInfo.getIndirectByVal() && I->NeedsCopy) ||
1558            (ArgInfo.getIndirectByVal() && TypeAlign < Align &&
1559             llvm::getOrEnforceKnownAlignment(Addr, Align, TD) < Align)) {
1560          // Create an aligned temporary, and copy to it.
1561          llvm::AllocaInst *AI = CreateMemTemp(I->Ty);
1562          if (Align > AI->getAlignment())
1563            AI->setAlignment(Align);
1564          Args.push_back(AI);
1565          EmitAggregateCopy(AI, Addr, I->Ty, RV.isVolatileQualified());
1566
1567          // Validate argument match.
1568          checkArgMatches(AI, IRArgNo, IRFuncTy);
1569        } else {
1570          // Skip the extra memcpy call.
1571          Args.push_back(Addr);
1572
1573          // Validate argument match.
1574          checkArgMatches(Addr, IRArgNo, IRFuncTy);
1575        }
1576      }
1577      break;
1578    }
1579
1580    case ABIArgInfo::Ignore:
1581      break;
1582
1583    case ABIArgInfo::Extend:
1584    case ABIArgInfo::Direct: {
1585      if (!isa<llvm::StructType>(ArgInfo.getCoerceToType()) &&
1586          ArgInfo.getCoerceToType() == ConvertType(info_it->type) &&
1587          ArgInfo.getDirectOffset() == 0) {
1588        llvm::Value *V;
1589        if (RV.isScalar())
1590          V = RV.getScalarVal();
1591        else
1592          V = Builder.CreateLoad(RV.getAggregateAddr());
1593
1594        // If the argument doesn't match, perform a bitcast to coerce it.  This
1595        // can happen due to trivial type mismatches.
1596        if (IRArgNo < IRFuncTy->getNumParams() &&
1597            V->getType() != IRFuncTy->getParamType(IRArgNo))
1598          V = Builder.CreateBitCast(V, IRFuncTy->getParamType(IRArgNo));
1599        Args.push_back(V);
1600
1601        checkArgMatches(V, IRArgNo, IRFuncTy);
1602        break;
1603      }
1604
1605      // FIXME: Avoid the conversion through memory if possible.
1606      llvm::Value *SrcPtr;
1607      if (RV.isScalar()) {
1608        SrcPtr = CreateMemTemp(I->Ty, "coerce");
1609        EmitStoreOfScalar(RV.getScalarVal(), SrcPtr, false, TypeAlign, I->Ty);
1610      } else if (RV.isComplex()) {
1611        SrcPtr = CreateMemTemp(I->Ty, "coerce");
1612        StoreComplexToAddr(RV.getComplexVal(), SrcPtr, false);
1613      } else
1614        SrcPtr = RV.getAggregateAddr();
1615
1616      // If the value is offset in memory, apply the offset now.
1617      if (unsigned Offs = ArgInfo.getDirectOffset()) {
1618        SrcPtr = Builder.CreateBitCast(SrcPtr, Builder.getInt8PtrTy());
1619        SrcPtr = Builder.CreateConstGEP1_32(SrcPtr, Offs);
1620        SrcPtr = Builder.CreateBitCast(SrcPtr,
1621                       llvm::PointerType::getUnqual(ArgInfo.getCoerceToType()));
1622
1623      }
1624
1625      // If the coerce-to type is a first class aggregate, we flatten it and
1626      // pass the elements. Either way is semantically identical, but fast-isel
1627      // and the optimizer generally likes scalar values better than FCAs.
1628      if (const llvm::StructType *STy =
1629            dyn_cast<llvm::StructType>(ArgInfo.getCoerceToType())) {
1630        SrcPtr = Builder.CreateBitCast(SrcPtr,
1631                                       llvm::PointerType::getUnqual(STy));
1632        for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) {
1633          llvm::Value *EltPtr = Builder.CreateConstGEP2_32(SrcPtr, 0, i);
1634          llvm::LoadInst *LI = Builder.CreateLoad(EltPtr);
1635          // We don't know what we're loading from.
1636          LI->setAlignment(1);
1637          Args.push_back(LI);
1638
1639          // Validate argument match.
1640          checkArgMatches(LI, IRArgNo, IRFuncTy);
1641        }
1642      } else {
1643        // In the simple case, just pass the coerced loaded value.
1644        Args.push_back(CreateCoercedLoad(SrcPtr, ArgInfo.getCoerceToType(),
1645                                         *this));
1646
1647        // Validate argument match.
1648        checkArgMatches(Args.back(), IRArgNo, IRFuncTy);
1649      }
1650
1651      break;
1652    }
1653
1654    case ABIArgInfo::Expand:
1655      ExpandTypeToArgs(I->Ty, RV, Args, IRFuncTy);
1656      IRArgNo = Args.size();
1657      break;
1658    }
1659  }
1660
1661  // If the callee is a bitcast of a function to a varargs pointer to function
1662  // type, check to see if we can remove the bitcast.  This handles some cases
1663  // with unprototyped functions.
1664  if (llvm::ConstantExpr *CE = dyn_cast<llvm::ConstantExpr>(Callee))
1665    if (llvm::Function *CalleeF = dyn_cast<llvm::Function>(CE->getOperand(0))) {
1666      const llvm::PointerType *CurPT=cast<llvm::PointerType>(Callee->getType());
1667      const llvm::FunctionType *CurFT =
1668        cast<llvm::FunctionType>(CurPT->getElementType());
1669      const llvm::FunctionType *ActualFT = CalleeF->getFunctionType();
1670
1671      if (CE->getOpcode() == llvm::Instruction::BitCast &&
1672          ActualFT->getReturnType() == CurFT->getReturnType() &&
1673          ActualFT->getNumParams() == CurFT->getNumParams() &&
1674          ActualFT->getNumParams() == Args.size() &&
1675          (CurFT->isVarArg() || !ActualFT->isVarArg())) {
1676        bool ArgsMatch = true;
1677        for (unsigned i = 0, e = ActualFT->getNumParams(); i != e; ++i)
1678          if (ActualFT->getParamType(i) != CurFT->getParamType(i)) {
1679            ArgsMatch = false;
1680            break;
1681          }
1682
1683        // Strip the cast if we can get away with it.  This is a nice cleanup,
1684        // but also allows us to inline the function at -O0 if it is marked
1685        // always_inline.
1686        if (ArgsMatch)
1687          Callee = CalleeF;
1688      }
1689    }
1690
1691  unsigned CallingConv;
1692  CodeGen::AttributeListType AttributeList;
1693  CGM.ConstructAttributeList(CallInfo, TargetDecl, AttributeList, CallingConv);
1694  llvm::AttrListPtr Attrs = llvm::AttrListPtr::get(AttributeList.begin(),
1695                                                   AttributeList.end());
1696
1697  llvm::BasicBlock *InvokeDest = 0;
1698  if (!(Attrs.getFnAttributes() & llvm::Attribute::NoUnwind))
1699    InvokeDest = getInvokeDest();
1700
1701  llvm::CallSite CS;
1702  if (!InvokeDest) {
1703    CS = Builder.CreateCall(Callee, Args.data(), Args.data()+Args.size());
1704  } else {
1705    llvm::BasicBlock *Cont = createBasicBlock("invoke.cont");
1706    CS = Builder.CreateInvoke(Callee, Cont, InvokeDest,
1707                              Args.data(), Args.data()+Args.size());
1708    EmitBlock(Cont);
1709  }
1710  if (callOrInvoke)
1711    *callOrInvoke = CS.getInstruction();
1712
1713  CS.setAttributes(Attrs);
1714  CS.setCallingConv(static_cast<llvm::CallingConv::ID>(CallingConv));
1715
1716  // If the call doesn't return, finish the basic block and clear the
1717  // insertion point; this allows the rest of IRgen to discard
1718  // unreachable code.
1719  if (CS.doesNotReturn()) {
1720    Builder.CreateUnreachable();
1721    Builder.ClearInsertionPoint();
1722
1723    // FIXME: For now, emit a dummy basic block because expr emitters in
1724    // generally are not ready to handle emitting expressions at unreachable
1725    // points.
1726    EnsureInsertPoint();
1727
1728    // Return a reasonable RValue.
1729    return GetUndefRValue(RetTy);
1730  }
1731
1732  llvm::Instruction *CI = CS.getInstruction();
1733  if (Builder.isNamePreserving() && !CI->getType()->isVoidTy())
1734    CI->setName("call");
1735
1736  // Emit any writebacks immediately.  Arguably this should happen
1737  // after any return-value munging.
1738  if (CallArgs.hasWritebacks())
1739    emitWritebacks(*this, CallArgs);
1740
1741  switch (RetAI.getKind()) {
1742  case ABIArgInfo::Indirect: {
1743    unsigned Alignment = getContext().getTypeAlignInChars(RetTy).getQuantity();
1744    if (RetTy->isAnyComplexType())
1745      return RValue::getComplex(LoadComplexFromAddr(Args[0], false));
1746    if (CodeGenFunction::hasAggregateLLVMType(RetTy))
1747      return RValue::getAggregate(Args[0]);
1748    return RValue::get(EmitLoadOfScalar(Args[0], false, Alignment, RetTy));
1749  }
1750
1751  case ABIArgInfo::Ignore:
1752    // If we are ignoring an argument that had a result, make sure to
1753    // construct the appropriate return value for our caller.
1754    return GetUndefRValue(RetTy);
1755
1756  case ABIArgInfo::Extend:
1757  case ABIArgInfo::Direct: {
1758    llvm::Type *RetIRTy = ConvertType(RetTy);
1759    if (RetAI.getCoerceToType() == RetIRTy && RetAI.getDirectOffset() == 0) {
1760      if (RetTy->isAnyComplexType()) {
1761        llvm::Value *Real = Builder.CreateExtractValue(CI, 0);
1762        llvm::Value *Imag = Builder.CreateExtractValue(CI, 1);
1763        return RValue::getComplex(std::make_pair(Real, Imag));
1764      }
1765      if (CodeGenFunction::hasAggregateLLVMType(RetTy)) {
1766        llvm::Value *DestPtr = ReturnValue.getValue();
1767        bool DestIsVolatile = ReturnValue.isVolatile();
1768
1769        if (!DestPtr) {
1770          DestPtr = CreateMemTemp(RetTy, "agg.tmp");
1771          DestIsVolatile = false;
1772        }
1773        BuildAggStore(*this, CI, DestPtr, DestIsVolatile, false);
1774        return RValue::getAggregate(DestPtr);
1775      }
1776
1777      // If the argument doesn't match, perform a bitcast to coerce it.  This
1778      // can happen due to trivial type mismatches.
1779      llvm::Value *V = CI;
1780      if (V->getType() != RetIRTy)
1781        V = Builder.CreateBitCast(V, RetIRTy);
1782      return RValue::get(V);
1783    }
1784
1785    llvm::Value *DestPtr = ReturnValue.getValue();
1786    bool DestIsVolatile = ReturnValue.isVolatile();
1787
1788    if (!DestPtr) {
1789      DestPtr = CreateMemTemp(RetTy, "coerce");
1790      DestIsVolatile = false;
1791    }
1792
1793    // If the value is offset in memory, apply the offset now.
1794    llvm::Value *StorePtr = DestPtr;
1795    if (unsigned Offs = RetAI.getDirectOffset()) {
1796      StorePtr = Builder.CreateBitCast(StorePtr, Builder.getInt8PtrTy());
1797      StorePtr = Builder.CreateConstGEP1_32(StorePtr, Offs);
1798      StorePtr = Builder.CreateBitCast(StorePtr,
1799                         llvm::PointerType::getUnqual(RetAI.getCoerceToType()));
1800    }
1801    CreateCoercedStore(CI, StorePtr, DestIsVolatile, *this);
1802
1803    unsigned Alignment = getContext().getTypeAlignInChars(RetTy).getQuantity();
1804    if (RetTy->isAnyComplexType())
1805      return RValue::getComplex(LoadComplexFromAddr(DestPtr, false));
1806    if (CodeGenFunction::hasAggregateLLVMType(RetTy))
1807      return RValue::getAggregate(DestPtr);
1808    return RValue::get(EmitLoadOfScalar(DestPtr, false, Alignment, RetTy));
1809  }
1810
1811  case ABIArgInfo::Expand:
1812    assert(0 && "Invalid ABI kind for return argument");
1813  }
1814
1815  assert(0 && "Unhandled ABIArgInfo::Kind");
1816  return RValue::get(0);
1817}
1818
1819/* VarArg handling */
1820
1821llvm::Value *CodeGenFunction::EmitVAArg(llvm::Value *VAListAddr, QualType Ty) {
1822  return CGM.getTypes().getABIInfo().EmitVAArg(VAListAddr, Ty, *this);
1823}
1824