CGCall.cpp revision f0cc2087b18c48b17c2f647c88a3e7eef19285fd 
1//===--- CGCall.cpp - Encapsulate calling convention details ----*- C++ -*-===//
2//
3//                     The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10// These classes wrap the information about a call or function
11// definition used to handle ABI compliancy.
12//
13//===----------------------------------------------------------------------===//
14
15#include "CGCall.h"
16#include "CGCXXABI.h"
17#include "ABIInfo.h"
18#include "CodeGenFunction.h"
19#include "CodeGenModule.h"
20#include "clang/Basic/TargetInfo.h"
21#include "clang/AST/Decl.h"
22#include "clang/AST/DeclCXX.h"
23#include "clang/AST/DeclObjC.h"
24#include "clang/Frontend/CodeGenOptions.h"
25#include "llvm/Attributes.h"
26#include "llvm/Support/CallSite.h"
27#include "llvm/Target/TargetData.h"
28#include "llvm/InlineAsm.h"
29#include "llvm/Transforms/Utils/Local.h"
30using namespace clang;
31using namespace CodeGen;
32
33/***/
34
35static unsigned ClangCallConvToLLVMCallConv(CallingConv CC) {
36  switch (CC) {
37  default: return llvm::CallingConv::C;
38  case CC_X86StdCall: return llvm::CallingConv::X86_StdCall;
39  case CC_X86FastCall: return llvm::CallingConv::X86_FastCall;
40  case CC_X86ThisCall: return llvm::CallingConv::X86_ThisCall;
41  case CC_AAPCS: return llvm::CallingConv::ARM_AAPCS;
42  case CC_AAPCS_VFP: return llvm::CallingConv::ARM_AAPCS_VFP;
43  // TODO: add support for CC_X86Pascal to llvm
44  }
45}
46
47/// Derives the 'this' type for codegen purposes, i.e. ignoring method
48/// qualification.
49/// FIXME: address space qualification?
50static CanQualType GetThisType(ASTContext &Context, const CXXRecordDecl *RD) {
51  QualType RecTy = Context.getTagDeclType(RD)->getCanonicalTypeInternal();
52  return Context.getPointerType(CanQualType::CreateUnsafe(RecTy));
53}
54
55/// Returns the canonical formal type of the given C++ method.
56static CanQual<FunctionProtoType> GetFormalType(const CXXMethodDecl *MD) {
57  return MD->getType()->getCanonicalTypeUnqualified()
58           .getAs<FunctionProtoType>();
59}
60
61/// Returns the "extra-canonicalized" return type, which discards
62/// qualifiers on the return type.  Codegen doesn't care about them,
63/// and it makes ABI code a little easier to be able to assume that
64/// all parameter and return types are top-level unqualified.
65static CanQualType GetReturnType(QualType RetTy) {
66  return RetTy->getCanonicalTypeUnqualified().getUnqualifiedType();
67}
68
69const CGFunctionInfo &
70CodeGenTypes::getFunctionInfo(CanQual<FunctionNoProtoType> FTNP) {
71  return getFunctionInfo(FTNP->getResultType().getUnqualifiedType(),
72                         SmallVector<CanQualType, 16>(),
73                         FTNP->getExtInfo());
74}
75
76/// \param Args - contains any initial parameters besides those
77///   in the formal type
78static const CGFunctionInfo &getFunctionInfo(CodeGenTypes &CGT,
79                                  SmallVectorImpl<CanQualType> &ArgTys,
80                                             CanQual<FunctionProtoType> FTP) {
81  // FIXME: Kill copy.
82  for (unsigned i = 0, e = FTP->getNumArgs(); i != e; ++i)
83    ArgTys.push_back(FTP->getArgType(i));
84  CanQualType ResTy = FTP->getResultType().getUnqualifiedType();
85  return CGT.getFunctionInfo(ResTy, ArgTys, FTP->getExtInfo());
86}
87
88const CGFunctionInfo &
89CodeGenTypes::getFunctionInfo(CanQual<FunctionProtoType> FTP) {
90  SmallVector<CanQualType, 16> ArgTys;
91  return ::getFunctionInfo(*this, ArgTys, FTP);
92}
93
94static CallingConv getCallingConventionForDecl(const Decl *D) {
95  // Set the appropriate calling convention for the Function.
96  if (D->hasAttr<StdCallAttr>())
97    return CC_X86StdCall;
98
99  if (D->hasAttr<FastCallAttr>())
100    return CC_X86FastCall;
101
102  if (D->hasAttr<ThisCallAttr>())
103    return CC_X86ThisCall;
104
105  if (D->hasAttr<PascalAttr>())
106    return CC_X86Pascal;
107
108  if (PcsAttr *PCS = D->getAttr<PcsAttr>())
109    return (PCS->getPCS() == PcsAttr::AAPCS ? CC_AAPCS : CC_AAPCS_VFP);
110
111  return CC_C;
112}
113
114const CGFunctionInfo &CodeGenTypes::getFunctionInfo(const CXXRecordDecl *RD,
115                                                 const FunctionProtoType *FTP) {
116  SmallVector<CanQualType, 16> ArgTys;
117
118  // Add the 'this' pointer.
119  ArgTys.push_back(GetThisType(Context, RD));
120
121  return ::getFunctionInfo(*this, ArgTys,
122              FTP->getCanonicalTypeUnqualified().getAs<FunctionProtoType>());
123}
124
125const CGFunctionInfo &CodeGenTypes::getFunctionInfo(const CXXMethodDecl *MD) {
126  SmallVector<CanQualType, 16> ArgTys;
127
128  assert(!isa<CXXConstructorDecl>(MD) && "wrong method for contructors!");
129  assert(!isa<CXXDestructorDecl>(MD) && "wrong method for destructors!");
130
131  // Add the 'this' pointer unless this is a static method.
132  if (MD->isInstance())
133    ArgTys.push_back(GetThisType(Context, MD->getParent()));
134
135  return ::getFunctionInfo(*this, ArgTys, GetFormalType(MD));
136}
137
138const CGFunctionInfo &CodeGenTypes::getFunctionInfo(const CXXConstructorDecl *D,
139                                                    CXXCtorType Type) {
140  SmallVector<CanQualType, 16> ArgTys;
141  ArgTys.push_back(GetThisType(Context, D->getParent()));
142  CanQualType ResTy = Context.VoidTy;
143
144  TheCXXABI.BuildConstructorSignature(D, Type, ResTy, ArgTys);
145
146  CanQual<FunctionProtoType> FTP = GetFormalType(D);
147
148  // Add the formal parameters.
149  for (unsigned i = 0, e = FTP->getNumArgs(); i != e; ++i)
150    ArgTys.push_back(FTP->getArgType(i));
151
152  return getFunctionInfo(ResTy, ArgTys, FTP->getExtInfo());
153}
154
155const CGFunctionInfo &CodeGenTypes::getFunctionInfo(const CXXDestructorDecl *D,
156                                                    CXXDtorType Type) {
157  SmallVector<CanQualType, 2> ArgTys;
158  ArgTys.push_back(GetThisType(Context, D->getParent()));
159  CanQualType ResTy = Context.VoidTy;
160
161  TheCXXABI.BuildDestructorSignature(D, Type, ResTy, ArgTys);
162
163  CanQual<FunctionProtoType> FTP = GetFormalType(D);
164  assert(FTP->getNumArgs() == 0 && "dtor with formal parameters");
165
166  return getFunctionInfo(ResTy, ArgTys, FTP->getExtInfo());
167}
168
169const CGFunctionInfo &CodeGenTypes::getFunctionInfo(const FunctionDecl *FD) {
170  if (const CXXMethodDecl *MD = dyn_cast<CXXMethodDecl>(FD))
171    if (MD->isInstance())
172      return getFunctionInfo(MD);
173
174  CanQualType FTy = FD->getType()->getCanonicalTypeUnqualified();
175  assert(isa<FunctionType>(FTy));
176  if (isa<FunctionNoProtoType>(FTy))
177    return getFunctionInfo(FTy.getAs<FunctionNoProtoType>());
178  assert(isa<FunctionProtoType>(FTy));
179  return getFunctionInfo(FTy.getAs<FunctionProtoType>());
180}
181
182const CGFunctionInfo &CodeGenTypes::getFunctionInfo(const ObjCMethodDecl *MD) {
183  SmallVector<CanQualType, 16> ArgTys;
184  ArgTys.push_back(Context.getCanonicalParamType(MD->getSelfDecl()->getType()));
185  ArgTys.push_back(Context.getCanonicalParamType(Context.getObjCSelType()));
186  // FIXME: Kill copy?
187  for (ObjCMethodDecl::param_const_iterator i = MD->param_begin(),
188         e = MD->param_end(); i != e; ++i) {
189    ArgTys.push_back(Context.getCanonicalParamType((*i)->getType()));
190  }
191
192  FunctionType::ExtInfo einfo;
193  einfo = einfo.withCallingConv(getCallingConventionForDecl(MD));
194
195  if (getContext().getLangOptions().ObjCAutoRefCount &&
196      MD->hasAttr<NSReturnsRetainedAttr>())
197    einfo = einfo.withProducesResult(true);
198
199  return getFunctionInfo(GetReturnType(MD->getResultType()), ArgTys, einfo);
200}
201
202const CGFunctionInfo &CodeGenTypes::getFunctionInfo(GlobalDecl GD) {
203  // FIXME: Do we need to handle ObjCMethodDecl?
204  const FunctionDecl *FD = cast<FunctionDecl>(GD.getDecl());
205
206  if (const CXXConstructorDecl *CD = dyn_cast<CXXConstructorDecl>(FD))
207    return getFunctionInfo(CD, GD.getCtorType());
208
209  if (const CXXDestructorDecl *DD = dyn_cast<CXXDestructorDecl>(FD))
210    return getFunctionInfo(DD, GD.getDtorType());
211
212  return getFunctionInfo(FD);
213}
214
215const CGFunctionInfo &CodeGenTypes::getFunctionInfo(QualType ResTy,
216                                                    const CallArgList &Args,
217                                            const FunctionType::ExtInfo &Info) {
218  // FIXME: Kill copy.
219  SmallVector<CanQualType, 16> ArgTys;
220  for (CallArgList::const_iterator i = Args.begin(), e = Args.end();
221       i != e; ++i)
222    ArgTys.push_back(Context.getCanonicalParamType(i->Ty));
223  return getFunctionInfo(GetReturnType(ResTy), ArgTys, Info);
224}
225
226const CGFunctionInfo &CodeGenTypes::getFunctionInfo(QualType ResTy,
227                                                    const FunctionArgList &Args,
228                                            const FunctionType::ExtInfo &Info) {
229  // FIXME: Kill copy.
230  SmallVector<CanQualType, 16> ArgTys;
231  for (FunctionArgList::const_iterator i = Args.begin(), e = Args.end();
232       i != e; ++i)
233    ArgTys.push_back(Context.getCanonicalParamType((*i)->getType()));
234  return getFunctionInfo(GetReturnType(ResTy), ArgTys, Info);
235}
236
237const CGFunctionInfo &CodeGenTypes::getNullaryFunctionInfo() {
238  SmallVector<CanQualType, 1> args;
239  return getFunctionInfo(getContext().VoidTy, args, FunctionType::ExtInfo());
240}
241
242const CGFunctionInfo &CodeGenTypes::getFunctionInfo(CanQualType ResTy,
243                           const SmallVectorImpl<CanQualType> &ArgTys,
244                                            const FunctionType::ExtInfo &Info) {
245#ifndef NDEBUG
246  for (SmallVectorImpl<CanQualType>::const_iterator
247         I = ArgTys.begin(), E = ArgTys.end(); I != E; ++I)
248    assert(I->isCanonicalAsParam());
249#endif
250
251  unsigned CC = ClangCallConvToLLVMCallConv(Info.getCC());
252
253  // Lookup or create unique function info.
254  llvm::FoldingSetNodeID ID;
255  CGFunctionInfo::Profile(ID, Info, ResTy, ArgTys.begin(), ArgTys.end());
256
257  void *InsertPos = 0;
258  CGFunctionInfo *FI = FunctionInfos.FindNodeOrInsertPos(ID, InsertPos);
259  if (FI)
260    return *FI;
261
262  // Construct the function info.
263  FI = new CGFunctionInfo(CC, Info.getNoReturn(), Info.getProducesResult(),
264                          Info.getHasRegParm(), Info.getRegParm(), ResTy,
265                          ArgTys.data(), ArgTys.size());
266  FunctionInfos.InsertNode(FI, InsertPos);
267
268  bool Inserted = FunctionsBeingProcessed.insert(FI); (void)Inserted;
269  assert(Inserted && "Recursively being processed?");
270
271  // Compute ABI information.
272  getABIInfo().computeInfo(*FI);
273
274  // Loop over all of the computed argument and return value info.  If any of
275  // them are direct or extend without a specified coerce type, specify the
276  // default now.
277  ABIArgInfo &RetInfo = FI->getReturnInfo();
278  if (RetInfo.canHaveCoerceToType() && RetInfo.getCoerceToType() == 0)
279    RetInfo.setCoerceToType(ConvertType(FI->getReturnType()));
280
281  for (CGFunctionInfo::arg_iterator I = FI->arg_begin(), E = FI->arg_end();
282       I != E; ++I)
283    if (I->info.canHaveCoerceToType() && I->info.getCoerceToType() == 0)
284      I->info.setCoerceToType(ConvertType(I->type));
285
286  bool Erased = FunctionsBeingProcessed.erase(FI); (void)Erased;
287  assert(Erased && "Not in set?");
288
289  return *FI;
290}
291
292CGFunctionInfo::CGFunctionInfo(unsigned _CallingConvention,
293                               bool _NoReturn, bool returnsRetained,
294                               bool _HasRegParm, unsigned _RegParm,
295                               CanQualType ResTy,
296                               const CanQualType *ArgTys,
297                               unsigned NumArgTys)
298  : CallingConvention(_CallingConvention),
299    EffectiveCallingConvention(_CallingConvention),
300    NoReturn(_NoReturn), ReturnsRetained(returnsRetained),
301    HasRegParm(_HasRegParm), RegParm(_RegParm)
302{
303  NumArgs = NumArgTys;
304
305  // FIXME: Coallocate with the CGFunctionInfo object.
306  Args = new ArgInfo[1 + NumArgTys];
307  Args[0].type = ResTy;
308  for (unsigned i = 0; i != NumArgTys; ++i)
309    Args[1 + i].type = ArgTys[i];
310}
311
312/***/
313
314void CodeGenTypes::GetExpandedTypes(QualType type,
315                     SmallVectorImpl<llvm::Type*> &expandedTypes) {
316  if (const ConstantArrayType *AT = Context.getAsConstantArrayType(type)) {
317    uint64_t NumElts = AT->getSize().getZExtValue();
318    for (uint64_t Elt = 0; Elt < NumElts; ++Elt)
319      GetExpandedTypes(AT->getElementType(), expandedTypes);
320  } else if (const RecordType *RT = type->getAsStructureType()) {
321    const RecordDecl *RD = RT->getDecl();
322    assert(!RD->hasFlexibleArrayMember() &&
323           "Cannot expand structure with flexible array.");
324    for (RecordDecl::field_iterator i = RD->field_begin(), e = RD->field_end();
325         i != e; ++i) {
326      const FieldDecl *FD = *i;
327      assert(!FD->isBitField() &&
328             "Cannot expand structure with bit-field members.");
329      GetExpandedTypes(FD->getType(), expandedTypes);
330    }
331  } else if (const ComplexType *CT = type->getAs<ComplexType>()) {
332    llvm::Type *EltTy = ConvertType(CT->getElementType());
333    expandedTypes.push_back(EltTy);
334    expandedTypes.push_back(EltTy);
335  } else
336    expandedTypes.push_back(ConvertType(type));
337}
338
339llvm::Function::arg_iterator
340CodeGenFunction::ExpandTypeFromArgs(QualType Ty, LValue LV,
341                                    llvm::Function::arg_iterator AI) {
342  assert(LV.isSimple() &&
343         "Unexpected non-simple lvalue during struct expansion.");
344  llvm::Value *Addr = LV.getAddress();
345
346  if (const ConstantArrayType *AT = getContext().getAsConstantArrayType(Ty)) {
347    unsigned NumElts = AT->getSize().getZExtValue();
348    QualType EltTy = AT->getElementType();
349    for (unsigned Elt = 0; Elt < NumElts; ++Elt) {
350      llvm::Value *EltAddr = Builder.CreateConstGEP2_32(Addr, 0, Elt);
351      LValue LV = MakeAddrLValue(EltAddr, EltTy);
352      AI = ExpandTypeFromArgs(EltTy, LV, AI);
353    }
354  } else if (const RecordType *RT = Ty->getAsStructureType()) {
355    RecordDecl *RD = RT->getDecl();
356    for (RecordDecl::field_iterator i = RD->field_begin(), e = RD->field_end();
357         i != e; ++i) {
358      FieldDecl *FD = *i;
359      QualType FT = FD->getType();
360
361      // FIXME: What are the right qualifiers here?
362      LValue LV = EmitLValueForField(Addr, FD, 0);
363      AI = ExpandTypeFromArgs(FT, LV, AI);
364    }
365  } else if (const ComplexType *CT = Ty->getAs<ComplexType>()) {
366    QualType EltTy = CT->getElementType();
367    llvm::Value *RealAddr = Builder.CreateStructGEP(Addr, 0, "real");
368    EmitStoreThroughLValue(RValue::get(AI++), MakeAddrLValue(RealAddr, EltTy));
369    llvm::Value *ImagAddr = Builder.CreateStructGEP(Addr, 1, "imag");
370    EmitStoreThroughLValue(RValue::get(AI++), MakeAddrLValue(ImagAddr, EltTy));
371  } else {
372    EmitStoreThroughLValue(RValue::get(AI), LV);
373    ++AI;
374  }
375
376  return AI;
377}
378
379/// EnterStructPointerForCoercedAccess - Given a struct pointer that we are
380/// accessing some number of bytes out of it, try to gep into the struct to get
381/// at its inner goodness.  Dive as deep as possible without entering an element
382/// with an in-memory size smaller than DstSize.
383static llvm::Value *
384EnterStructPointerForCoercedAccess(llvm::Value *SrcPtr,
385                                   llvm::StructType *SrcSTy,
386                                   uint64_t DstSize, CodeGenFunction &CGF) {
387  // We can't dive into a zero-element struct.
388  if (SrcSTy->getNumElements() == 0) return SrcPtr;
389
390  llvm::Type *FirstElt = SrcSTy->getElementType(0);
391
392  // If the first elt is at least as large as what we're looking for, or if the
393  // first element is the same size as the whole struct, we can enter it.
394  uint64_t FirstEltSize =
395    CGF.CGM.getTargetData().getTypeAllocSize(FirstElt);
396  if (FirstEltSize < DstSize &&
397      FirstEltSize < CGF.CGM.getTargetData().getTypeAllocSize(SrcSTy))
398    return SrcPtr;
399
400  // GEP into the first element.
401  SrcPtr = CGF.Builder.CreateConstGEP2_32(SrcPtr, 0, 0, "coerce.dive");
402
403  // If the first element is a struct, recurse.
404  llvm::Type *SrcTy =
405    cast<llvm::PointerType>(SrcPtr->getType())->getElementType();
406  if (llvm::StructType *SrcSTy = dyn_cast<llvm::StructType>(SrcTy))
407    return EnterStructPointerForCoercedAccess(SrcPtr, SrcSTy, DstSize, CGF);
408
409  return SrcPtr;
410}
411
412/// CoerceIntOrPtrToIntOrPtr - Convert a value Val to the specific Ty where both
413/// are either integers or pointers.  This does a truncation of the value if it
414/// is too large or a zero extension if it is too small.
415static llvm::Value *CoerceIntOrPtrToIntOrPtr(llvm::Value *Val,
416                                             llvm::Type *Ty,
417                                             CodeGenFunction &CGF) {
418  if (Val->getType() == Ty)
419    return Val;
420
421  if (isa<llvm::PointerType>(Val->getType())) {
422    // If this is Pointer->Pointer avoid conversion to and from int.
423    if (isa<llvm::PointerType>(Ty))
424      return CGF.Builder.CreateBitCast(Val, Ty, "coerce.val");
425
426    // Convert the pointer to an integer so we can play with its width.
427    Val = CGF.Builder.CreatePtrToInt(Val, CGF.IntPtrTy, "coerce.val.pi");
428  }
429
430  llvm::Type *DestIntTy = Ty;
431  if (isa<llvm::PointerType>(DestIntTy))
432    DestIntTy = CGF.IntPtrTy;
433
434  if (Val->getType() != DestIntTy)
435    Val = CGF.Builder.CreateIntCast(Val, DestIntTy, false, "coerce.val.ii");
436
437  if (isa<llvm::PointerType>(Ty))
438    Val = CGF.Builder.CreateIntToPtr(Val, Ty, "coerce.val.ip");
439  return Val;
440}
441
442
443
444/// CreateCoercedLoad - Create a load from \arg SrcPtr interpreted as
445/// a pointer to an object of type \arg Ty.
446///
447/// This safely handles the case when the src type is smaller than the
448/// destination type; in this situation the values of bits which not
449/// present in the src are undefined.
450static llvm::Value *CreateCoercedLoad(llvm::Value *SrcPtr,
451                                      llvm::Type *Ty,
452                                      CodeGenFunction &CGF) {
453  llvm::Type *SrcTy =
454    cast<llvm::PointerType>(SrcPtr->getType())->getElementType();
455
456  // If SrcTy and Ty are the same, just do a load.
457  if (SrcTy == Ty)
458    return CGF.Builder.CreateLoad(SrcPtr);
459
460  uint64_t DstSize = CGF.CGM.getTargetData().getTypeAllocSize(Ty);
461
462  if (llvm::StructType *SrcSTy = dyn_cast<llvm::StructType>(SrcTy)) {
463    SrcPtr = EnterStructPointerForCoercedAccess(SrcPtr, SrcSTy, DstSize, CGF);
464    SrcTy = cast<llvm::PointerType>(SrcPtr->getType())->getElementType();
465  }
466
467  uint64_t SrcSize = CGF.CGM.getTargetData().getTypeAllocSize(SrcTy);
468
469  // If the source and destination are integer or pointer types, just do an
470  // extension or truncation to the desired type.
471  if ((isa<llvm::IntegerType>(Ty) || isa<llvm::PointerType>(Ty)) &&
472      (isa<llvm::IntegerType>(SrcTy) || isa<llvm::PointerType>(SrcTy))) {
473    llvm::LoadInst *Load = CGF.Builder.CreateLoad(SrcPtr);
474    return CoerceIntOrPtrToIntOrPtr(Load, Ty, CGF);
475  }
476
477  // If load is legal, just bitcast the src pointer.
478  if (SrcSize >= DstSize) {
479    // Generally SrcSize is never greater than DstSize, since this means we are
480    // losing bits. However, this can happen in cases where the structure has
481    // additional padding, for example due to a user specified alignment.
482    //
483    // FIXME: Assert that we aren't truncating non-padding bits when have access
484    // to that information.
485    llvm::Value *Casted =
486      CGF.Builder.CreateBitCast(SrcPtr, llvm::PointerType::getUnqual(Ty));
487    llvm::LoadInst *Load = CGF.Builder.CreateLoad(Casted);
488    // FIXME: Use better alignment / avoid requiring aligned load.
489    Load->setAlignment(1);
490    return Load;
491  }
492
493  // Otherwise do coercion through memory. This is stupid, but
494  // simple.
495  llvm::Value *Tmp = CGF.CreateTempAlloca(Ty);
496  llvm::Value *Casted =
497    CGF.Builder.CreateBitCast(Tmp, llvm::PointerType::getUnqual(SrcTy));
498  llvm::StoreInst *Store =
499    CGF.Builder.CreateStore(CGF.Builder.CreateLoad(SrcPtr), Casted);
500  // FIXME: Use better alignment / avoid requiring aligned store.
501  Store->setAlignment(1);
502  return CGF.Builder.CreateLoad(Tmp);
503}
504
505// Function to store a first-class aggregate into memory.  We prefer to
506// store the elements rather than the aggregate to be more friendly to
507// fast-isel.
508// FIXME: Do we need to recurse here?
509static void BuildAggStore(CodeGenFunction &CGF, llvm::Value *Val,
510                          llvm::Value *DestPtr, bool DestIsVolatile,
511                          bool LowAlignment) {
512  // Prefer scalar stores to first-class aggregate stores.
513  if (llvm::StructType *STy =
514        dyn_cast<llvm::StructType>(Val->getType())) {
515    for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) {
516      llvm::Value *EltPtr = CGF.Builder.CreateConstGEP2_32(DestPtr, 0, i);
517      llvm::Value *Elt = CGF.Builder.CreateExtractValue(Val, i);
518      llvm::StoreInst *SI = CGF.Builder.CreateStore(Elt, EltPtr,
519                                                    DestIsVolatile);
520      if (LowAlignment)
521        SI->setAlignment(1);
522    }
523  } else {
524    CGF.Builder.CreateStore(Val, DestPtr, DestIsVolatile);
525  }
526}
527
528/// CreateCoercedStore - Create a store to \arg DstPtr from \arg Src,
529/// where the source and destination may have different types.
530///
531/// This safely handles the case when the src type is larger than the
532/// destination type; the upper bits of the src will be lost.
533static void CreateCoercedStore(llvm::Value *Src,
534                               llvm::Value *DstPtr,
535                               bool DstIsVolatile,
536                               CodeGenFunction &CGF) {
537  llvm::Type *SrcTy = Src->getType();
538  llvm::Type *DstTy =
539    cast<llvm::PointerType>(DstPtr->getType())->getElementType();
540  if (SrcTy == DstTy) {
541    CGF.Builder.CreateStore(Src, DstPtr, DstIsVolatile);
542    return;
543  }
544
545  uint64_t SrcSize = CGF.CGM.getTargetData().getTypeAllocSize(SrcTy);
546
547  if (llvm::StructType *DstSTy = dyn_cast<llvm::StructType>(DstTy)) {
548    DstPtr = EnterStructPointerForCoercedAccess(DstPtr, DstSTy, SrcSize, CGF);
549    DstTy = cast<llvm::PointerType>(DstPtr->getType())->getElementType();
550  }
551
552  // If the source and destination are integer or pointer types, just do an
553  // extension or truncation to the desired type.
554  if ((isa<llvm::IntegerType>(SrcTy) || isa<llvm::PointerType>(SrcTy)) &&
555      (isa<llvm::IntegerType>(DstTy) || isa<llvm::PointerType>(DstTy))) {
556    Src = CoerceIntOrPtrToIntOrPtr(Src, DstTy, CGF);
557    CGF.Builder.CreateStore(Src, DstPtr, DstIsVolatile);
558    return;
559  }
560
561  uint64_t DstSize = CGF.CGM.getTargetData().getTypeAllocSize(DstTy);
562
563  // If store is legal, just bitcast the src pointer.
564  if (SrcSize <= DstSize) {
565    llvm::Value *Casted =
566      CGF.Builder.CreateBitCast(DstPtr, llvm::PointerType::getUnqual(SrcTy));
567    // FIXME: Use better alignment / avoid requiring aligned store.
568    BuildAggStore(CGF, Src, Casted, DstIsVolatile, true);
569  } else {
570    // Otherwise do coercion through memory. This is stupid, but
571    // simple.
572
573    // Generally SrcSize is never greater than DstSize, since this means we are
574    // losing bits. However, this can happen in cases where the structure has
575    // additional padding, for example due to a user specified alignment.
576    //
577    // FIXME: Assert that we aren't truncating non-padding bits when have access
578    // to that information.
579    llvm::Value *Tmp = CGF.CreateTempAlloca(SrcTy);
580    CGF.Builder.CreateStore(Src, Tmp);
581    llvm::Value *Casted =
582      CGF.Builder.CreateBitCast(Tmp, llvm::PointerType::getUnqual(DstTy));
583    llvm::LoadInst *Load = CGF.Builder.CreateLoad(Casted);
584    // FIXME: Use better alignment / avoid requiring aligned load.
585    Load->setAlignment(1);
586    CGF.Builder.CreateStore(Load, DstPtr, DstIsVolatile);
587  }
588}
589
590/***/
591
592bool CodeGenModule::ReturnTypeUsesSRet(const CGFunctionInfo &FI) {
593  return FI.getReturnInfo().isIndirect();
594}
595
596bool CodeGenModule::ReturnTypeUsesFPRet(QualType ResultType) {
597  if (const BuiltinType *BT = ResultType->getAs<BuiltinType>()) {
598    switch (BT->getKind()) {
599    default:
600      return false;
601    case BuiltinType::Float:
602      return getContext().getTargetInfo().useObjCFPRetForRealType(TargetInfo::Float);
603    case BuiltinType::Double:
604      return getContext().getTargetInfo().useObjCFPRetForRealType(TargetInfo::Double);
605    case BuiltinType::LongDouble:
606      return getContext().getTargetInfo().useObjCFPRetForRealType(
607        TargetInfo::LongDouble);
608    }
609  }
610
611  return false;
612}
613
614bool CodeGenModule::ReturnTypeUsesFP2Ret(QualType ResultType) {
615  if (const ComplexType *CT = ResultType->getAs<ComplexType>()) {
616    if (const BuiltinType *BT = CT->getElementType()->getAs<BuiltinType>()) {
617      if (BT->getKind() == BuiltinType::LongDouble)
618        return getContext().getTargetInfo().useObjCFP2RetForComplexLongDouble();
619    }
620  }
621
622  return false;
623}
624
625llvm::FunctionType *CodeGenTypes::GetFunctionType(GlobalDecl GD) {
626  const CGFunctionInfo &FI = getFunctionInfo(GD);
627
628  // For definition purposes, don't consider a K&R function variadic.
629  bool Variadic = false;
630  if (const FunctionProtoType *FPT =
631        cast<FunctionDecl>(GD.getDecl())->getType()->getAs<FunctionProtoType>())
632    Variadic = FPT->isVariadic();
633
634  return GetFunctionType(FI, Variadic);
635}
636
637llvm::FunctionType *
638CodeGenTypes::GetFunctionType(const CGFunctionInfo &FI, bool isVariadic) {
639
640  bool Inserted = FunctionsBeingProcessed.insert(&FI); (void)Inserted;
641  assert(Inserted && "Recursively being processed?");
642
643  SmallVector<llvm::Type*, 8> argTypes;
644  llvm::Type *resultType = 0;
645
646  const ABIArgInfo &retAI = FI.getReturnInfo();
647  switch (retAI.getKind()) {
648  case ABIArgInfo::Expand:
649    llvm_unreachable("Invalid ABI kind for return argument");
650
651  case ABIArgInfo::Extend:
652  case ABIArgInfo::Direct:
653    resultType = retAI.getCoerceToType();
654    break;
655
656  case ABIArgInfo::Indirect: {
657    assert(!retAI.getIndirectAlign() && "Align unused on indirect return.");
658    resultType = llvm::Type::getVoidTy(getLLVMContext());
659
660    QualType ret = FI.getReturnType();
661    llvm::Type *ty = ConvertType(ret);
662    unsigned addressSpace = Context.getTargetAddressSpace(ret);
663    argTypes.push_back(llvm::PointerType::get(ty, addressSpace));
664    break;
665  }
666
667  case ABIArgInfo::Ignore:
668    resultType = llvm::Type::getVoidTy(getLLVMContext());
669    break;
670  }
671
672  for (CGFunctionInfo::const_arg_iterator it = FI.arg_begin(),
673         ie = FI.arg_end(); it != ie; ++it) {
674    const ABIArgInfo &argAI = it->info;
675
676    switch (argAI.getKind()) {
677    case ABIArgInfo::Ignore:
678      break;
679
680    case ABIArgInfo::Indirect: {
681      // indirect arguments are always on the stack, which is addr space #0.
682      llvm::Type *LTy = ConvertTypeForMem(it->type);
683      argTypes.push_back(LTy->getPointerTo());
684      break;
685    }
686
687    case ABIArgInfo::Extend:
688    case ABIArgInfo::Direct: {
689      // Insert a padding type to ensure proper alignment.
690      if (llvm::Type *PaddingType = argAI.getPaddingType())
691        argTypes.push_back(PaddingType);
692      // If the coerce-to type is a first class aggregate, flatten it.  Either
693      // way is semantically identical, but fast-isel and the optimizer
694      // generally likes scalar values better than FCAs.
695      llvm::Type *argType = argAI.getCoerceToType();
696      if (llvm::StructType *st = dyn_cast<llvm::StructType>(argType)) {
697        for (unsigned i = 0, e = st->getNumElements(); i != e; ++i)
698          argTypes.push_back(st->getElementType(i));
699      } else {
700        argTypes.push_back(argType);
701      }
702      break;
703    }
704
705    case ABIArgInfo::Expand:
706      GetExpandedTypes(it->type, argTypes);
707      break;
708    }
709  }
710
711  bool Erased = FunctionsBeingProcessed.erase(&FI); (void)Erased;
712  assert(Erased && "Not in set?");
713
714  return llvm::FunctionType::get(resultType, argTypes, isVariadic);
715}
716
717llvm::Type *CodeGenTypes::GetFunctionTypeForVTable(GlobalDecl GD) {
718  const CXXMethodDecl *MD = cast<CXXMethodDecl>(GD.getDecl());
719  const FunctionProtoType *FPT = MD->getType()->getAs<FunctionProtoType>();
720
721  if (!isFuncTypeConvertible(FPT))
722    return llvm::StructType::get(getLLVMContext());
723
724  const CGFunctionInfo *Info;
725  if (isa<CXXDestructorDecl>(MD))
726    Info = &getFunctionInfo(cast<CXXDestructorDecl>(MD), GD.getDtorType());
727  else
728    Info = &getFunctionInfo(MD);
729  return GetFunctionType(*Info, FPT->isVariadic());
730}
731
732void CodeGenModule::ConstructAttributeList(const CGFunctionInfo &FI,
733                                           const Decl *TargetDecl,
734                                           AttributeListType &PAL,
735                                           unsigned &CallingConv) {
736  unsigned FuncAttrs = 0;
737  unsigned RetAttrs = 0;
738
739  CallingConv = FI.getEffectiveCallingConvention();
740
741  if (FI.isNoReturn())
742    FuncAttrs |= llvm::Attribute::NoReturn;
743
744  // FIXME: handle sseregparm someday...
745  if (TargetDecl) {
746    if (TargetDecl->hasAttr<ReturnsTwiceAttr>())
747      FuncAttrs |= llvm::Attribute::ReturnsTwice;
748    if (TargetDecl->hasAttr<NoThrowAttr>())
749      FuncAttrs |= llvm::Attribute::NoUnwind;
750    else if (const FunctionDecl *Fn = dyn_cast<FunctionDecl>(TargetDecl)) {
751      const FunctionProtoType *FPT = Fn->getType()->getAs<FunctionProtoType>();
752      if (FPT && FPT->isNothrow(getContext()))
753        FuncAttrs |= llvm::Attribute::NoUnwind;
754    }
755
756    if (TargetDecl->hasAttr<NoReturnAttr>())
757      FuncAttrs |= llvm::Attribute::NoReturn;
758
759    if (TargetDecl->hasAttr<ReturnsTwiceAttr>())
760      FuncAttrs |= llvm::Attribute::ReturnsTwice;
761
762    // 'const' and 'pure' attribute functions are also nounwind.
763    if (TargetDecl->hasAttr<ConstAttr>()) {
764      FuncAttrs |= llvm::Attribute::ReadNone;
765      FuncAttrs |= llvm::Attribute::NoUnwind;
766    } else if (TargetDecl->hasAttr<PureAttr>()) {
767      FuncAttrs |= llvm::Attribute::ReadOnly;
768      FuncAttrs |= llvm::Attribute::NoUnwind;
769    }
770    if (TargetDecl->hasAttr<MallocAttr>())
771      RetAttrs |= llvm::Attribute::NoAlias;
772  }
773
774  if (CodeGenOpts.OptimizeSize)
775    FuncAttrs |= llvm::Attribute::OptimizeForSize;
776  if (CodeGenOpts.DisableRedZone)
777    FuncAttrs |= llvm::Attribute::NoRedZone;
778  if (CodeGenOpts.NoImplicitFloat)
779    FuncAttrs |= llvm::Attribute::NoImplicitFloat;
780
781  QualType RetTy = FI.getReturnType();
782  unsigned Index = 1;
783  const ABIArgInfo &RetAI = FI.getReturnInfo();
784  switch (RetAI.getKind()) {
785  case ABIArgInfo::Extend:
786   if (RetTy->hasSignedIntegerRepresentation())
787     RetAttrs |= llvm::Attribute::SExt;
788   else if (RetTy->hasUnsignedIntegerRepresentation())
789     RetAttrs |= llvm::Attribute::ZExt;
790    break;
791  case ABIArgInfo::Direct:
792  case ABIArgInfo::Ignore:
793    break;
794
795  case ABIArgInfo::Indirect:
796    PAL.push_back(llvm::AttributeWithIndex::get(Index,
797                                                llvm::Attribute::StructRet));
798    ++Index;
799    // sret disables readnone and readonly
800    FuncAttrs &= ~(llvm::Attribute::ReadOnly |
801                   llvm::Attribute::ReadNone);
802    break;
803
804  case ABIArgInfo::Expand:
805    llvm_unreachable("Invalid ABI kind for return argument");
806  }
807
808  if (RetAttrs)
809    PAL.push_back(llvm::AttributeWithIndex::get(0, RetAttrs));
810
811  // FIXME: RegParm should be reduced in case of global register variable.
812  signed RegParm;
813  if (FI.getHasRegParm())
814    RegParm = FI.getRegParm();
815  else
816    RegParm = CodeGenOpts.NumRegisterParameters;
817
818  unsigned PointerWidth = getContext().getTargetInfo().getPointerWidth(0);
819  for (CGFunctionInfo::const_arg_iterator it = FI.arg_begin(),
820         ie = FI.arg_end(); it != ie; ++it) {
821    QualType ParamType = it->type;
822    const ABIArgInfo &AI = it->info;
823    unsigned Attributes = 0;
824
825    // 'restrict' -> 'noalias' is done in EmitFunctionProlog when we
826    // have the corresponding parameter variable.  It doesn't make
827    // sense to do it here because parameters are so messed up.
828    switch (AI.getKind()) {
829    case ABIArgInfo::Extend:
830      if (ParamType->isSignedIntegerOrEnumerationType())
831        Attributes |= llvm::Attribute::SExt;
832      else if (ParamType->isUnsignedIntegerOrEnumerationType())
833        Attributes |= llvm::Attribute::ZExt;
834      // FALL THROUGH
835    case ABIArgInfo::Direct:
836      if (RegParm > 0 &&
837          (ParamType->isIntegerType() || ParamType->isPointerType() ||
838           ParamType->isReferenceType())) {
839        RegParm -=
840        (Context.getTypeSize(ParamType) + PointerWidth - 1) / PointerWidth;
841        if (RegParm >= 0)
842          Attributes |= llvm::Attribute::InReg;
843      }
844      // FIXME: handle sseregparm someday...
845
846      // Increment Index if there is padding.
847      Index += (AI.getPaddingType() != 0);
848
849      if (llvm::StructType *STy =
850            dyn_cast<llvm::StructType>(AI.getCoerceToType()))
851        Index += STy->getNumElements()-1;  // 1 will be added below.
852      break;
853
854    case ABIArgInfo::Indirect:
855      if (AI.getIndirectByVal())
856        Attributes |= llvm::Attribute::ByVal;
857
858      Attributes |=
859        llvm::Attribute::constructAlignmentFromInt(AI.getIndirectAlign());
860      // byval disables readnone and readonly.
861      FuncAttrs &= ~(llvm::Attribute::ReadOnly |
862                     llvm::Attribute::ReadNone);
863      break;
864
865    case ABIArgInfo::Ignore:
866      // Skip increment, no matching LLVM parameter.
867      continue;
868
869    case ABIArgInfo::Expand: {
870      SmallVector<llvm::Type*, 8> types;
871      // FIXME: This is rather inefficient. Do we ever actually need to do
872      // anything here? The result should be just reconstructed on the other
873      // side, so extension should be a non-issue.
874      getTypes().GetExpandedTypes(ParamType, types);
875      Index += types.size();
876      continue;
877    }
878    }
879
880    if (Attributes)
881      PAL.push_back(llvm::AttributeWithIndex::get(Index, Attributes));
882    ++Index;
883  }
884  if (FuncAttrs)
885    PAL.push_back(llvm::AttributeWithIndex::get(~0, FuncAttrs));
886}
887
888/// An argument came in as a promoted argument; demote it back to its
889/// declared type.
890static llvm::Value *emitArgumentDemotion(CodeGenFunction &CGF,
891                                         const VarDecl *var,
892                                         llvm::Value *value) {
893  llvm::Type *varType = CGF.ConvertType(var->getType());
894
895  // This can happen with promotions that actually don't change the
896  // underlying type, like the enum promotions.
897  if (value->getType() == varType) return value;
898
899  assert((varType->isIntegerTy() || varType->isFloatingPointTy())
900         && "unexpected promotion type");
901
902  if (isa<llvm::IntegerType>(varType))
903    return CGF.Builder.CreateTrunc(value, varType, "arg.unpromote");
904
905  return CGF.Builder.CreateFPCast(value, varType, "arg.unpromote");
906}
907
908void CodeGenFunction::EmitFunctionProlog(const CGFunctionInfo &FI,
909                                         llvm::Function *Fn,
910                                         const FunctionArgList &Args) {
911  // If this is an implicit-return-zero function, go ahead and
912  // initialize the return value.  TODO: it might be nice to have
913  // a more general mechanism for this that didn't require synthesized
914  // return statements.
915  if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(CurFuncDecl)) {
916    if (FD->hasImplicitReturnZero()) {
917      QualType RetTy = FD->getResultType().getUnqualifiedType();
918      llvm::Type* LLVMTy = CGM.getTypes().ConvertType(RetTy);
919      llvm::Constant* Zero = llvm::Constant::getNullValue(LLVMTy);
920      Builder.CreateStore(Zero, ReturnValue);
921    }
922  }
923
924  // FIXME: We no longer need the types from FunctionArgList; lift up and
925  // simplify.
926
927  // Emit allocs for param decls.  Give the LLVM Argument nodes names.
928  llvm::Function::arg_iterator AI = Fn->arg_begin();
929
930  // Name the struct return argument.
931  if (CGM.ReturnTypeUsesSRet(FI)) {
932    AI->setName("agg.result");
933    AI->addAttr(llvm::Attribute::NoAlias);
934    ++AI;
935  }
936
937  assert(FI.arg_size() == Args.size() &&
938         "Mismatch between function signature & arguments.");
939  unsigned ArgNo = 1;
940  CGFunctionInfo::const_arg_iterator info_it = FI.arg_begin();
941  for (FunctionArgList::const_iterator i = Args.begin(), e = Args.end();
942       i != e; ++i, ++info_it, ++ArgNo) {
943    const VarDecl *Arg = *i;
944    QualType Ty = info_it->type;
945    const ABIArgInfo &ArgI = info_it->info;
946
947    bool isPromoted =
948      isa<ParmVarDecl>(Arg) && cast<ParmVarDecl>(Arg)->isKNRPromoted();
949
950    switch (ArgI.getKind()) {
951    case ABIArgInfo::Indirect: {
952      llvm::Value *V = AI;
953
954      if (hasAggregateLLVMType(Ty)) {
955        // Aggregates and complex variables are accessed by reference.  All we
956        // need to do is realign the value, if requested
957        if (ArgI.getIndirectRealign()) {
958          llvm::Value *AlignedTemp = CreateMemTemp(Ty, "coerce");
959
960          // Copy from the incoming argument pointer to the temporary with the
961          // appropriate alignment.
962          //
963          // FIXME: We should have a common utility for generating an aggregate
964          // copy.
965          llvm::Type *I8PtrTy = Builder.getInt8PtrTy();
966          CharUnits Size = getContext().getTypeSizeInChars(Ty);
967          llvm::Value *Dst = Builder.CreateBitCast(AlignedTemp, I8PtrTy);
968          llvm::Value *Src = Builder.CreateBitCast(V, I8PtrTy);
969          Builder.CreateMemCpy(Dst,
970                               Src,
971                               llvm::ConstantInt::get(IntPtrTy,
972                                                      Size.getQuantity()),
973                               ArgI.getIndirectAlign(),
974                               false);
975          V = AlignedTemp;
976        }
977      } else {
978        // Load scalar value from indirect argument.
979        CharUnits Alignment = getContext().getTypeAlignInChars(Ty);
980        V = EmitLoadOfScalar(V, false, Alignment.getQuantity(), Ty);
981
982        if (isPromoted)
983          V = emitArgumentDemotion(*this, Arg, V);
984      }
985      EmitParmDecl(*Arg, V, ArgNo);
986      break;
987    }
988
989    case ABIArgInfo::Extend:
990    case ABIArgInfo::Direct: {
991      // If we have the trivial case, handle it with no muss and fuss.
992      if (!isa<llvm::StructType>(ArgI.getCoerceToType()) &&
993          ArgI.getCoerceToType() == ConvertType(Ty) &&
994          ArgI.getDirectOffset() == 0) {
995        assert(AI != Fn->arg_end() && "Argument mismatch!");
996        llvm::Value *V = AI;
997
998        if (Arg->getType().isRestrictQualified())
999          AI->addAttr(llvm::Attribute::NoAlias);
1000
1001        // Ensure the argument is the correct type.
1002        if (V->getType() != ArgI.getCoerceToType())
1003          V = Builder.CreateBitCast(V, ArgI.getCoerceToType());
1004
1005        if (isPromoted)
1006          V = emitArgumentDemotion(*this, Arg, V);
1007
1008        EmitParmDecl(*Arg, V, ArgNo);
1009        break;
1010      }
1011
1012      llvm::AllocaInst *Alloca = CreateMemTemp(Ty, "coerce");
1013
1014      // The alignment we need to use is the max of the requested alignment for
1015      // the argument plus the alignment required by our access code below.
1016      unsigned AlignmentToUse =
1017        CGM.getTargetData().getABITypeAlignment(ArgI.getCoerceToType());
1018      AlignmentToUse = std::max(AlignmentToUse,
1019                        (unsigned)getContext().getDeclAlign(Arg).getQuantity());
1020
1021      Alloca->setAlignment(AlignmentToUse);
1022      llvm::Value *V = Alloca;
1023      llvm::Value *Ptr = V;    // Pointer to store into.
1024
1025      // If the value is offset in memory, apply the offset now.
1026      if (unsigned Offs = ArgI.getDirectOffset()) {
1027        Ptr = Builder.CreateBitCast(Ptr, Builder.getInt8PtrTy());
1028        Ptr = Builder.CreateConstGEP1_32(Ptr, Offs);
1029        Ptr = Builder.CreateBitCast(Ptr,
1030                          llvm::PointerType::getUnqual(ArgI.getCoerceToType()));
1031      }
1032
1033      // Skip the dummy padding argument.
1034      if (ArgI.getPaddingType())
1035        ++AI;
1036
1037      // If the coerce-to type is a first class aggregate, we flatten it and
1038      // pass the elements. Either way is semantically identical, but fast-isel
1039      // and the optimizer generally likes scalar values better than FCAs.
1040      if (llvm::StructType *STy =
1041            dyn_cast<llvm::StructType>(ArgI.getCoerceToType())) {
1042        Ptr = Builder.CreateBitCast(Ptr, llvm::PointerType::getUnqual(STy));
1043
1044        for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) {
1045          assert(AI != Fn->arg_end() && "Argument mismatch!");
1046          AI->setName(Arg->getName() + ".coerce" + Twine(i));
1047          llvm::Value *EltPtr = Builder.CreateConstGEP2_32(Ptr, 0, i);
1048          Builder.CreateStore(AI++, EltPtr);
1049        }
1050      } else {
1051        // Simple case, just do a coerced store of the argument into the alloca.
1052        assert(AI != Fn->arg_end() && "Argument mismatch!");
1053        AI->setName(Arg->getName() + ".coerce");
1054        CreateCoercedStore(AI++, Ptr, /*DestIsVolatile=*/false, *this);
1055      }
1056
1057
1058      // Match to what EmitParmDecl is expecting for this type.
1059      if (!CodeGenFunction::hasAggregateLLVMType(Ty)) {
1060        V = EmitLoadOfScalar(V, false, AlignmentToUse, Ty);
1061        if (isPromoted)
1062          V = emitArgumentDemotion(*this, Arg, V);
1063      }
1064      EmitParmDecl(*Arg, V, ArgNo);
1065      continue;  // Skip ++AI increment, already done.
1066    }
1067
1068    case ABIArgInfo::Expand: {
1069      // If this structure was expanded into multiple arguments then
1070      // we need to create a temporary and reconstruct it from the
1071      // arguments.
1072      llvm::AllocaInst *Alloca = CreateMemTemp(Ty);
1073      CharUnits Align = getContext().getDeclAlign(Arg);
1074      Alloca->setAlignment(Align.getQuantity());
1075      LValue LV = MakeAddrLValue(Alloca, Ty, Align);
1076      llvm::Function::arg_iterator End = ExpandTypeFromArgs(Ty, LV, AI);
1077      EmitParmDecl(*Arg, Alloca, ArgNo);
1078
1079      // Name the arguments used in expansion and increment AI.
1080      unsigned Index = 0;
1081      for (; AI != End; ++AI, ++Index)
1082        AI->setName(Arg->getName() + "." + Twine(Index));
1083      continue;
1084    }
1085
1086    case ABIArgInfo::Ignore:
1087      // Initialize the local variable appropriately.
1088      if (hasAggregateLLVMType(Ty))
1089        EmitParmDecl(*Arg, CreateMemTemp(Ty), ArgNo);
1090      else
1091        EmitParmDecl(*Arg, llvm::UndefValue::get(ConvertType(Arg->getType())),
1092                     ArgNo);
1093
1094      // Skip increment, no matching LLVM parameter.
1095      continue;
1096    }
1097
1098    ++AI;
1099  }
1100  assert(AI == Fn->arg_end() && "Argument mismatch!");
1101}
1102
1103/// Try to emit a fused autorelease of a return result.
1104static llvm::Value *tryEmitFusedAutoreleaseOfResult(CodeGenFunction &CGF,
1105                                                    llvm::Value *result) {
1106  // We must be immediately followed the cast.
1107  llvm::BasicBlock *BB = CGF.Builder.GetInsertBlock();
1108  if (BB->empty()) return 0;
1109  if (&BB->back() != result) return 0;
1110
1111  llvm::Type *resultType = result->getType();
1112
1113  // result is in a BasicBlock and is therefore an Instruction.
1114  llvm::Instruction *generator = cast<llvm::Instruction>(result);
1115
1116  SmallVector<llvm::Instruction*,4> insnsToKill;
1117
1118  // Look for:
1119  //  %generator = bitcast %type1* %generator2 to %type2*
1120  while (llvm::BitCastInst *bitcast = dyn_cast<llvm::BitCastInst>(generator)) {
1121    // We would have emitted this as a constant if the operand weren't
1122    // an Instruction.
1123    generator = cast<llvm::Instruction>(bitcast->getOperand(0));
1124
1125    // Require the generator to be immediately followed by the cast.
1126    if (generator->getNextNode() != bitcast)
1127      return 0;
1128
1129    insnsToKill.push_back(bitcast);
1130  }
1131
1132  // Look for:
1133  //   %generator = call i8* @objc_retain(i8* %originalResult)
1134  // or
1135  //   %generator = call i8* @objc_retainAutoreleasedReturnValue(i8* %originalResult)
1136  llvm::CallInst *call = dyn_cast<llvm::CallInst>(generator);
1137  if (!call) return 0;
1138
1139  bool doRetainAutorelease;
1140
1141  if (call->getCalledValue() == CGF.CGM.getARCEntrypoints().objc_retain) {
1142    doRetainAutorelease = true;
1143  } else if (call->getCalledValue() == CGF.CGM.getARCEntrypoints()
1144                                          .objc_retainAutoreleasedReturnValue) {
1145    doRetainAutorelease = false;
1146
1147    // Look for an inline asm immediately preceding the call and kill it, too.
1148    llvm::Instruction *prev = call->getPrevNode();
1149    if (llvm::CallInst *asmCall = dyn_cast_or_null<llvm::CallInst>(prev))
1150      if (asmCall->getCalledValue()
1151            == CGF.CGM.getARCEntrypoints().retainAutoreleasedReturnValueMarker)
1152        insnsToKill.push_back(prev);
1153  } else {
1154    return 0;
1155  }
1156
1157  result = call->getArgOperand(0);
1158  insnsToKill.push_back(call);
1159
1160  // Keep killing bitcasts, for sanity.  Note that we no longer care
1161  // about precise ordering as long as there's exactly one use.
1162  while (llvm::BitCastInst *bitcast = dyn_cast<llvm::BitCastInst>(result)) {
1163    if (!bitcast->hasOneUse()) break;
1164    insnsToKill.push_back(bitcast);
1165    result = bitcast->getOperand(0);
1166  }
1167
1168  // Delete all the unnecessary instructions, from latest to earliest.
1169  for (SmallVectorImpl<llvm::Instruction*>::iterator
1170         i = insnsToKill.begin(), e = insnsToKill.end(); i != e; ++i)
1171    (*i)->eraseFromParent();
1172
1173  // Do the fused retain/autorelease if we were asked to.
1174  if (doRetainAutorelease)
1175    result = CGF.EmitARCRetainAutoreleaseReturnValue(result);
1176
1177  // Cast back to the result type.
1178  return CGF.Builder.CreateBitCast(result, resultType);
1179}
1180
1181/// Emit an ARC autorelease of the result of a function.
1182static llvm::Value *emitAutoreleaseOfResult(CodeGenFunction &CGF,
1183                                            llvm::Value *result) {
1184  // At -O0, try to emit a fused retain/autorelease.
1185  if (CGF.shouldUseFusedARCCalls())
1186    if (llvm::Value *fused = tryEmitFusedAutoreleaseOfResult(CGF, result))
1187      return fused;
1188
1189  return CGF.EmitARCAutoreleaseReturnValue(result);
1190}
1191
1192void CodeGenFunction::EmitFunctionEpilog(const CGFunctionInfo &FI) {
1193  // Functions with no result always return void.
1194  if (ReturnValue == 0) {
1195    Builder.CreateRetVoid();
1196    return;
1197  }
1198
1199  llvm::DebugLoc RetDbgLoc;
1200  llvm::Value *RV = 0;
1201  QualType RetTy = FI.getReturnType();
1202  const ABIArgInfo &RetAI = FI.getReturnInfo();
1203
1204  switch (RetAI.getKind()) {
1205  case ABIArgInfo::Indirect: {
1206    unsigned Alignment = getContext().getTypeAlignInChars(RetTy).getQuantity();
1207    if (RetTy->isAnyComplexType()) {
1208      ComplexPairTy RT = LoadComplexFromAddr(ReturnValue, false);
1209      StoreComplexToAddr(RT, CurFn->arg_begin(), false);
1210    } else if (CodeGenFunction::hasAggregateLLVMType(RetTy)) {
1211      // Do nothing; aggregrates get evaluated directly into the destination.
1212    } else {
1213      EmitStoreOfScalar(Builder.CreateLoad(ReturnValue), CurFn->arg_begin(),
1214                        false, Alignment, RetTy);
1215    }
1216    break;
1217  }
1218
1219  case ABIArgInfo::Extend:
1220  case ABIArgInfo::Direct:
1221    if (RetAI.getCoerceToType() == ConvertType(RetTy) &&
1222        RetAI.getDirectOffset() == 0) {
1223      // The internal return value temp always will have pointer-to-return-type
1224      // type, just do a load.
1225
1226      // If the instruction right before the insertion point is a store to the
1227      // return value, we can elide the load, zap the store, and usually zap the
1228      // alloca.
1229      llvm::BasicBlock *InsertBB = Builder.GetInsertBlock();
1230      llvm::StoreInst *SI = 0;
1231      if (InsertBB->empty() ||
1232          !(SI = dyn_cast<llvm::StoreInst>(&InsertBB->back())) ||
1233          SI->getPointerOperand() != ReturnValue || SI->isVolatile()) {
1234        RV = Builder.CreateLoad(ReturnValue);
1235      } else {
1236        // Get the stored value and nuke the now-dead store.
1237        RetDbgLoc = SI->getDebugLoc();
1238        RV = SI->getValueOperand();
1239        SI->eraseFromParent();
1240
1241        // If that was the only use of the return value, nuke it as well now.
1242        if (ReturnValue->use_empty() && isa<llvm::AllocaInst>(ReturnValue)) {
1243          cast<llvm::AllocaInst>(ReturnValue)->eraseFromParent();
1244          ReturnValue = 0;
1245        }
1246      }
1247    } else {
1248      llvm::Value *V = ReturnValue;
1249      // If the value is offset in memory, apply the offset now.
1250      if (unsigned Offs = RetAI.getDirectOffset()) {
1251        V = Builder.CreateBitCast(V, Builder.getInt8PtrTy());
1252        V = Builder.CreateConstGEP1_32(V, Offs);
1253        V = Builder.CreateBitCast(V,
1254                         llvm::PointerType::getUnqual(RetAI.getCoerceToType()));
1255      }
1256
1257      RV = CreateCoercedLoad(V, RetAI.getCoerceToType(), *this);
1258    }
1259
1260    // In ARC, end functions that return a retainable type with a call
1261    // to objc_autoreleaseReturnValue.
1262    if (AutoreleaseResult) {
1263      assert(getLangOptions().ObjCAutoRefCount &&
1264             !FI.isReturnsRetained() &&
1265             RetTy->isObjCRetainableType());
1266      RV = emitAutoreleaseOfResult(*this, RV);
1267    }
1268
1269    break;
1270
1271  case ABIArgInfo::Ignore:
1272    break;
1273
1274  case ABIArgInfo::Expand:
1275    llvm_unreachable("Invalid ABI kind for return argument");
1276  }
1277
1278  llvm::Instruction *Ret = RV ? Builder.CreateRet(RV) : Builder.CreateRetVoid();
1279  if (!RetDbgLoc.isUnknown())
1280    Ret->setDebugLoc(RetDbgLoc);
1281}
1282
1283void CodeGenFunction::EmitDelegateCallArg(CallArgList &args,
1284                                          const VarDecl *param) {
1285  // StartFunction converted the ABI-lowered parameter(s) into a
1286  // local alloca.  We need to turn that into an r-value suitable
1287  // for EmitCall.
1288  llvm::Value *local = GetAddrOfLocalVar(param);
1289
1290  QualType type = param->getType();
1291
1292  // For the most part, we just need to load the alloca, except:
1293  // 1) aggregate r-values are actually pointers to temporaries, and
1294  // 2) references to aggregates are pointers directly to the aggregate.
1295  // I don't know why references to non-aggregates are different here.
1296  if (const ReferenceType *ref = type->getAs<ReferenceType>()) {
1297    if (hasAggregateLLVMType(ref->getPointeeType()))
1298      return args.add(RValue::getAggregate(local), type);
1299
1300    // Locals which are references to scalars are represented
1301    // with allocas holding the pointer.
1302    return args.add(RValue::get(Builder.CreateLoad(local)), type);
1303  }
1304
1305  if (type->isAnyComplexType()) {
1306    ComplexPairTy complex = LoadComplexFromAddr(local, /*volatile*/ false);
1307    return args.add(RValue::getComplex(complex), type);
1308  }
1309
1310  if (hasAggregateLLVMType(type))
1311    return args.add(RValue::getAggregate(local), type);
1312
1313  unsigned alignment = getContext().getDeclAlign(param).getQuantity();
1314  llvm::Value *value = EmitLoadOfScalar(local, false, alignment, type);
1315  return args.add(RValue::get(value), type);
1316}
1317
1318static bool isProvablyNull(llvm::Value *addr) {
1319  return isa<llvm::ConstantPointerNull>(addr);
1320}
1321
1322static bool isProvablyNonNull(llvm::Value *addr) {
1323  return isa<llvm::AllocaInst>(addr);
1324}
1325
1326/// Emit the actual writing-back of a writeback.
1327static void emitWriteback(CodeGenFunction &CGF,
1328                          const CallArgList::Writeback &writeback) {
1329  llvm::Value *srcAddr = writeback.Address;
1330  assert(!isProvablyNull(srcAddr) &&
1331         "shouldn't have writeback for provably null argument");
1332
1333  llvm::BasicBlock *contBB = 0;
1334
1335  // If the argument wasn't provably non-null, we need to null check
1336  // before doing the store.
1337  bool provablyNonNull = isProvablyNonNull(srcAddr);
1338  if (!provablyNonNull) {
1339    llvm::BasicBlock *writebackBB = CGF.createBasicBlock("icr.writeback");
1340    contBB = CGF.createBasicBlock("icr.done");
1341
1342    llvm::Value *isNull = CGF.Builder.CreateIsNull(srcAddr, "icr.isnull");
1343    CGF.Builder.CreateCondBr(isNull, contBB, writebackBB);
1344    CGF.EmitBlock(writebackBB);
1345  }
1346
1347  // Load the value to writeback.
1348  llvm::Value *value = CGF.Builder.CreateLoad(writeback.Temporary);
1349
1350  // Cast it back, in case we're writing an id to a Foo* or something.
1351  value = CGF.Builder.CreateBitCast(value,
1352               cast<llvm::PointerType>(srcAddr->getType())->getElementType(),
1353                            "icr.writeback-cast");
1354
1355  // Perform the writeback.
1356  QualType srcAddrType = writeback.AddressType;
1357  CGF.EmitStoreThroughLValue(RValue::get(value),
1358                             CGF.MakeAddrLValue(srcAddr, srcAddrType));
1359
1360  // Jump to the continuation block.
1361  if (!provablyNonNull)
1362    CGF.EmitBlock(contBB);
1363}
1364
1365static void emitWritebacks(CodeGenFunction &CGF,
1366                           const CallArgList &args) {
1367  for (CallArgList::writeback_iterator
1368         i = args.writeback_begin(), e = args.writeback_end(); i != e; ++i)
1369    emitWriteback(CGF, *i);
1370}
1371
1372/// Emit an argument that's being passed call-by-writeback.  That is,
1373/// we are passing the address of
1374static void emitWritebackArg(CodeGenFunction &CGF, CallArgList &args,
1375                             const ObjCIndirectCopyRestoreExpr *CRE) {
1376  llvm::Value *srcAddr = CGF.EmitScalarExpr(CRE->getSubExpr());
1377
1378  // The dest and src types don't necessarily match in LLVM terms
1379  // because of the crazy ObjC compatibility rules.
1380
1381  llvm::PointerType *destType =
1382    cast<llvm::PointerType>(CGF.ConvertType(CRE->getType()));
1383
1384  // If the address is a constant null, just pass the appropriate null.
1385  if (isProvablyNull(srcAddr)) {
1386    args.add(RValue::get(llvm::ConstantPointerNull::get(destType)),
1387             CRE->getType());
1388    return;
1389  }
1390
1391  QualType srcAddrType =
1392    CRE->getSubExpr()->getType()->castAs<PointerType>()->getPointeeType();
1393
1394  // Create the temporary.
1395  llvm::Value *temp = CGF.CreateTempAlloca(destType->getElementType(),
1396                                           "icr.temp");
1397
1398  // Zero-initialize it if we're not doing a copy-initialization.
1399  bool shouldCopy = CRE->shouldCopy();
1400  if (!shouldCopy) {
1401    llvm::Value *null =
1402      llvm::ConstantPointerNull::get(
1403        cast<llvm::PointerType>(destType->getElementType()));
1404    CGF.Builder.CreateStore(null, temp);
1405  }
1406
1407  llvm::BasicBlock *contBB = 0;
1408
1409  // If the address is *not* known to be non-null, we need to switch.
1410  llvm::Value *finalArgument;
1411
1412  bool provablyNonNull = isProvablyNonNull(srcAddr);
1413  if (provablyNonNull) {
1414    finalArgument = temp;
1415  } else {
1416    llvm::Value *isNull = CGF.Builder.CreateIsNull(srcAddr, "icr.isnull");
1417
1418    finalArgument = CGF.Builder.CreateSelect(isNull,
1419                                   llvm::ConstantPointerNull::get(destType),
1420                                             temp, "icr.argument");
1421
1422    // If we need to copy, then the load has to be conditional, which
1423    // means we need control flow.
1424    if (shouldCopy) {
1425      contBB = CGF.createBasicBlock("icr.cont");
1426      llvm::BasicBlock *copyBB = CGF.createBasicBlock("icr.copy");
1427      CGF.Builder.CreateCondBr(isNull, contBB, copyBB);
1428      CGF.EmitBlock(copyBB);
1429    }
1430  }
1431
1432  // Perform a copy if necessary.
1433  if (shouldCopy) {
1434    LValue srcLV = CGF.MakeAddrLValue(srcAddr, srcAddrType);
1435    RValue srcRV = CGF.EmitLoadOfLValue(srcLV);
1436    assert(srcRV.isScalar());
1437
1438    llvm::Value *src = srcRV.getScalarVal();
1439    src = CGF.Builder.CreateBitCast(src, destType->getElementType(),
1440                                    "icr.cast");
1441
1442    // Use an ordinary store, not a store-to-lvalue.
1443    CGF.Builder.CreateStore(src, temp);
1444  }
1445
1446  // Finish the control flow if we needed it.
1447  if (shouldCopy && !provablyNonNull)
1448    CGF.EmitBlock(contBB);
1449
1450  args.addWriteback(srcAddr, srcAddrType, temp);
1451  args.add(RValue::get(finalArgument), CRE->getType());
1452}
1453
1454void CodeGenFunction::EmitCallArg(CallArgList &args, const Expr *E,
1455                                  QualType type) {
1456  if (const ObjCIndirectCopyRestoreExpr *CRE
1457        = dyn_cast<ObjCIndirectCopyRestoreExpr>(E)) {
1458    assert(getContext().getLangOptions().ObjCAutoRefCount);
1459    assert(getContext().hasSameType(E->getType(), type));
1460    return emitWritebackArg(*this, args, CRE);
1461  }
1462
1463  assert(type->isReferenceType() == E->isGLValue() &&
1464         "reference binding to unmaterialized r-value!");
1465
1466  if (E->isGLValue()) {
1467    assert(E->getObjectKind() == OK_Ordinary);
1468    return args.add(EmitReferenceBindingToExpr(E, /*InitializedDecl=*/0),
1469                    type);
1470  }
1471
1472  if (hasAggregateLLVMType(type) && !E->getType()->isAnyComplexType() &&
1473      isa<ImplicitCastExpr>(E) &&
1474      cast<CastExpr>(E)->getCastKind() == CK_LValueToRValue) {
1475    LValue L = EmitLValue(cast<CastExpr>(E)->getSubExpr());
1476    assert(L.isSimple());
1477    args.add(L.asAggregateRValue(), type, /*NeedsCopy*/true);
1478    return;
1479  }
1480
1481  args.add(EmitAnyExprToTemp(E), type);
1482}
1483
1484/// Emits a call or invoke instruction to the given function, depending
1485/// on the current state of the EH stack.
1486llvm::CallSite
1487CodeGenFunction::EmitCallOrInvoke(llvm::Value *Callee,
1488                                  ArrayRef<llvm::Value *> Args,
1489                                  const Twine &Name) {
1490  llvm::BasicBlock *InvokeDest = getInvokeDest();
1491  if (!InvokeDest)
1492    return Builder.CreateCall(Callee, Args, Name);
1493
1494  llvm::BasicBlock *ContBB = createBasicBlock("invoke.cont");
1495  llvm::InvokeInst *Invoke = Builder.CreateInvoke(Callee, ContBB, InvokeDest,
1496                                                  Args, Name);
1497  EmitBlock(ContBB);
1498  return Invoke;
1499}
1500
1501llvm::CallSite
1502CodeGenFunction::EmitCallOrInvoke(llvm::Value *Callee,
1503                                  const Twine &Name) {
1504  return EmitCallOrInvoke(Callee, ArrayRef<llvm::Value *>(), Name);
1505}
1506
1507static void checkArgMatches(llvm::Value *Elt, unsigned &ArgNo,
1508                            llvm::FunctionType *FTy) {
1509  if (ArgNo < FTy->getNumParams())
1510    assert(Elt->getType() == FTy->getParamType(ArgNo));
1511  else
1512    assert(FTy->isVarArg());
1513  ++ArgNo;
1514}
1515
1516void CodeGenFunction::ExpandTypeToArgs(QualType Ty, RValue RV,
1517                                       SmallVector<llvm::Value*,16> &Args,
1518                                       llvm::FunctionType *IRFuncTy) {
1519  if (const ConstantArrayType *AT = getContext().getAsConstantArrayType(Ty)) {
1520    unsigned NumElts = AT->getSize().getZExtValue();
1521    QualType EltTy = AT->getElementType();
1522    llvm::Value *Addr = RV.getAggregateAddr();
1523    for (unsigned Elt = 0; Elt < NumElts; ++Elt) {
1524      llvm::Value *EltAddr = Builder.CreateConstGEP2_32(Addr, 0, Elt);
1525      LValue LV = MakeAddrLValue(EltAddr, EltTy);
1526      RValue EltRV;
1527      if (EltTy->isAnyComplexType())
1528        // FIXME: Volatile?
1529        EltRV = RValue::getComplex(LoadComplexFromAddr(LV.getAddress(), false));
1530      else if (CodeGenFunction::hasAggregateLLVMType(EltTy))
1531        EltRV = LV.asAggregateRValue();
1532      else
1533        EltRV = EmitLoadOfLValue(LV);
1534      ExpandTypeToArgs(EltTy, EltRV, Args, IRFuncTy);
1535    }
1536  } else if (const RecordType *RT = Ty->getAsStructureType()) {
1537    RecordDecl *RD = RT->getDecl();
1538    assert(RV.isAggregate() && "Unexpected rvalue during struct expansion");
1539    llvm::Value *Addr = RV.getAggregateAddr();
1540    for (RecordDecl::field_iterator i = RD->field_begin(), e = RD->field_end();
1541         i != e; ++i) {
1542      FieldDecl *FD = *i;
1543      QualType FT = FD->getType();
1544
1545      // FIXME: What are the right qualifiers here?
1546      LValue LV = EmitLValueForField(Addr, FD, 0);
1547      RValue FldRV;
1548      if (FT->isAnyComplexType())
1549        // FIXME: Volatile?
1550        FldRV = RValue::getComplex(LoadComplexFromAddr(LV.getAddress(), false));
1551      else if (CodeGenFunction::hasAggregateLLVMType(FT))
1552        FldRV = LV.asAggregateRValue();
1553      else
1554        FldRV = EmitLoadOfLValue(LV);
1555      ExpandTypeToArgs(FT, FldRV, Args, IRFuncTy);
1556    }
1557  } else if (Ty->isAnyComplexType()) {
1558    ComplexPairTy CV = RV.getComplexVal();
1559    Args.push_back(CV.first);
1560    Args.push_back(CV.second);
1561  } else {
1562    assert(RV.isScalar() &&
1563           "Unexpected non-scalar rvalue during struct expansion.");
1564
1565    // Insert a bitcast as needed.
1566    llvm::Value *V = RV.getScalarVal();
1567    if (Args.size() < IRFuncTy->getNumParams() &&
1568        V->getType() != IRFuncTy->getParamType(Args.size()))
1569      V = Builder.CreateBitCast(V, IRFuncTy->getParamType(Args.size()));
1570
1571    Args.push_back(V);
1572  }
1573}
1574
1575
1576RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
1577                                 llvm::Value *Callee,
1578                                 ReturnValueSlot ReturnValue,
1579                                 const CallArgList &CallArgs,
1580                                 const Decl *TargetDecl,
1581                                 llvm::Instruction **callOrInvoke) {
1582  // FIXME: We no longer need the types from CallArgs; lift up and simplify.
1583  SmallVector<llvm::Value*, 16> Args;
1584
1585  // Handle struct-return functions by passing a pointer to the
1586  // location that we would like to return into.
1587  QualType RetTy = CallInfo.getReturnType();
1588  const ABIArgInfo &RetAI = CallInfo.getReturnInfo();
1589
1590  // IRArgNo - Keep track of the argument number in the callee we're looking at.
1591  unsigned IRArgNo = 0;
1592  llvm::FunctionType *IRFuncTy =
1593    cast<llvm::FunctionType>(
1594                  cast<llvm::PointerType>(Callee->getType())->getElementType());
1595
1596  // If the call returns a temporary with struct return, create a temporary
1597  // alloca to hold the result, unless one is given to us.
1598  if (CGM.ReturnTypeUsesSRet(CallInfo)) {
1599    llvm::Value *Value = ReturnValue.getValue();
1600    if (!Value)
1601      Value = CreateMemTemp(RetTy);
1602    Args.push_back(Value);
1603    checkArgMatches(Value, IRArgNo, IRFuncTy);
1604  }
1605
1606  assert(CallInfo.arg_size() == CallArgs.size() &&
1607         "Mismatch between function signature & arguments.");
1608  CGFunctionInfo::const_arg_iterator info_it = CallInfo.arg_begin();
1609  for (CallArgList::const_iterator I = CallArgs.begin(), E = CallArgs.end();
1610       I != E; ++I, ++info_it) {
1611    const ABIArgInfo &ArgInfo = info_it->info;
1612    RValue RV = I->RV;
1613
1614    unsigned TypeAlign =
1615      getContext().getTypeAlignInChars(I->Ty).getQuantity();
1616    switch (ArgInfo.getKind()) {
1617    case ABIArgInfo::Indirect: {
1618      if (RV.isScalar() || RV.isComplex()) {
1619        // Make a temporary alloca to pass the argument.
1620        llvm::AllocaInst *AI = CreateMemTemp(I->Ty);
1621        if (ArgInfo.getIndirectAlign() > AI->getAlignment())
1622          AI->setAlignment(ArgInfo.getIndirectAlign());
1623        Args.push_back(AI);
1624
1625        if (RV.isScalar())
1626          EmitStoreOfScalar(RV.getScalarVal(), Args.back(), false,
1627                            TypeAlign, I->Ty);
1628        else
1629          StoreComplexToAddr(RV.getComplexVal(), Args.back(), false);
1630
1631        // Validate argument match.
1632        checkArgMatches(AI, IRArgNo, IRFuncTy);
1633      } else {
1634        // We want to avoid creating an unnecessary temporary+copy here;
1635        // however, we need one in two cases:
1636        // 1. If the argument is not byval, and we are required to copy the
1637        //    source.  (This case doesn't occur on any common architecture.)
1638        // 2. If the argument is byval, RV is not sufficiently aligned, and
1639        //    we cannot force it to be sufficiently aligned.
1640        llvm::Value *Addr = RV.getAggregateAddr();
1641        unsigned Align = ArgInfo.getIndirectAlign();
1642        const llvm::TargetData *TD = &CGM.getTargetData();
1643        if ((!ArgInfo.getIndirectByVal() && I->NeedsCopy) ||
1644            (ArgInfo.getIndirectByVal() && TypeAlign < Align &&
1645             llvm::getOrEnforceKnownAlignment(Addr, Align, TD) < Align)) {
1646          // Create an aligned temporary, and copy to it.
1647          llvm::AllocaInst *AI = CreateMemTemp(I->Ty);
1648          if (Align > AI->getAlignment())
1649            AI->setAlignment(Align);
1650          Args.push_back(AI);
1651          EmitAggregateCopy(AI, Addr, I->Ty, RV.isVolatileQualified());
1652
1653          // Validate argument match.
1654          checkArgMatches(AI, IRArgNo, IRFuncTy);
1655        } else {
1656          // Skip the extra memcpy call.
1657          Args.push_back(Addr);
1658
1659          // Validate argument match.
1660          checkArgMatches(Addr, IRArgNo, IRFuncTy);
1661        }
1662      }
1663      break;
1664    }
1665
1666    case ABIArgInfo::Ignore:
1667      break;
1668
1669    case ABIArgInfo::Extend:
1670    case ABIArgInfo::Direct: {
1671      // Insert a padding argument to ensure proper alignment.
1672      if (llvm::Type *PaddingType = ArgInfo.getPaddingType()) {
1673        Args.push_back(llvm::UndefValue::get(PaddingType));
1674        ++IRArgNo;
1675      }
1676
1677      if (!isa<llvm::StructType>(ArgInfo.getCoerceToType()) &&
1678          ArgInfo.getCoerceToType() == ConvertType(info_it->type) &&
1679          ArgInfo.getDirectOffset() == 0) {
1680        llvm::Value *V;
1681        if (RV.isScalar())
1682          V = RV.getScalarVal();
1683        else
1684          V = Builder.CreateLoad(RV.getAggregateAddr());
1685
1686        // If the argument doesn't match, perform a bitcast to coerce it.  This
1687        // can happen due to trivial type mismatches.
1688        if (IRArgNo < IRFuncTy->getNumParams() &&
1689            V->getType() != IRFuncTy->getParamType(IRArgNo))
1690          V = Builder.CreateBitCast(V, IRFuncTy->getParamType(IRArgNo));
1691        Args.push_back(V);
1692
1693        checkArgMatches(V, IRArgNo, IRFuncTy);
1694        break;
1695      }
1696
1697      // FIXME: Avoid the conversion through memory if possible.
1698      llvm::Value *SrcPtr;
1699      if (RV.isScalar()) {
1700        SrcPtr = CreateMemTemp(I->Ty, "coerce");
1701        EmitStoreOfScalar(RV.getScalarVal(), SrcPtr, false, TypeAlign, I->Ty);
1702      } else if (RV.isComplex()) {
1703        SrcPtr = CreateMemTemp(I->Ty, "coerce");
1704        StoreComplexToAddr(RV.getComplexVal(), SrcPtr, false);
1705      } else
1706        SrcPtr = RV.getAggregateAddr();
1707
1708      // If the value is offset in memory, apply the offset now.
1709      if (unsigned Offs = ArgInfo.getDirectOffset()) {
1710        SrcPtr = Builder.CreateBitCast(SrcPtr, Builder.getInt8PtrTy());
1711        SrcPtr = Builder.CreateConstGEP1_32(SrcPtr, Offs);
1712        SrcPtr = Builder.CreateBitCast(SrcPtr,
1713                       llvm::PointerType::getUnqual(ArgInfo.getCoerceToType()));
1714
1715      }
1716
1717      // If the coerce-to type is a first class aggregate, we flatten it and
1718      // pass the elements. Either way is semantically identical, but fast-isel
1719      // and the optimizer generally likes scalar values better than FCAs.
1720      if (llvm::StructType *STy =
1721            dyn_cast<llvm::StructType>(ArgInfo.getCoerceToType())) {
1722        SrcPtr = Builder.CreateBitCast(SrcPtr,
1723                                       llvm::PointerType::getUnqual(STy));
1724        for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) {
1725          llvm::Value *EltPtr = Builder.CreateConstGEP2_32(SrcPtr, 0, i);
1726          llvm::LoadInst *LI = Builder.CreateLoad(EltPtr);
1727          // We don't know what we're loading from.
1728          LI->setAlignment(1);
1729          Args.push_back(LI);
1730
1731          // Validate argument match.
1732          checkArgMatches(LI, IRArgNo, IRFuncTy);
1733        }
1734      } else {
1735        // In the simple case, just pass the coerced loaded value.
1736        Args.push_back(CreateCoercedLoad(SrcPtr, ArgInfo.getCoerceToType(),
1737                                         *this));
1738
1739        // Validate argument match.
1740        checkArgMatches(Args.back(), IRArgNo, IRFuncTy);
1741      }
1742
1743      break;
1744    }
1745
1746    case ABIArgInfo::Expand:
1747      ExpandTypeToArgs(I->Ty, RV, Args, IRFuncTy);
1748      IRArgNo = Args.size();
1749      break;
1750    }
1751  }
1752
1753  // If the callee is a bitcast of a function to a varargs pointer to function
1754  // type, check to see if we can remove the bitcast.  This handles some cases
1755  // with unprototyped functions.
1756  if (llvm::ConstantExpr *CE = dyn_cast<llvm::ConstantExpr>(Callee))
1757    if (llvm::Function *CalleeF = dyn_cast<llvm::Function>(CE->getOperand(0))) {
1758      llvm::PointerType *CurPT=cast<llvm::PointerType>(Callee->getType());
1759      llvm::FunctionType *CurFT =
1760        cast<llvm::FunctionType>(CurPT->getElementType());
1761      llvm::FunctionType *ActualFT = CalleeF->getFunctionType();
1762
1763      if (CE->getOpcode() == llvm::Instruction::BitCast &&
1764          ActualFT->getReturnType() == CurFT->getReturnType() &&
1765          ActualFT->getNumParams() == CurFT->getNumParams() &&
1766          ActualFT->getNumParams() == Args.size() &&
1767          (CurFT->isVarArg() || !ActualFT->isVarArg())) {
1768        bool ArgsMatch = true;
1769        for (unsigned i = 0, e = ActualFT->getNumParams(); i != e; ++i)
1770          if (ActualFT->getParamType(i) != CurFT->getParamType(i)) {
1771            ArgsMatch = false;
1772            break;
1773          }
1774
1775        // Strip the cast if we can get away with it.  This is a nice cleanup,
1776        // but also allows us to inline the function at -O0 if it is marked
1777        // always_inline.
1778        if (ArgsMatch)
1779          Callee = CalleeF;
1780      }
1781    }
1782
1783  unsigned CallingConv;
1784  CodeGen::AttributeListType AttributeList;
1785  CGM.ConstructAttributeList(CallInfo, TargetDecl, AttributeList, CallingConv);
1786  llvm::AttrListPtr Attrs = llvm::AttrListPtr::get(AttributeList.begin(),
1787                                                   AttributeList.end());
1788
1789  llvm::BasicBlock *InvokeDest = 0;
1790  if (!(Attrs.getFnAttributes() & llvm::Attribute::NoUnwind))
1791    InvokeDest = getInvokeDest();
1792
1793  llvm::CallSite CS;
1794  if (!InvokeDest) {
1795    CS = Builder.CreateCall(Callee, Args);
1796  } else {
1797    llvm::BasicBlock *Cont = createBasicBlock("invoke.cont");
1798    CS = Builder.CreateInvoke(Callee, Cont, InvokeDest, Args);
1799    EmitBlock(Cont);
1800  }
1801  if (callOrInvoke)
1802    *callOrInvoke = CS.getInstruction();
1803
1804  CS.setAttributes(Attrs);
1805  CS.setCallingConv(static_cast<llvm::CallingConv::ID>(CallingConv));
1806
1807  // If the call doesn't return, finish the basic block and clear the
1808  // insertion point; this allows the rest of IRgen to discard
1809  // unreachable code.
1810  if (CS.doesNotReturn()) {
1811    Builder.CreateUnreachable();
1812    Builder.ClearInsertionPoint();
1813
1814    // FIXME: For now, emit a dummy basic block because expr emitters in
1815    // generally are not ready to handle emitting expressions at unreachable
1816    // points.
1817    EnsureInsertPoint();
1818
1819    // Return a reasonable RValue.
1820    return GetUndefRValue(RetTy);
1821  }
1822
1823  llvm::Instruction *CI = CS.getInstruction();
1824  if (Builder.isNamePreserving() && !CI->getType()->isVoidTy())
1825    CI->setName("call");
1826
1827  // Emit any writebacks immediately.  Arguably this should happen
1828  // after any return-value munging.
1829  if (CallArgs.hasWritebacks())
1830    emitWritebacks(*this, CallArgs);
1831
1832  switch (RetAI.getKind()) {
1833  case ABIArgInfo::Indirect: {
1834    unsigned Alignment = getContext().getTypeAlignInChars(RetTy).getQuantity();
1835    if (RetTy->isAnyComplexType())
1836      return RValue::getComplex(LoadComplexFromAddr(Args[0], false));
1837    if (CodeGenFunction::hasAggregateLLVMType(RetTy))
1838      return RValue::getAggregate(Args[0]);
1839    return RValue::get(EmitLoadOfScalar(Args[0], false, Alignment, RetTy));
1840  }
1841
1842  case ABIArgInfo::Ignore:
1843    // If we are ignoring an argument that had a result, make sure to
1844    // construct the appropriate return value for our caller.
1845    return GetUndefRValue(RetTy);
1846
1847  case ABIArgInfo::Extend:
1848  case ABIArgInfo::Direct: {
1849    llvm::Type *RetIRTy = ConvertType(RetTy);
1850    if (RetAI.getCoerceToType() == RetIRTy && RetAI.getDirectOffset() == 0) {
1851      if (RetTy->isAnyComplexType()) {
1852        llvm::Value *Real = Builder.CreateExtractValue(CI, 0);
1853        llvm::Value *Imag = Builder.CreateExtractValue(CI, 1);
1854        return RValue::getComplex(std::make_pair(Real, Imag));
1855      }
1856      if (CodeGenFunction::hasAggregateLLVMType(RetTy)) {
1857        llvm::Value *DestPtr = ReturnValue.getValue();
1858        bool DestIsVolatile = ReturnValue.isVolatile();
1859
1860        if (!DestPtr) {
1861          DestPtr = CreateMemTemp(RetTy, "agg.tmp");
1862          DestIsVolatile = false;
1863        }
1864        BuildAggStore(*this, CI, DestPtr, DestIsVolatile, false);
1865        return RValue::getAggregate(DestPtr);
1866      }
1867
1868      // If the argument doesn't match, perform a bitcast to coerce it.  This
1869      // can happen due to trivial type mismatches.
1870      llvm::Value *V = CI;
1871      if (V->getType() != RetIRTy)
1872        V = Builder.CreateBitCast(V, RetIRTy);
1873      return RValue::get(V);
1874    }
1875
1876    llvm::Value *DestPtr = ReturnValue.getValue();
1877    bool DestIsVolatile = ReturnValue.isVolatile();
1878
1879    if (!DestPtr) {
1880      DestPtr = CreateMemTemp(RetTy, "coerce");
1881      DestIsVolatile = false;
1882    }
1883
1884    // If the value is offset in memory, apply the offset now.
1885    llvm::Value *StorePtr = DestPtr;
1886    if (unsigned Offs = RetAI.getDirectOffset()) {
1887      StorePtr = Builder.CreateBitCast(StorePtr, Builder.getInt8PtrTy());
1888      StorePtr = Builder.CreateConstGEP1_32(StorePtr, Offs);
1889      StorePtr = Builder.CreateBitCast(StorePtr,
1890                         llvm::PointerType::getUnqual(RetAI.getCoerceToType()));
1891    }
1892    CreateCoercedStore(CI, StorePtr, DestIsVolatile, *this);
1893
1894    unsigned Alignment = getContext().getTypeAlignInChars(RetTy).getQuantity();
1895    if (RetTy->isAnyComplexType())
1896      return RValue::getComplex(LoadComplexFromAddr(DestPtr, false));
1897    if (CodeGenFunction::hasAggregateLLVMType(RetTy))
1898      return RValue::getAggregate(DestPtr);
1899    return RValue::get(EmitLoadOfScalar(DestPtr, false, Alignment, RetTy));
1900  }
1901
1902  case ABIArgInfo::Expand:
1903    llvm_unreachable("Invalid ABI kind for return argument");
1904  }
1905
1906  llvm_unreachable("Unhandled ABIArgInfo::Kind");
1907}
1908
1909/* VarArg handling */
1910
1911llvm::Value *CodeGenFunction::EmitVAArg(llvm::Value *VAListAddr, QualType Ty) {
1912  return CGM.getTypes().getABIInfo().EmitVAArg(VAListAddr, Ty, *this);
1913}
1914