CGCall.cpp revision 1bb94a417a53a524784ec98f232acc70e62370b2 
1//===--- CGCall.cpp - Encapsulate calling convention details ----*- C++ -*-===//
2//
3//                     The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10// These classes wrap the information about a call or function
11// definition used to handle ABI compliancy.
12//
13//===----------------------------------------------------------------------===//
14
15#include "CGCall.h"
16#include "CGCXXABI.h"
17#include "ABIInfo.h"
18#include "CodeGenFunction.h"
19#include "CodeGenModule.h"
20#include "clang/Basic/TargetInfo.h"
21#include "clang/AST/Decl.h"
22#include "clang/AST/DeclCXX.h"
23#include "clang/AST/DeclObjC.h"
24#include "clang/Frontend/CodeGenOptions.h"
25#include "llvm/Attributes.h"
26#include "llvm/Support/CallSite.h"
27#include "llvm/Target/TargetData.h"
28#include "llvm/InlineAsm.h"
29#include "llvm/Transforms/Utils/Local.h"
30using namespace clang;
31using namespace CodeGen;
32
33/***/
34
35static unsigned ClangCallConvToLLVMCallConv(CallingConv CC) {
36  switch (CC) {
37  default: return llvm::CallingConv::C;
38  case CC_X86StdCall: return llvm::CallingConv::X86_StdCall;
39  case CC_X86FastCall: return llvm::CallingConv::X86_FastCall;
40  case CC_X86ThisCall: return llvm::CallingConv::X86_ThisCall;
41  case CC_AAPCS: return llvm::CallingConv::ARM_AAPCS;
42  case CC_AAPCS_VFP: return llvm::CallingConv::ARM_AAPCS_VFP;
43  // TODO: add support for CC_X86Pascal to llvm
44  }
45}
46
47/// Derives the 'this' type for codegen purposes, i.e. ignoring method
48/// qualification.
49/// FIXME: address space qualification?
50static CanQualType GetThisType(ASTContext &Context, const CXXRecordDecl *RD) {
51  QualType RecTy = Context.getTagDeclType(RD)->getCanonicalTypeInternal();
52  return Context.getPointerType(CanQualType::CreateUnsafe(RecTy));
53}
54
55/// Returns the canonical formal type of the given C++ method.
56static CanQual<FunctionProtoType> GetFormalType(const CXXMethodDecl *MD) {
57  return MD->getType()->getCanonicalTypeUnqualified()
58           .getAs<FunctionProtoType>();
59}
60
61/// Returns the "extra-canonicalized" return type, which discards
62/// qualifiers on the return type.  Codegen doesn't care about them,
63/// and it makes ABI code a little easier to be able to assume that
64/// all parameter and return types are top-level unqualified.
65static CanQualType GetReturnType(QualType RetTy) {
66  return RetTy->getCanonicalTypeUnqualified().getUnqualifiedType();
67}
68
69const CGFunctionInfo &
70CodeGenTypes::getFunctionInfo(CanQual<FunctionNoProtoType> FTNP) {
71  return getFunctionInfo(FTNP->getResultType().getUnqualifiedType(),
72                         SmallVector<CanQualType, 16>(),
73                         FTNP->getExtInfo());
74}
75
76/// \param Args - contains any initial parameters besides those
77///   in the formal type
78static const CGFunctionInfo &getFunctionInfo(CodeGenTypes &CGT,
79                                  SmallVectorImpl<CanQualType> &ArgTys,
80                                             CanQual<FunctionProtoType> FTP) {
81  // FIXME: Kill copy.
82  for (unsigned i = 0, e = FTP->getNumArgs(); i != e; ++i)
83    ArgTys.push_back(FTP->getArgType(i));
84  CanQualType ResTy = FTP->getResultType().getUnqualifiedType();
85  return CGT.getFunctionInfo(ResTy, ArgTys, FTP->getExtInfo());
86}
87
88const CGFunctionInfo &
89CodeGenTypes::getFunctionInfo(CanQual<FunctionProtoType> FTP) {
90  SmallVector<CanQualType, 16> ArgTys;
91  return ::getFunctionInfo(*this, ArgTys, FTP);
92}
93
94static CallingConv getCallingConventionForDecl(const Decl *D) {
95  // Set the appropriate calling convention for the Function.
96  if (D->hasAttr<StdCallAttr>())
97    return CC_X86StdCall;
98
99  if (D->hasAttr<FastCallAttr>())
100    return CC_X86FastCall;
101
102  if (D->hasAttr<ThisCallAttr>())
103    return CC_X86ThisCall;
104
105  if (D->hasAttr<PascalAttr>())
106    return CC_X86Pascal;
107
108  if (PcsAttr *PCS = D->getAttr<PcsAttr>())
109    return (PCS->getPCS() == PcsAttr::AAPCS ? CC_AAPCS : CC_AAPCS_VFP);
110
111  return CC_C;
112}
113
114const CGFunctionInfo &CodeGenTypes::getFunctionInfo(const CXXRecordDecl *RD,
115                                                 const FunctionProtoType *FTP) {
116  SmallVector<CanQualType, 16> ArgTys;
117
118  // Add the 'this' pointer.
119  ArgTys.push_back(GetThisType(Context, RD));
120
121  return ::getFunctionInfo(*this, ArgTys,
122              FTP->getCanonicalTypeUnqualified().getAs<FunctionProtoType>());
123}
124
125const CGFunctionInfo &CodeGenTypes::getFunctionInfo(const CXXMethodDecl *MD) {
126  SmallVector<CanQualType, 16> ArgTys;
127
128  assert(!isa<CXXConstructorDecl>(MD) && "wrong method for contructors!");
129  assert(!isa<CXXDestructorDecl>(MD) && "wrong method for destructors!");
130
131  // Add the 'this' pointer unless this is a static method.
132  if (MD->isInstance())
133    ArgTys.push_back(GetThisType(Context, MD->getParent()));
134
135  return ::getFunctionInfo(*this, ArgTys, GetFormalType(MD));
136}
137
138const CGFunctionInfo &CodeGenTypes::getFunctionInfo(const CXXConstructorDecl *D,
139                                                    CXXCtorType Type) {
140  SmallVector<CanQualType, 16> ArgTys;
141  ArgTys.push_back(GetThisType(Context, D->getParent()));
142  CanQualType ResTy = Context.VoidTy;
143
144  TheCXXABI.BuildConstructorSignature(D, Type, ResTy, ArgTys);
145
146  CanQual<FunctionProtoType> FTP = GetFormalType(D);
147
148  // Add the formal parameters.
149  for (unsigned i = 0, e = FTP->getNumArgs(); i != e; ++i)
150    ArgTys.push_back(FTP->getArgType(i));
151
152  return getFunctionInfo(ResTy, ArgTys, FTP->getExtInfo());
153}
154
155const CGFunctionInfo &CodeGenTypes::getFunctionInfo(const CXXDestructorDecl *D,
156                                                    CXXDtorType Type) {
157  SmallVector<CanQualType, 2> ArgTys;
158  ArgTys.push_back(GetThisType(Context, D->getParent()));
159  CanQualType ResTy = Context.VoidTy;
160
161  TheCXXABI.BuildDestructorSignature(D, Type, ResTy, ArgTys);
162
163  CanQual<FunctionProtoType> FTP = GetFormalType(D);
164  assert(FTP->getNumArgs() == 0 && "dtor with formal parameters");
165
166  return getFunctionInfo(ResTy, ArgTys, FTP->getExtInfo());
167}
168
169const CGFunctionInfo &CodeGenTypes::getFunctionInfo(const FunctionDecl *FD) {
170  if (const CXXMethodDecl *MD = dyn_cast<CXXMethodDecl>(FD))
171    if (MD->isInstance())
172      return getFunctionInfo(MD);
173
174  CanQualType FTy = FD->getType()->getCanonicalTypeUnqualified();
175  assert(isa<FunctionType>(FTy));
176  if (isa<FunctionNoProtoType>(FTy))
177    return getFunctionInfo(FTy.getAs<FunctionNoProtoType>());
178  assert(isa<FunctionProtoType>(FTy));
179  return getFunctionInfo(FTy.getAs<FunctionProtoType>());
180}
181
182const CGFunctionInfo &CodeGenTypes::getFunctionInfo(const ObjCMethodDecl *MD) {
183  SmallVector<CanQualType, 16> ArgTys;
184  ArgTys.push_back(Context.getCanonicalParamType(MD->getSelfDecl()->getType()));
185  ArgTys.push_back(Context.getCanonicalParamType(Context.getObjCSelType()));
186  // FIXME: Kill copy?
187  for (ObjCMethodDecl::param_const_iterator i = MD->param_begin(),
188         e = MD->param_end(); i != e; ++i) {
189    ArgTys.push_back(Context.getCanonicalParamType((*i)->getType()));
190  }
191
192  FunctionType::ExtInfo einfo;
193  einfo = einfo.withCallingConv(getCallingConventionForDecl(MD));
194
195  if (getContext().getLangOptions().ObjCAutoRefCount &&
196      MD->hasAttr<NSReturnsRetainedAttr>())
197    einfo = einfo.withProducesResult(true);
198
199  return getFunctionInfo(GetReturnType(MD->getResultType()), ArgTys, einfo);
200}
201
202const CGFunctionInfo &CodeGenTypes::getFunctionInfo(GlobalDecl GD) {
203  // FIXME: Do we need to handle ObjCMethodDecl?
204  const FunctionDecl *FD = cast<FunctionDecl>(GD.getDecl());
205
206  if (const CXXConstructorDecl *CD = dyn_cast<CXXConstructorDecl>(FD))
207    return getFunctionInfo(CD, GD.getCtorType());
208
209  if (const CXXDestructorDecl *DD = dyn_cast<CXXDestructorDecl>(FD))
210    return getFunctionInfo(DD, GD.getDtorType());
211
212  return getFunctionInfo(FD);
213}
214
215const CGFunctionInfo &CodeGenTypes::getFunctionInfo(QualType ResTy,
216                                                    const CallArgList &Args,
217                                            const FunctionType::ExtInfo &Info) {
218  // FIXME: Kill copy.
219  SmallVector<CanQualType, 16> ArgTys;
220  for (CallArgList::const_iterator i = Args.begin(), e = Args.end();
221       i != e; ++i)
222    ArgTys.push_back(Context.getCanonicalParamType(i->Ty));
223  return getFunctionInfo(GetReturnType(ResTy), ArgTys, Info);
224}
225
226const CGFunctionInfo &CodeGenTypes::getFunctionInfo(QualType ResTy,
227                                                    const FunctionArgList &Args,
228                                            const FunctionType::ExtInfo &Info) {
229  // FIXME: Kill copy.
230  SmallVector<CanQualType, 16> ArgTys;
231  for (FunctionArgList::const_iterator i = Args.begin(), e = Args.end();
232       i != e; ++i)
233    ArgTys.push_back(Context.getCanonicalParamType((*i)->getType()));
234  return getFunctionInfo(GetReturnType(ResTy), ArgTys, Info);
235}
236
237const CGFunctionInfo &CodeGenTypes::getNullaryFunctionInfo() {
238  SmallVector<CanQualType, 1> args;
239  return getFunctionInfo(getContext().VoidTy, args, FunctionType::ExtInfo());
240}
241
242const CGFunctionInfo &CodeGenTypes::getFunctionInfo(CanQualType ResTy,
243                           const SmallVectorImpl<CanQualType> &ArgTys,
244                                            const FunctionType::ExtInfo &Info) {
245#ifndef NDEBUG
246  for (SmallVectorImpl<CanQualType>::const_iterator
247         I = ArgTys.begin(), E = ArgTys.end(); I != E; ++I)
248    assert(I->isCanonicalAsParam());
249#endif
250
251  unsigned CC = ClangCallConvToLLVMCallConv(Info.getCC());
252
253  // Lookup or create unique function info.
254  llvm::FoldingSetNodeID ID;
255  CGFunctionInfo::Profile(ID, Info, ResTy, ArgTys.begin(), ArgTys.end());
256
257  void *InsertPos = 0;
258  CGFunctionInfo *FI = FunctionInfos.FindNodeOrInsertPos(ID, InsertPos);
259  if (FI)
260    return *FI;
261
262  // Construct the function info.
263  FI = new CGFunctionInfo(CC, Info.getNoReturn(), Info.getProducesResult(),
264                          Info.getHasRegParm(), Info.getRegParm(), ResTy,
265                          ArgTys.data(), ArgTys.size());
266  FunctionInfos.InsertNode(FI, InsertPos);
267
268  bool Inserted = FunctionsBeingProcessed.insert(FI); (void)Inserted;
269  assert(Inserted && "Recursively being processed?");
270
271  // Compute ABI information.
272  getABIInfo().computeInfo(*FI);
273
274  // Loop over all of the computed argument and return value info.  If any of
275  // them are direct or extend without a specified coerce type, specify the
276  // default now.
277  ABIArgInfo &RetInfo = FI->getReturnInfo();
278  if (RetInfo.canHaveCoerceToType() && RetInfo.getCoerceToType() == 0)
279    RetInfo.setCoerceToType(ConvertType(FI->getReturnType()));
280
281  for (CGFunctionInfo::arg_iterator I = FI->arg_begin(), E = FI->arg_end();
282       I != E; ++I)
283    if (I->info.canHaveCoerceToType() && I->info.getCoerceToType() == 0)
284      I->info.setCoerceToType(ConvertType(I->type));
285
286  bool Erased = FunctionsBeingProcessed.erase(FI); (void)Erased;
287  assert(Erased && "Not in set?");
288
289  return *FI;
290}
291
292CGFunctionInfo::CGFunctionInfo(unsigned _CallingConvention,
293                               bool _NoReturn, bool returnsRetained,
294                               bool _HasRegParm, unsigned _RegParm,
295                               CanQualType ResTy,
296                               const CanQualType *ArgTys,
297                               unsigned NumArgTys)
298  : CallingConvention(_CallingConvention),
299    EffectiveCallingConvention(_CallingConvention),
300    NoReturn(_NoReturn), ReturnsRetained(returnsRetained),
301    HasRegParm(_HasRegParm), RegParm(_RegParm)
302{
303  NumArgs = NumArgTys;
304
305  // FIXME: Coallocate with the CGFunctionInfo object.
306  Args = new ArgInfo[1 + NumArgTys];
307  Args[0].type = ResTy;
308  for (unsigned i = 0; i != NumArgTys; ++i)
309    Args[1 + i].type = ArgTys[i];
310}
311
312/***/
313
314void CodeGenTypes::GetExpandedTypes(QualType type,
315                     SmallVectorImpl<llvm::Type*> &expandedTypes) {
316  if (const ConstantArrayType *AT = Context.getAsConstantArrayType(type)) {
317    uint64_t NumElts = AT->getSize().getZExtValue();
318    for (uint64_t Elt = 0; Elt < NumElts; ++Elt)
319      GetExpandedTypes(AT->getElementType(), expandedTypes);
320  } else if (const RecordType *RT = type->getAsStructureType()) {
321    const RecordDecl *RD = RT->getDecl();
322    assert(!RD->hasFlexibleArrayMember() &&
323           "Cannot expand structure with flexible array.");
324    for (RecordDecl::field_iterator i = RD->field_begin(), e = RD->field_end();
325         i != e; ++i) {
326      const FieldDecl *FD = *i;
327      assert(!FD->isBitField() &&
328             "Cannot expand structure with bit-field members.");
329      GetExpandedTypes(FD->getType(), expandedTypes);
330    }
331  } else if (const ComplexType *CT = type->getAs<ComplexType>()) {
332    llvm::Type *EltTy = ConvertType(CT->getElementType());
333    expandedTypes.push_back(EltTy);
334    expandedTypes.push_back(EltTy);
335  } else
336    expandedTypes.push_back(ConvertType(type));
337}
338
339llvm::Function::arg_iterator
340CodeGenFunction::ExpandTypeFromArgs(QualType Ty, LValue LV,
341                                    llvm::Function::arg_iterator AI) {
342  assert(LV.isSimple() &&
343         "Unexpected non-simple lvalue during struct expansion.");
344  llvm::Value *Addr = LV.getAddress();
345
346  if (const ConstantArrayType *AT = getContext().getAsConstantArrayType(Ty)) {
347    unsigned NumElts = AT->getSize().getZExtValue();
348    QualType EltTy = AT->getElementType();
349    for (unsigned Elt = 0; Elt < NumElts; ++Elt) {
350      llvm::Value *EltAddr = Builder.CreateConstGEP2_32(Addr, 0, Elt);
351      LValue LV = MakeAddrLValue(EltAddr, EltTy);
352      AI = ExpandTypeFromArgs(EltTy, LV, AI);
353    }
354  } else if (const RecordType *RT = Ty->getAsStructureType()) {
355    RecordDecl *RD = RT->getDecl();
356    for (RecordDecl::field_iterator i = RD->field_begin(), e = RD->field_end();
357         i != e; ++i) {
358      FieldDecl *FD = *i;
359      QualType FT = FD->getType();
360
361      // FIXME: What are the right qualifiers here?
362      LValue LV = EmitLValueForField(Addr, FD, 0);
363      AI = ExpandTypeFromArgs(FT, LV, AI);
364    }
365  } else if (const ComplexType *CT = Ty->getAs<ComplexType>()) {
366    QualType EltTy = CT->getElementType();
367    llvm::Value *RealAddr = Builder.CreateStructGEP(Addr, 0, "real");
368    EmitStoreThroughLValue(RValue::get(AI++), MakeAddrLValue(RealAddr, EltTy));
369    llvm::Value *ImagAddr = Builder.CreateStructGEP(Addr, 1, "imag");
370    EmitStoreThroughLValue(RValue::get(AI++), MakeAddrLValue(ImagAddr, EltTy));
371  } else {
372    EmitStoreThroughLValue(RValue::get(AI), LV);
373    ++AI;
374  }
375
376  return AI;
377}
378
379/// EnterStructPointerForCoercedAccess - Given a struct pointer that we are
380/// accessing some number of bytes out of it, try to gep into the struct to get
381/// at its inner goodness.  Dive as deep as possible without entering an element
382/// with an in-memory size smaller than DstSize.
383static llvm::Value *
384EnterStructPointerForCoercedAccess(llvm::Value *SrcPtr,
385                                   llvm::StructType *SrcSTy,
386                                   uint64_t DstSize, CodeGenFunction &CGF) {
387  // We can't dive into a zero-element struct.
388  if (SrcSTy->getNumElements() == 0) return SrcPtr;
389
390  llvm::Type *FirstElt = SrcSTy->getElementType(0);
391
392  // If the first elt is at least as large as what we're looking for, or if the
393  // first element is the same size as the whole struct, we can enter it.
394  uint64_t FirstEltSize =
395    CGF.CGM.getTargetData().getTypeAllocSize(FirstElt);
396  if (FirstEltSize < DstSize &&
397      FirstEltSize < CGF.CGM.getTargetData().getTypeAllocSize(SrcSTy))
398    return SrcPtr;
399
400  // GEP into the first element.
401  SrcPtr = CGF.Builder.CreateConstGEP2_32(SrcPtr, 0, 0, "coerce.dive");
402
403  // If the first element is a struct, recurse.
404  llvm::Type *SrcTy =
405    cast<llvm::PointerType>(SrcPtr->getType())->getElementType();
406  if (llvm::StructType *SrcSTy = dyn_cast<llvm::StructType>(SrcTy))
407    return EnterStructPointerForCoercedAccess(SrcPtr, SrcSTy, DstSize, CGF);
408
409  return SrcPtr;
410}
411
412/// CoerceIntOrPtrToIntOrPtr - Convert a value Val to the specific Ty where both
413/// are either integers or pointers.  This does a truncation of the value if it
414/// is too large or a zero extension if it is too small.
415static llvm::Value *CoerceIntOrPtrToIntOrPtr(llvm::Value *Val,
416                                             llvm::Type *Ty,
417                                             CodeGenFunction &CGF) {
418  if (Val->getType() == Ty)
419    return Val;
420
421  if (isa<llvm::PointerType>(Val->getType())) {
422    // If this is Pointer->Pointer avoid conversion to and from int.
423    if (isa<llvm::PointerType>(Ty))
424      return CGF.Builder.CreateBitCast(Val, Ty, "coerce.val");
425
426    // Convert the pointer to an integer so we can play with its width.
427    Val = CGF.Builder.CreatePtrToInt(Val, CGF.IntPtrTy, "coerce.val.pi");
428  }
429
430  llvm::Type *DestIntTy = Ty;
431  if (isa<llvm::PointerType>(DestIntTy))
432    DestIntTy = CGF.IntPtrTy;
433
434  if (Val->getType() != DestIntTy)
435    Val = CGF.Builder.CreateIntCast(Val, DestIntTy, false, "coerce.val.ii");
436
437  if (isa<llvm::PointerType>(Ty))
438    Val = CGF.Builder.CreateIntToPtr(Val, Ty, "coerce.val.ip");
439  return Val;
440}
441
442
443
444/// CreateCoercedLoad - Create a load from \arg SrcPtr interpreted as
445/// a pointer to an object of type \arg Ty.
446///
447/// This safely handles the case when the src type is smaller than the
448/// destination type; in this situation the values of bits which not
449/// present in the src are undefined.
450static llvm::Value *CreateCoercedLoad(llvm::Value *SrcPtr,
451                                      llvm::Type *Ty,
452                                      CodeGenFunction &CGF) {
453  llvm::Type *SrcTy =
454    cast<llvm::PointerType>(SrcPtr->getType())->getElementType();
455
456  // If SrcTy and Ty are the same, just do a load.
457  if (SrcTy == Ty)
458    return CGF.Builder.CreateLoad(SrcPtr);
459
460  uint64_t DstSize = CGF.CGM.getTargetData().getTypeAllocSize(Ty);
461
462  if (llvm::StructType *SrcSTy = dyn_cast<llvm::StructType>(SrcTy)) {
463    SrcPtr = EnterStructPointerForCoercedAccess(SrcPtr, SrcSTy, DstSize, CGF);
464    SrcTy = cast<llvm::PointerType>(SrcPtr->getType())->getElementType();
465  }
466
467  uint64_t SrcSize = CGF.CGM.getTargetData().getTypeAllocSize(SrcTy);
468
469  // If the source and destination are integer or pointer types, just do an
470  // extension or truncation to the desired type.
471  if ((isa<llvm::IntegerType>(Ty) || isa<llvm::PointerType>(Ty)) &&
472      (isa<llvm::IntegerType>(SrcTy) || isa<llvm::PointerType>(SrcTy))) {
473    llvm::LoadInst *Load = CGF.Builder.CreateLoad(SrcPtr);
474    return CoerceIntOrPtrToIntOrPtr(Load, Ty, CGF);
475  }
476
477  // If load is legal, just bitcast the src pointer.
478  if (SrcSize >= DstSize) {
479    // Generally SrcSize is never greater than DstSize, since this means we are
480    // losing bits. However, this can happen in cases where the structure has
481    // additional padding, for example due to a user specified alignment.
482    //
483    // FIXME: Assert that we aren't truncating non-padding bits when have access
484    // to that information.
485    llvm::Value *Casted =
486      CGF.Builder.CreateBitCast(SrcPtr, llvm::PointerType::getUnqual(Ty));
487    llvm::LoadInst *Load = CGF.Builder.CreateLoad(Casted);
488    // FIXME: Use better alignment / avoid requiring aligned load.
489    Load->setAlignment(1);
490    return Load;
491  }
492
493  // Otherwise do coercion through memory. This is stupid, but
494  // simple.
495  llvm::Value *Tmp = CGF.CreateTempAlloca(Ty);
496  llvm::Value *Casted =
497    CGF.Builder.CreateBitCast(Tmp, llvm::PointerType::getUnqual(SrcTy));
498  llvm::StoreInst *Store =
499    CGF.Builder.CreateStore(CGF.Builder.CreateLoad(SrcPtr), Casted);
500  // FIXME: Use better alignment / avoid requiring aligned store.
501  Store->setAlignment(1);
502  return CGF.Builder.CreateLoad(Tmp);
503}
504
505// Function to store a first-class aggregate into memory.  We prefer to
506// store the elements rather than the aggregate to be more friendly to
507// fast-isel.
508// FIXME: Do we need to recurse here?
509static void BuildAggStore(CodeGenFunction &CGF, llvm::Value *Val,
510                          llvm::Value *DestPtr, bool DestIsVolatile,
511                          bool LowAlignment) {
512  // Prefer scalar stores to first-class aggregate stores.
513  if (llvm::StructType *STy =
514        dyn_cast<llvm::StructType>(Val->getType())) {
515    for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) {
516      llvm::Value *EltPtr = CGF.Builder.CreateConstGEP2_32(DestPtr, 0, i);
517      llvm::Value *Elt = CGF.Builder.CreateExtractValue(Val, i);
518      llvm::StoreInst *SI = CGF.Builder.CreateStore(Elt, EltPtr,
519                                                    DestIsVolatile);
520      if (LowAlignment)
521        SI->setAlignment(1);
522    }
523  } else {
524    CGF.Builder.CreateStore(Val, DestPtr, DestIsVolatile);
525  }
526}
527
528/// CreateCoercedStore - Create a store to \arg DstPtr from \arg Src,
529/// where the source and destination may have different types.
530///
531/// This safely handles the case when the src type is larger than the
532/// destination type; the upper bits of the src will be lost.
533static void CreateCoercedStore(llvm::Value *Src,
534                               llvm::Value *DstPtr,
535                               bool DstIsVolatile,
536                               CodeGenFunction &CGF) {
537  llvm::Type *SrcTy = Src->getType();
538  llvm::Type *DstTy =
539    cast<llvm::PointerType>(DstPtr->getType())->getElementType();
540  if (SrcTy == DstTy) {
541    CGF.Builder.CreateStore(Src, DstPtr, DstIsVolatile);
542    return;
543  }
544
545  uint64_t SrcSize = CGF.CGM.getTargetData().getTypeAllocSize(SrcTy);
546
547  if (llvm::StructType *DstSTy = dyn_cast<llvm::StructType>(DstTy)) {
548    DstPtr = EnterStructPointerForCoercedAccess(DstPtr, DstSTy, SrcSize, CGF);
549    DstTy = cast<llvm::PointerType>(DstPtr->getType())->getElementType();
550  }
551
552  // If the source and destination are integer or pointer types, just do an
553  // extension or truncation to the desired type.
554  if ((isa<llvm::IntegerType>(SrcTy) || isa<llvm::PointerType>(SrcTy)) &&
555      (isa<llvm::IntegerType>(DstTy) || isa<llvm::PointerType>(DstTy))) {
556    Src = CoerceIntOrPtrToIntOrPtr(Src, DstTy, CGF);
557    CGF.Builder.CreateStore(Src, DstPtr, DstIsVolatile);
558    return;
559  }
560
561  uint64_t DstSize = CGF.CGM.getTargetData().getTypeAllocSize(DstTy);
562
563  // If store is legal, just bitcast the src pointer.
564  if (SrcSize <= DstSize) {
565    llvm::Value *Casted =
566      CGF.Builder.CreateBitCast(DstPtr, llvm::PointerType::getUnqual(SrcTy));
567    // FIXME: Use better alignment / avoid requiring aligned store.
568    BuildAggStore(CGF, Src, Casted, DstIsVolatile, true);
569  } else {
570    // Otherwise do coercion through memory. This is stupid, but
571    // simple.
572
573    // Generally SrcSize is never greater than DstSize, since this means we are
574    // losing bits. However, this can happen in cases where the structure has
575    // additional padding, for example due to a user specified alignment.
576    //
577    // FIXME: Assert that we aren't truncating non-padding bits when have access
578    // to that information.
579    llvm::Value *Tmp = CGF.CreateTempAlloca(SrcTy);
580    CGF.Builder.CreateStore(Src, Tmp);
581    llvm::Value *Casted =
582      CGF.Builder.CreateBitCast(Tmp, llvm::PointerType::getUnqual(DstTy));
583    llvm::LoadInst *Load = CGF.Builder.CreateLoad(Casted);
584    // FIXME: Use better alignment / avoid requiring aligned load.
585    Load->setAlignment(1);
586    CGF.Builder.CreateStore(Load, DstPtr, DstIsVolatile);
587  }
588}
589
590/***/
591
592bool CodeGenModule::ReturnTypeUsesSRet(const CGFunctionInfo &FI) {
593  return FI.getReturnInfo().isIndirect();
594}
595
596bool CodeGenModule::ReturnTypeUsesFPRet(QualType ResultType) {
597  if (const BuiltinType *BT = ResultType->getAs<BuiltinType>()) {
598    switch (BT->getKind()) {
599    default:
600      return false;
601    case BuiltinType::Float:
602      return getContext().getTargetInfo().useObjCFPRetForRealType(TargetInfo::Float);
603    case BuiltinType::Double:
604      return getContext().getTargetInfo().useObjCFPRetForRealType(TargetInfo::Double);
605    case BuiltinType::LongDouble:
606      return getContext().getTargetInfo().useObjCFPRetForRealType(
607        TargetInfo::LongDouble);
608    }
609  }
610
611  return false;
612}
613
614bool CodeGenModule::ReturnTypeUsesFP2Ret(QualType ResultType) {
615  if (const ComplexType *CT = ResultType->getAs<ComplexType>()) {
616    if (const BuiltinType *BT = CT->getElementType()->getAs<BuiltinType>()) {
617      if (BT->getKind() == BuiltinType::LongDouble)
618        return getContext().getTargetInfo().useObjCFP2RetForComplexLongDouble();
619    }
620  }
621
622  return false;
623}
624
625llvm::FunctionType *CodeGenTypes::GetFunctionType(GlobalDecl GD) {
626  const CGFunctionInfo &FI = getFunctionInfo(GD);
627
628  // For definition purposes, don't consider a K&R function variadic.
629  bool Variadic = false;
630  if (const FunctionProtoType *FPT =
631        cast<FunctionDecl>(GD.getDecl())->getType()->getAs<FunctionProtoType>())
632    Variadic = FPT->isVariadic();
633
634  return GetFunctionType(FI, Variadic);
635}
636
637llvm::FunctionType *
638CodeGenTypes::GetFunctionType(const CGFunctionInfo &FI, bool isVariadic) {
639
640  bool Inserted = FunctionsBeingProcessed.insert(&FI); (void)Inserted;
641  assert(Inserted && "Recursively being processed?");
642
643  SmallVector<llvm::Type*, 8> argTypes;
644  llvm::Type *resultType = 0;
645
646  const ABIArgInfo &retAI = FI.getReturnInfo();
647  switch (retAI.getKind()) {
648  case ABIArgInfo::Expand:
649    llvm_unreachable("Invalid ABI kind for return argument");
650
651  case ABIArgInfo::Extend:
652  case ABIArgInfo::Direct:
653    resultType = retAI.getCoerceToType();
654    break;
655
656  case ABIArgInfo::Indirect: {
657    assert(!retAI.getIndirectAlign() && "Align unused on indirect return.");
658    resultType = llvm::Type::getVoidTy(getLLVMContext());
659
660    QualType ret = FI.getReturnType();
661    llvm::Type *ty = ConvertType(ret);
662    unsigned addressSpace = Context.getTargetAddressSpace(ret);
663    argTypes.push_back(llvm::PointerType::get(ty, addressSpace));
664    break;
665  }
666
667  case ABIArgInfo::Ignore:
668    resultType = llvm::Type::getVoidTy(getLLVMContext());
669    break;
670  }
671
672  for (CGFunctionInfo::const_arg_iterator it = FI.arg_begin(),
673         ie = FI.arg_end(); it != ie; ++it) {
674    const ABIArgInfo &argAI = it->info;
675
676    switch (argAI.getKind()) {
677    case ABIArgInfo::Ignore:
678      break;
679
680    case ABIArgInfo::Indirect: {
681      // indirect arguments are always on the stack, which is addr space #0.
682      llvm::Type *LTy = ConvertTypeForMem(it->type);
683      argTypes.push_back(LTy->getPointerTo());
684      break;
685    }
686
687    case ABIArgInfo::Extend:
688    case ABIArgInfo::Direct: {
689      // If the coerce-to type is a first class aggregate, flatten it.  Either
690      // way is semantically identical, but fast-isel and the optimizer
691      // generally likes scalar values better than FCAs.
692      llvm::Type *argType = argAI.getCoerceToType();
693      if (llvm::StructType *st = dyn_cast<llvm::StructType>(argType)) {
694        for (unsigned i = 0, e = st->getNumElements(); i != e; ++i)
695          argTypes.push_back(st->getElementType(i));
696      } else {
697        argTypes.push_back(argType);
698      }
699      break;
700    }
701
702    case ABIArgInfo::Expand:
703      GetExpandedTypes(it->type, argTypes);
704      break;
705    }
706  }
707
708  bool Erased = FunctionsBeingProcessed.erase(&FI); (void)Erased;
709  assert(Erased && "Not in set?");
710
711  return llvm::FunctionType::get(resultType, argTypes, isVariadic);
712}
713
714llvm::Type *CodeGenTypes::GetFunctionTypeForVTable(GlobalDecl GD) {
715  const CXXMethodDecl *MD = cast<CXXMethodDecl>(GD.getDecl());
716  const FunctionProtoType *FPT = MD->getType()->getAs<FunctionProtoType>();
717
718  if (!isFuncTypeConvertible(FPT))
719    return llvm::StructType::get(getLLVMContext());
720
721  const CGFunctionInfo *Info;
722  if (isa<CXXDestructorDecl>(MD))
723    Info = &getFunctionInfo(cast<CXXDestructorDecl>(MD), GD.getDtorType());
724  else
725    Info = &getFunctionInfo(MD);
726  return GetFunctionType(*Info, FPT->isVariadic());
727}
728
729void CodeGenModule::ConstructAttributeList(const CGFunctionInfo &FI,
730                                           const Decl *TargetDecl,
731                                           AttributeListType &PAL,
732                                           unsigned &CallingConv) {
733  unsigned FuncAttrs = 0;
734  unsigned RetAttrs = 0;
735
736  CallingConv = FI.getEffectiveCallingConvention();
737
738  if (FI.isNoReturn())
739    FuncAttrs |= llvm::Attribute::NoReturn;
740
741  // FIXME: handle sseregparm someday...
742  if (TargetDecl) {
743    if (TargetDecl->hasAttr<ReturnsTwiceAttr>())
744      FuncAttrs |= llvm::Attribute::ReturnsTwice;
745    if (TargetDecl->hasAttr<NoThrowAttr>())
746      FuncAttrs |= llvm::Attribute::NoUnwind;
747    else if (const FunctionDecl *Fn = dyn_cast<FunctionDecl>(TargetDecl)) {
748      const FunctionProtoType *FPT = Fn->getType()->getAs<FunctionProtoType>();
749      if (FPT && FPT->isNothrow(getContext()))
750        FuncAttrs |= llvm::Attribute::NoUnwind;
751    }
752
753    if (TargetDecl->hasAttr<NoReturnAttr>())
754      FuncAttrs |= llvm::Attribute::NoReturn;
755
756    if (TargetDecl->hasAttr<ReturnsTwiceAttr>())
757      FuncAttrs |= llvm::Attribute::ReturnsTwice;
758
759    // 'const' and 'pure' attribute functions are also nounwind.
760    if (TargetDecl->hasAttr<ConstAttr>()) {
761      FuncAttrs |= llvm::Attribute::ReadNone;
762      FuncAttrs |= llvm::Attribute::NoUnwind;
763    } else if (TargetDecl->hasAttr<PureAttr>()) {
764      FuncAttrs |= llvm::Attribute::ReadOnly;
765      FuncAttrs |= llvm::Attribute::NoUnwind;
766    }
767    if (TargetDecl->hasAttr<MallocAttr>())
768      RetAttrs |= llvm::Attribute::NoAlias;
769  }
770
771  if (CodeGenOpts.OptimizeSize)
772    FuncAttrs |= llvm::Attribute::OptimizeForSize;
773  if (CodeGenOpts.DisableRedZone)
774    FuncAttrs |= llvm::Attribute::NoRedZone;
775  if (CodeGenOpts.NoImplicitFloat)
776    FuncAttrs |= llvm::Attribute::NoImplicitFloat;
777
778  QualType RetTy = FI.getReturnType();
779  unsigned Index = 1;
780  const ABIArgInfo &RetAI = FI.getReturnInfo();
781  switch (RetAI.getKind()) {
782  case ABIArgInfo::Extend:
783   if (RetTy->hasSignedIntegerRepresentation())
784     RetAttrs |= llvm::Attribute::SExt;
785   else if (RetTy->hasUnsignedIntegerRepresentation())
786     RetAttrs |= llvm::Attribute::ZExt;
787    break;
788  case ABIArgInfo::Direct:
789  case ABIArgInfo::Ignore:
790    break;
791
792  case ABIArgInfo::Indirect:
793    PAL.push_back(llvm::AttributeWithIndex::get(Index,
794                                                llvm::Attribute::StructRet));
795    ++Index;
796    // sret disables readnone and readonly
797    FuncAttrs &= ~(llvm::Attribute::ReadOnly |
798                   llvm::Attribute::ReadNone);
799    break;
800
801  case ABIArgInfo::Expand:
802    llvm_unreachable("Invalid ABI kind for return argument");
803  }
804
805  if (RetAttrs)
806    PAL.push_back(llvm::AttributeWithIndex::get(0, RetAttrs));
807
808  // FIXME: RegParm should be reduced in case of global register variable.
809  signed RegParm;
810  if (FI.getHasRegParm())
811    RegParm = FI.getRegParm();
812  else
813    RegParm = CodeGenOpts.NumRegisterParameters;
814
815  unsigned PointerWidth = getContext().getTargetInfo().getPointerWidth(0);
816  for (CGFunctionInfo::const_arg_iterator it = FI.arg_begin(),
817         ie = FI.arg_end(); it != ie; ++it) {
818    QualType ParamType = it->type;
819    const ABIArgInfo &AI = it->info;
820    unsigned Attributes = 0;
821
822    // 'restrict' -> 'noalias' is done in EmitFunctionProlog when we
823    // have the corresponding parameter variable.  It doesn't make
824    // sense to do it here because parameters are so messed up.
825    switch (AI.getKind()) {
826    case ABIArgInfo::Extend:
827      if (ParamType->isSignedIntegerOrEnumerationType())
828        Attributes |= llvm::Attribute::SExt;
829      else if (ParamType->isUnsignedIntegerOrEnumerationType())
830        Attributes |= llvm::Attribute::ZExt;
831      // FALL THROUGH
832    case ABIArgInfo::Direct:
833      if (RegParm > 0 &&
834          (ParamType->isIntegerType() || ParamType->isPointerType())) {
835        RegParm -=
836        (Context.getTypeSize(ParamType) + PointerWidth - 1) / PointerWidth;
837        if (RegParm >= 0)
838          Attributes |= llvm::Attribute::InReg;
839      }
840      // FIXME: handle sseregparm someday...
841
842      if (llvm::StructType *STy =
843            dyn_cast<llvm::StructType>(AI.getCoerceToType()))
844        Index += STy->getNumElements()-1;  // 1 will be added below.
845      break;
846
847    case ABIArgInfo::Indirect:
848      if (AI.getIndirectByVal())
849        Attributes |= llvm::Attribute::ByVal;
850
851      Attributes |=
852        llvm::Attribute::constructAlignmentFromInt(AI.getIndirectAlign());
853      // byval disables readnone and readonly.
854      FuncAttrs &= ~(llvm::Attribute::ReadOnly |
855                     llvm::Attribute::ReadNone);
856      break;
857
858    case ABIArgInfo::Ignore:
859      // Skip increment, no matching LLVM parameter.
860      continue;
861
862    case ABIArgInfo::Expand: {
863      SmallVector<llvm::Type*, 8> types;
864      // FIXME: This is rather inefficient. Do we ever actually need to do
865      // anything here? The result should be just reconstructed on the other
866      // side, so extension should be a non-issue.
867      getTypes().GetExpandedTypes(ParamType, types);
868      Index += types.size();
869      continue;
870    }
871    }
872
873    if (Attributes)
874      PAL.push_back(llvm::AttributeWithIndex::get(Index, Attributes));
875    ++Index;
876  }
877  if (FuncAttrs)
878    PAL.push_back(llvm::AttributeWithIndex::get(~0, FuncAttrs));
879}
880
881/// An argument came in as a promoted argument; demote it back to its
882/// declared type.
883static llvm::Value *emitArgumentDemotion(CodeGenFunction &CGF,
884                                         const VarDecl *var,
885                                         llvm::Value *value) {
886  llvm::Type *varType = CGF.ConvertType(var->getType());
887
888  // This can happen with promotions that actually don't change the
889  // underlying type, like the enum promotions.
890  if (value->getType() == varType) return value;
891
892  assert((varType->isIntegerTy() || varType->isFloatingPointTy())
893         && "unexpected promotion type");
894
895  if (isa<llvm::IntegerType>(varType))
896    return CGF.Builder.CreateTrunc(value, varType, "arg.unpromote");
897
898  return CGF.Builder.CreateFPCast(value, varType, "arg.unpromote");
899}
900
901void CodeGenFunction::EmitFunctionProlog(const CGFunctionInfo &FI,
902                                         llvm::Function *Fn,
903                                         const FunctionArgList &Args) {
904  // If this is an implicit-return-zero function, go ahead and
905  // initialize the return value.  TODO: it might be nice to have
906  // a more general mechanism for this that didn't require synthesized
907  // return statements.
908  if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(CurFuncDecl)) {
909    if (FD->hasImplicitReturnZero()) {
910      QualType RetTy = FD->getResultType().getUnqualifiedType();
911      llvm::Type* LLVMTy = CGM.getTypes().ConvertType(RetTy);
912      llvm::Constant* Zero = llvm::Constant::getNullValue(LLVMTy);
913      Builder.CreateStore(Zero, ReturnValue);
914    }
915  }
916
917  // FIXME: We no longer need the types from FunctionArgList; lift up and
918  // simplify.
919
920  // Emit allocs for param decls.  Give the LLVM Argument nodes names.
921  llvm::Function::arg_iterator AI = Fn->arg_begin();
922
923  // Name the struct return argument.
924  if (CGM.ReturnTypeUsesSRet(FI)) {
925    AI->setName("agg.result");
926    AI->addAttr(llvm::Attribute::NoAlias);
927    ++AI;
928  }
929
930  assert(FI.arg_size() == Args.size() &&
931         "Mismatch between function signature & arguments.");
932  unsigned ArgNo = 1;
933  CGFunctionInfo::const_arg_iterator info_it = FI.arg_begin();
934  for (FunctionArgList::const_iterator i = Args.begin(), e = Args.end();
935       i != e; ++i, ++info_it, ++ArgNo) {
936    const VarDecl *Arg = *i;
937    QualType Ty = info_it->type;
938    const ABIArgInfo &ArgI = info_it->info;
939
940    bool isPromoted =
941      isa<ParmVarDecl>(Arg) && cast<ParmVarDecl>(Arg)->isKNRPromoted();
942
943    switch (ArgI.getKind()) {
944    case ABIArgInfo::Indirect: {
945      llvm::Value *V = AI;
946
947      if (hasAggregateLLVMType(Ty)) {
948        // Aggregates and complex variables are accessed by reference.  All we
949        // need to do is realign the value, if requested
950        if (ArgI.getIndirectRealign()) {
951          llvm::Value *AlignedTemp = CreateMemTemp(Ty, "coerce");
952
953          // Copy from the incoming argument pointer to the temporary with the
954          // appropriate alignment.
955          //
956          // FIXME: We should have a common utility for generating an aggregate
957          // copy.
958          llvm::Type *I8PtrTy = Builder.getInt8PtrTy();
959          CharUnits Size = getContext().getTypeSizeInChars(Ty);
960          llvm::Value *Dst = Builder.CreateBitCast(AlignedTemp, I8PtrTy);
961          llvm::Value *Src = Builder.CreateBitCast(V, I8PtrTy);
962          Builder.CreateMemCpy(Dst,
963                               Src,
964                               llvm::ConstantInt::get(IntPtrTy,
965                                                      Size.getQuantity()),
966                               ArgI.getIndirectAlign(),
967                               false);
968          V = AlignedTemp;
969        }
970      } else {
971        // Load scalar value from indirect argument.
972        CharUnits Alignment = getContext().getTypeAlignInChars(Ty);
973        V = EmitLoadOfScalar(V, false, Alignment.getQuantity(), Ty);
974
975        if (isPromoted)
976          V = emitArgumentDemotion(*this, Arg, V);
977      }
978      EmitParmDecl(*Arg, V, ArgNo);
979      break;
980    }
981
982    case ABIArgInfo::Extend:
983    case ABIArgInfo::Direct: {
984      // If we have the trivial case, handle it with no muss and fuss.
985      if (!isa<llvm::StructType>(ArgI.getCoerceToType()) &&
986          ArgI.getCoerceToType() == ConvertType(Ty) &&
987          ArgI.getDirectOffset() == 0) {
988        assert(AI != Fn->arg_end() && "Argument mismatch!");
989        llvm::Value *V = AI;
990
991        if (Arg->getType().isRestrictQualified())
992          AI->addAttr(llvm::Attribute::NoAlias);
993
994        // Ensure the argument is the correct type.
995        if (V->getType() != ArgI.getCoerceToType())
996          V = Builder.CreateBitCast(V, ArgI.getCoerceToType());
997
998        if (isPromoted)
999          V = emitArgumentDemotion(*this, Arg, V);
1000
1001        EmitParmDecl(*Arg, V, ArgNo);
1002        break;
1003      }
1004
1005      llvm::AllocaInst *Alloca = CreateMemTemp(Ty, "coerce");
1006
1007      // The alignment we need to use is the max of the requested alignment for
1008      // the argument plus the alignment required by our access code below.
1009      unsigned AlignmentToUse =
1010        CGM.getTargetData().getABITypeAlignment(ArgI.getCoerceToType());
1011      AlignmentToUse = std::max(AlignmentToUse,
1012                        (unsigned)getContext().getDeclAlign(Arg).getQuantity());
1013
1014      Alloca->setAlignment(AlignmentToUse);
1015      llvm::Value *V = Alloca;
1016      llvm::Value *Ptr = V;    // Pointer to store into.
1017
1018      // If the value is offset in memory, apply the offset now.
1019      if (unsigned Offs = ArgI.getDirectOffset()) {
1020        Ptr = Builder.CreateBitCast(Ptr, Builder.getInt8PtrTy());
1021        Ptr = Builder.CreateConstGEP1_32(Ptr, Offs);
1022        Ptr = Builder.CreateBitCast(Ptr,
1023                          llvm::PointerType::getUnqual(ArgI.getCoerceToType()));
1024      }
1025
1026      // If the coerce-to type is a first class aggregate, we flatten it and
1027      // pass the elements. Either way is semantically identical, but fast-isel
1028      // and the optimizer generally likes scalar values better than FCAs.
1029      if (llvm::StructType *STy =
1030            dyn_cast<llvm::StructType>(ArgI.getCoerceToType())) {
1031        Ptr = Builder.CreateBitCast(Ptr, llvm::PointerType::getUnqual(STy));
1032
1033        for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) {
1034          assert(AI != Fn->arg_end() && "Argument mismatch!");
1035          AI->setName(Arg->getName() + ".coerce" + Twine(i));
1036          llvm::Value *EltPtr = Builder.CreateConstGEP2_32(Ptr, 0, i);
1037          Builder.CreateStore(AI++, EltPtr);
1038        }
1039      } else {
1040        // Simple case, just do a coerced store of the argument into the alloca.
1041        assert(AI != Fn->arg_end() && "Argument mismatch!");
1042        AI->setName(Arg->getName() + ".coerce");
1043        CreateCoercedStore(AI++, Ptr, /*DestIsVolatile=*/false, *this);
1044      }
1045
1046
1047      // Match to what EmitParmDecl is expecting for this type.
1048      if (!CodeGenFunction::hasAggregateLLVMType(Ty)) {
1049        V = EmitLoadOfScalar(V, false, AlignmentToUse, Ty);
1050        if (isPromoted)
1051          V = emitArgumentDemotion(*this, Arg, V);
1052      }
1053      EmitParmDecl(*Arg, V, ArgNo);
1054      continue;  // Skip ++AI increment, already done.
1055    }
1056
1057    case ABIArgInfo::Expand: {
1058      // If this structure was expanded into multiple arguments then
1059      // we need to create a temporary and reconstruct it from the
1060      // arguments.
1061      llvm::AllocaInst *Alloca = CreateMemTemp(Ty);
1062      Alloca->setAlignment(getContext().getDeclAlign(Arg).getQuantity());
1063      LValue LV = MakeAddrLValue(Alloca, Ty, Alloca->getAlignment());
1064      llvm::Function::arg_iterator End = ExpandTypeFromArgs(Ty, LV, AI);
1065      EmitParmDecl(*Arg, Alloca, ArgNo);
1066
1067      // Name the arguments used in expansion and increment AI.
1068      unsigned Index = 0;
1069      for (; AI != End; ++AI, ++Index)
1070        AI->setName(Arg->getName() + "." + Twine(Index));
1071      continue;
1072    }
1073
1074    case ABIArgInfo::Ignore:
1075      // Initialize the local variable appropriately.
1076      if (hasAggregateLLVMType(Ty))
1077        EmitParmDecl(*Arg, CreateMemTemp(Ty), ArgNo);
1078      else
1079        EmitParmDecl(*Arg, llvm::UndefValue::get(ConvertType(Arg->getType())),
1080                     ArgNo);
1081
1082      // Skip increment, no matching LLVM parameter.
1083      continue;
1084    }
1085
1086    ++AI;
1087  }
1088  assert(AI == Fn->arg_end() && "Argument mismatch!");
1089}
1090
1091/// Try to emit a fused autorelease of a return result.
1092static llvm::Value *tryEmitFusedAutoreleaseOfResult(CodeGenFunction &CGF,
1093                                                    llvm::Value *result) {
1094  // We must be immediately followed the cast.
1095  llvm::BasicBlock *BB = CGF.Builder.GetInsertBlock();
1096  if (BB->empty()) return 0;
1097  if (&BB->back() != result) return 0;
1098
1099  llvm::Type *resultType = result->getType();
1100
1101  // result is in a BasicBlock and is therefore an Instruction.
1102  llvm::Instruction *generator = cast<llvm::Instruction>(result);
1103
1104  SmallVector<llvm::Instruction*,4> insnsToKill;
1105
1106  // Look for:
1107  //  %generator = bitcast %type1* %generator2 to %type2*
1108  while (llvm::BitCastInst *bitcast = dyn_cast<llvm::BitCastInst>(generator)) {
1109    // We would have emitted this as a constant if the operand weren't
1110    // an Instruction.
1111    generator = cast<llvm::Instruction>(bitcast->getOperand(0));
1112
1113    // Require the generator to be immediately followed by the cast.
1114    if (generator->getNextNode() != bitcast)
1115      return 0;
1116
1117    insnsToKill.push_back(bitcast);
1118  }
1119
1120  // Look for:
1121  //   %generator = call i8* @objc_retain(i8* %originalResult)
1122  // or
1123  //   %generator = call i8* @objc_retainAutoreleasedReturnValue(i8* %originalResult)
1124  llvm::CallInst *call = dyn_cast<llvm::CallInst>(generator);
1125  if (!call) return 0;
1126
1127  bool doRetainAutorelease;
1128
1129  if (call->getCalledValue() == CGF.CGM.getARCEntrypoints().objc_retain) {
1130    doRetainAutorelease = true;
1131  } else if (call->getCalledValue() == CGF.CGM.getARCEntrypoints()
1132                                          .objc_retainAutoreleasedReturnValue) {
1133    doRetainAutorelease = false;
1134
1135    // Look for an inline asm immediately preceding the call and kill it, too.
1136    llvm::Instruction *prev = call->getPrevNode();
1137    if (llvm::CallInst *asmCall = dyn_cast_or_null<llvm::CallInst>(prev))
1138      if (asmCall->getCalledValue()
1139            == CGF.CGM.getARCEntrypoints().retainAutoreleasedReturnValueMarker)
1140        insnsToKill.push_back(prev);
1141  } else {
1142    return 0;
1143  }
1144
1145  result = call->getArgOperand(0);
1146  insnsToKill.push_back(call);
1147
1148  // Keep killing bitcasts, for sanity.  Note that we no longer care
1149  // about precise ordering as long as there's exactly one use.
1150  while (llvm::BitCastInst *bitcast = dyn_cast<llvm::BitCastInst>(result)) {
1151    if (!bitcast->hasOneUse()) break;
1152    insnsToKill.push_back(bitcast);
1153    result = bitcast->getOperand(0);
1154  }
1155
1156  // Delete all the unnecessary instructions, from latest to earliest.
1157  for (SmallVectorImpl<llvm::Instruction*>::iterator
1158         i = insnsToKill.begin(), e = insnsToKill.end(); i != e; ++i)
1159    (*i)->eraseFromParent();
1160
1161  // Do the fused retain/autorelease if we were asked to.
1162  if (doRetainAutorelease)
1163    result = CGF.EmitARCRetainAutoreleaseReturnValue(result);
1164
1165  // Cast back to the result type.
1166  return CGF.Builder.CreateBitCast(result, resultType);
1167}
1168
1169/// Emit an ARC autorelease of the result of a function.
1170static llvm::Value *emitAutoreleaseOfResult(CodeGenFunction &CGF,
1171                                            llvm::Value *result) {
1172  // At -O0, try to emit a fused retain/autorelease.
1173  if (CGF.shouldUseFusedARCCalls())
1174    if (llvm::Value *fused = tryEmitFusedAutoreleaseOfResult(CGF, result))
1175      return fused;
1176
1177  return CGF.EmitARCAutoreleaseReturnValue(result);
1178}
1179
1180void CodeGenFunction::EmitFunctionEpilog(const CGFunctionInfo &FI) {
1181  // Functions with no result always return void.
1182  if (ReturnValue == 0) {
1183    Builder.CreateRetVoid();
1184    return;
1185  }
1186
1187  llvm::DebugLoc RetDbgLoc;
1188  llvm::Value *RV = 0;
1189  QualType RetTy = FI.getReturnType();
1190  const ABIArgInfo &RetAI = FI.getReturnInfo();
1191
1192  switch (RetAI.getKind()) {
1193  case ABIArgInfo::Indirect: {
1194    unsigned Alignment = getContext().getTypeAlignInChars(RetTy).getQuantity();
1195    if (RetTy->isAnyComplexType()) {
1196      ComplexPairTy RT = LoadComplexFromAddr(ReturnValue, false);
1197      StoreComplexToAddr(RT, CurFn->arg_begin(), false);
1198    } else if (CodeGenFunction::hasAggregateLLVMType(RetTy)) {
1199      // Do nothing; aggregrates get evaluated directly into the destination.
1200    } else {
1201      EmitStoreOfScalar(Builder.CreateLoad(ReturnValue), CurFn->arg_begin(),
1202                        false, Alignment, RetTy);
1203    }
1204    break;
1205  }
1206
1207  case ABIArgInfo::Extend:
1208  case ABIArgInfo::Direct:
1209    if (RetAI.getCoerceToType() == ConvertType(RetTy) &&
1210        RetAI.getDirectOffset() == 0) {
1211      // The internal return value temp always will have pointer-to-return-type
1212      // type, just do a load.
1213
1214      // If the instruction right before the insertion point is a store to the
1215      // return value, we can elide the load, zap the store, and usually zap the
1216      // alloca.
1217      llvm::BasicBlock *InsertBB = Builder.GetInsertBlock();
1218      llvm::StoreInst *SI = 0;
1219      if (InsertBB->empty() ||
1220          !(SI = dyn_cast<llvm::StoreInst>(&InsertBB->back())) ||
1221          SI->getPointerOperand() != ReturnValue || SI->isVolatile()) {
1222        RV = Builder.CreateLoad(ReturnValue);
1223      } else {
1224        // Get the stored value and nuke the now-dead store.
1225        RetDbgLoc = SI->getDebugLoc();
1226        RV = SI->getValueOperand();
1227        SI->eraseFromParent();
1228
1229        // If that was the only use of the return value, nuke it as well now.
1230        if (ReturnValue->use_empty() && isa<llvm::AllocaInst>(ReturnValue)) {
1231          cast<llvm::AllocaInst>(ReturnValue)->eraseFromParent();
1232          ReturnValue = 0;
1233        }
1234      }
1235    } else {
1236      llvm::Value *V = ReturnValue;
1237      // If the value is offset in memory, apply the offset now.
1238      if (unsigned Offs = RetAI.getDirectOffset()) {
1239        V = Builder.CreateBitCast(V, Builder.getInt8PtrTy());
1240        V = Builder.CreateConstGEP1_32(V, Offs);
1241        V = Builder.CreateBitCast(V,
1242                         llvm::PointerType::getUnqual(RetAI.getCoerceToType()));
1243      }
1244
1245      RV = CreateCoercedLoad(V, RetAI.getCoerceToType(), *this);
1246    }
1247
1248    // In ARC, end functions that return a retainable type with a call
1249    // to objc_autoreleaseReturnValue.
1250    if (AutoreleaseResult) {
1251      assert(getLangOptions().ObjCAutoRefCount &&
1252             !FI.isReturnsRetained() &&
1253             RetTy->isObjCRetainableType());
1254      RV = emitAutoreleaseOfResult(*this, RV);
1255    }
1256
1257    break;
1258
1259  case ABIArgInfo::Ignore:
1260    break;
1261
1262  case ABIArgInfo::Expand:
1263    llvm_unreachable("Invalid ABI kind for return argument");
1264  }
1265
1266  llvm::Instruction *Ret = RV ? Builder.CreateRet(RV) : Builder.CreateRetVoid();
1267  if (!RetDbgLoc.isUnknown())
1268    Ret->setDebugLoc(RetDbgLoc);
1269}
1270
1271void CodeGenFunction::EmitDelegateCallArg(CallArgList &args,
1272                                          const VarDecl *param) {
1273  // StartFunction converted the ABI-lowered parameter(s) into a
1274  // local alloca.  We need to turn that into an r-value suitable
1275  // for EmitCall.
1276  llvm::Value *local = GetAddrOfLocalVar(param);
1277
1278  QualType type = param->getType();
1279
1280  // For the most part, we just need to load the alloca, except:
1281  // 1) aggregate r-values are actually pointers to temporaries, and
1282  // 2) references to aggregates are pointers directly to the aggregate.
1283  // I don't know why references to non-aggregates are different here.
1284  if (const ReferenceType *ref = type->getAs<ReferenceType>()) {
1285    if (hasAggregateLLVMType(ref->getPointeeType()))
1286      return args.add(RValue::getAggregate(local), type);
1287
1288    // Locals which are references to scalars are represented
1289    // with allocas holding the pointer.
1290    return args.add(RValue::get(Builder.CreateLoad(local)), type);
1291  }
1292
1293  if (type->isAnyComplexType()) {
1294    ComplexPairTy complex = LoadComplexFromAddr(local, /*volatile*/ false);
1295    return args.add(RValue::getComplex(complex), type);
1296  }
1297
1298  if (hasAggregateLLVMType(type))
1299    return args.add(RValue::getAggregate(local), type);
1300
1301  unsigned alignment = getContext().getDeclAlign(param).getQuantity();
1302  llvm::Value *value = EmitLoadOfScalar(local, false, alignment, type);
1303  return args.add(RValue::get(value), type);
1304}
1305
1306static bool isProvablyNull(llvm::Value *addr) {
1307  return isa<llvm::ConstantPointerNull>(addr);
1308}
1309
1310static bool isProvablyNonNull(llvm::Value *addr) {
1311  return isa<llvm::AllocaInst>(addr);
1312}
1313
1314/// Emit the actual writing-back of a writeback.
1315static void emitWriteback(CodeGenFunction &CGF,
1316                          const CallArgList::Writeback &writeback) {
1317  llvm::Value *srcAddr = writeback.Address;
1318  assert(!isProvablyNull(srcAddr) &&
1319         "shouldn't have writeback for provably null argument");
1320
1321  llvm::BasicBlock *contBB = 0;
1322
1323  // If the argument wasn't provably non-null, we need to null check
1324  // before doing the store.
1325  bool provablyNonNull = isProvablyNonNull(srcAddr);
1326  if (!provablyNonNull) {
1327    llvm::BasicBlock *writebackBB = CGF.createBasicBlock("icr.writeback");
1328    contBB = CGF.createBasicBlock("icr.done");
1329
1330    llvm::Value *isNull = CGF.Builder.CreateIsNull(srcAddr, "icr.isnull");
1331    CGF.Builder.CreateCondBr(isNull, contBB, writebackBB);
1332    CGF.EmitBlock(writebackBB);
1333  }
1334
1335  // Load the value to writeback.
1336  llvm::Value *value = CGF.Builder.CreateLoad(writeback.Temporary);
1337
1338  // Cast it back, in case we're writing an id to a Foo* or something.
1339  value = CGF.Builder.CreateBitCast(value,
1340               cast<llvm::PointerType>(srcAddr->getType())->getElementType(),
1341                            "icr.writeback-cast");
1342
1343  // Perform the writeback.
1344  QualType srcAddrType = writeback.AddressType;
1345  CGF.EmitStoreThroughLValue(RValue::get(value),
1346                             CGF.MakeAddrLValue(srcAddr, srcAddrType));
1347
1348  // Jump to the continuation block.
1349  if (!provablyNonNull)
1350    CGF.EmitBlock(contBB);
1351}
1352
1353static void emitWritebacks(CodeGenFunction &CGF,
1354                           const CallArgList &args) {
1355  for (CallArgList::writeback_iterator
1356         i = args.writeback_begin(), e = args.writeback_end(); i != e; ++i)
1357    emitWriteback(CGF, *i);
1358}
1359
1360/// Emit an argument that's being passed call-by-writeback.  That is,
1361/// we are passing the address of
1362static void emitWritebackArg(CodeGenFunction &CGF, CallArgList &args,
1363                             const ObjCIndirectCopyRestoreExpr *CRE) {
1364  llvm::Value *srcAddr = CGF.EmitScalarExpr(CRE->getSubExpr());
1365
1366  // The dest and src types don't necessarily match in LLVM terms
1367  // because of the crazy ObjC compatibility rules.
1368
1369  llvm::PointerType *destType =
1370    cast<llvm::PointerType>(CGF.ConvertType(CRE->getType()));
1371
1372  // If the address is a constant null, just pass the appropriate null.
1373  if (isProvablyNull(srcAddr)) {
1374    args.add(RValue::get(llvm::ConstantPointerNull::get(destType)),
1375             CRE->getType());
1376    return;
1377  }
1378
1379  QualType srcAddrType =
1380    CRE->getSubExpr()->getType()->castAs<PointerType>()->getPointeeType();
1381
1382  // Create the temporary.
1383  llvm::Value *temp = CGF.CreateTempAlloca(destType->getElementType(),
1384                                           "icr.temp");
1385
1386  // Zero-initialize it if we're not doing a copy-initialization.
1387  bool shouldCopy = CRE->shouldCopy();
1388  if (!shouldCopy) {
1389    llvm::Value *null =
1390      llvm::ConstantPointerNull::get(
1391        cast<llvm::PointerType>(destType->getElementType()));
1392    CGF.Builder.CreateStore(null, temp);
1393  }
1394
1395  llvm::BasicBlock *contBB = 0;
1396
1397  // If the address is *not* known to be non-null, we need to switch.
1398  llvm::Value *finalArgument;
1399
1400  bool provablyNonNull = isProvablyNonNull(srcAddr);
1401  if (provablyNonNull) {
1402    finalArgument = temp;
1403  } else {
1404    llvm::Value *isNull = CGF.Builder.CreateIsNull(srcAddr, "icr.isnull");
1405
1406    finalArgument = CGF.Builder.CreateSelect(isNull,
1407                                   llvm::ConstantPointerNull::get(destType),
1408                                             temp, "icr.argument");
1409
1410    // If we need to copy, then the load has to be conditional, which
1411    // means we need control flow.
1412    if (shouldCopy) {
1413      contBB = CGF.createBasicBlock("icr.cont");
1414      llvm::BasicBlock *copyBB = CGF.createBasicBlock("icr.copy");
1415      CGF.Builder.CreateCondBr(isNull, contBB, copyBB);
1416      CGF.EmitBlock(copyBB);
1417    }
1418  }
1419
1420  // Perform a copy if necessary.
1421  if (shouldCopy) {
1422    LValue srcLV = CGF.MakeAddrLValue(srcAddr, srcAddrType);
1423    RValue srcRV = CGF.EmitLoadOfLValue(srcLV);
1424    assert(srcRV.isScalar());
1425
1426    llvm::Value *src = srcRV.getScalarVal();
1427    src = CGF.Builder.CreateBitCast(src, destType->getElementType(),
1428                                    "icr.cast");
1429
1430    // Use an ordinary store, not a store-to-lvalue.
1431    CGF.Builder.CreateStore(src, temp);
1432  }
1433
1434  // Finish the control flow if we needed it.
1435  if (shouldCopy && !provablyNonNull)
1436    CGF.EmitBlock(contBB);
1437
1438  args.addWriteback(srcAddr, srcAddrType, temp);
1439  args.add(RValue::get(finalArgument), CRE->getType());
1440}
1441
1442void CodeGenFunction::EmitCallArg(CallArgList &args, const Expr *E,
1443                                  QualType type) {
1444  if (const ObjCIndirectCopyRestoreExpr *CRE
1445        = dyn_cast<ObjCIndirectCopyRestoreExpr>(E)) {
1446    assert(getContext().getLangOptions().ObjCAutoRefCount);
1447    assert(getContext().hasSameType(E->getType(), type));
1448    return emitWritebackArg(*this, args, CRE);
1449  }
1450
1451  assert(type->isReferenceType() == E->isGLValue() &&
1452         "reference binding to unmaterialized r-value!");
1453
1454  if (E->isGLValue()) {
1455    assert(E->getObjectKind() == OK_Ordinary);
1456    return args.add(EmitReferenceBindingToExpr(E, /*InitializedDecl=*/0),
1457                    type);
1458  }
1459
1460  if (hasAggregateLLVMType(type) && !E->getType()->isAnyComplexType() &&
1461      isa<ImplicitCastExpr>(E) &&
1462      cast<CastExpr>(E)->getCastKind() == CK_LValueToRValue) {
1463    LValue L = EmitLValue(cast<CastExpr>(E)->getSubExpr());
1464    assert(L.isSimple());
1465    args.add(RValue::getAggregate(L.getAddress(), L.isVolatileQualified()),
1466             type, /*NeedsCopy*/true);
1467    return;
1468  }
1469
1470  args.add(EmitAnyExprToTemp(E), type);
1471}
1472
1473/// Emits a call or invoke instruction to the given function, depending
1474/// on the current state of the EH stack.
1475llvm::CallSite
1476CodeGenFunction::EmitCallOrInvoke(llvm::Value *Callee,
1477                                  ArrayRef<llvm::Value *> Args,
1478                                  const Twine &Name) {
1479  llvm::BasicBlock *InvokeDest = getInvokeDest();
1480  if (!InvokeDest)
1481    return Builder.CreateCall(Callee, Args, Name);
1482
1483  llvm::BasicBlock *ContBB = createBasicBlock("invoke.cont");
1484  llvm::InvokeInst *Invoke = Builder.CreateInvoke(Callee, ContBB, InvokeDest,
1485                                                  Args, Name);
1486  EmitBlock(ContBB);
1487  return Invoke;
1488}
1489
1490llvm::CallSite
1491CodeGenFunction::EmitCallOrInvoke(llvm::Value *Callee,
1492                                  const Twine &Name) {
1493  return EmitCallOrInvoke(Callee, ArrayRef<llvm::Value *>(), Name);
1494}
1495
1496static void checkArgMatches(llvm::Value *Elt, unsigned &ArgNo,
1497                            llvm::FunctionType *FTy) {
1498  if (ArgNo < FTy->getNumParams())
1499    assert(Elt->getType() == FTy->getParamType(ArgNo));
1500  else
1501    assert(FTy->isVarArg());
1502  ++ArgNo;
1503}
1504
1505void CodeGenFunction::ExpandTypeToArgs(QualType Ty, RValue RV,
1506                                       SmallVector<llvm::Value*,16> &Args,
1507                                       llvm::FunctionType *IRFuncTy) {
1508  if (const ConstantArrayType *AT = getContext().getAsConstantArrayType(Ty)) {
1509    unsigned NumElts = AT->getSize().getZExtValue();
1510    QualType EltTy = AT->getElementType();
1511    llvm::Value *Addr = RV.getAggregateAddr();
1512    for (unsigned Elt = 0; Elt < NumElts; ++Elt) {
1513      llvm::Value *EltAddr = Builder.CreateConstGEP2_32(Addr, 0, Elt);
1514      LValue LV = MakeAddrLValue(EltAddr, EltTy);
1515      RValue EltRV;
1516      if (CodeGenFunction::hasAggregateLLVMType(EltTy))
1517        EltRV = RValue::getAggregate(LV.getAddress());
1518      else
1519        EltRV = EmitLoadOfLValue(LV);
1520      ExpandTypeToArgs(EltTy, EltRV, Args, IRFuncTy);
1521    }
1522  } else if (const RecordType *RT = Ty->getAsStructureType()) {
1523    RecordDecl *RD = RT->getDecl();
1524    assert(RV.isAggregate() && "Unexpected rvalue during struct expansion");
1525    llvm::Value *Addr = RV.getAggregateAddr();
1526    for (RecordDecl::field_iterator i = RD->field_begin(), e = RD->field_end();
1527         i != e; ++i) {
1528      FieldDecl *FD = *i;
1529      QualType FT = FD->getType();
1530
1531      // FIXME: What are the right qualifiers here?
1532      LValue LV = EmitLValueForField(Addr, FD, 0);
1533      RValue FldRV;
1534      if (CodeGenFunction::hasAggregateLLVMType(FT))
1535        FldRV = RValue::getAggregate(LV.getAddress());
1536      else
1537        FldRV = EmitLoadOfLValue(LV);
1538      ExpandTypeToArgs(FT, FldRV, Args, IRFuncTy);
1539    }
1540  } else if (isa<ComplexType>(Ty)) {
1541    ComplexPairTy CV = RV.getComplexVal();
1542    Args.push_back(CV.first);
1543    Args.push_back(CV.second);
1544  } else {
1545    assert(RV.isScalar() &&
1546           "Unexpected non-scalar rvalue during struct expansion.");
1547
1548    // Insert a bitcast as needed.
1549    llvm::Value *V = RV.getScalarVal();
1550    if (Args.size() < IRFuncTy->getNumParams() &&
1551        V->getType() != IRFuncTy->getParamType(Args.size()))
1552      V = Builder.CreateBitCast(V, IRFuncTy->getParamType(Args.size()));
1553
1554    Args.push_back(V);
1555  }
1556}
1557
1558
1559RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
1560                                 llvm::Value *Callee,
1561                                 ReturnValueSlot ReturnValue,
1562                                 const CallArgList &CallArgs,
1563                                 const Decl *TargetDecl,
1564                                 llvm::Instruction **callOrInvoke) {
1565  // FIXME: We no longer need the types from CallArgs; lift up and simplify.
1566  SmallVector<llvm::Value*, 16> Args;
1567
1568  // Handle struct-return functions by passing a pointer to the
1569  // location that we would like to return into.
1570  QualType RetTy = CallInfo.getReturnType();
1571  const ABIArgInfo &RetAI = CallInfo.getReturnInfo();
1572
1573  // IRArgNo - Keep track of the argument number in the callee we're looking at.
1574  unsigned IRArgNo = 0;
1575  llvm::FunctionType *IRFuncTy =
1576    cast<llvm::FunctionType>(
1577                  cast<llvm::PointerType>(Callee->getType())->getElementType());
1578
1579  // If the call returns a temporary with struct return, create a temporary
1580  // alloca to hold the result, unless one is given to us.
1581  if (CGM.ReturnTypeUsesSRet(CallInfo)) {
1582    llvm::Value *Value = ReturnValue.getValue();
1583    if (!Value)
1584      Value = CreateMemTemp(RetTy);
1585    Args.push_back(Value);
1586    checkArgMatches(Value, IRArgNo, IRFuncTy);
1587  }
1588
1589  assert(CallInfo.arg_size() == CallArgs.size() &&
1590         "Mismatch between function signature & arguments.");
1591  CGFunctionInfo::const_arg_iterator info_it = CallInfo.arg_begin();
1592  for (CallArgList::const_iterator I = CallArgs.begin(), E = CallArgs.end();
1593       I != E; ++I, ++info_it) {
1594    const ABIArgInfo &ArgInfo = info_it->info;
1595    RValue RV = I->RV;
1596
1597    unsigned TypeAlign =
1598      getContext().getTypeAlignInChars(I->Ty).getQuantity();
1599    switch (ArgInfo.getKind()) {
1600    case ABIArgInfo::Indirect: {
1601      if (RV.isScalar() || RV.isComplex()) {
1602        // Make a temporary alloca to pass the argument.
1603        llvm::AllocaInst *AI = CreateMemTemp(I->Ty);
1604        if (ArgInfo.getIndirectAlign() > AI->getAlignment())
1605          AI->setAlignment(ArgInfo.getIndirectAlign());
1606        Args.push_back(AI);
1607
1608        if (RV.isScalar())
1609          EmitStoreOfScalar(RV.getScalarVal(), Args.back(), false,
1610                            TypeAlign, I->Ty);
1611        else
1612          StoreComplexToAddr(RV.getComplexVal(), Args.back(), false);
1613
1614        // Validate argument match.
1615        checkArgMatches(AI, IRArgNo, IRFuncTy);
1616      } else {
1617        // We want to avoid creating an unnecessary temporary+copy here;
1618        // however, we need one in two cases:
1619        // 1. If the argument is not byval, and we are required to copy the
1620        //    source.  (This case doesn't occur on any common architecture.)
1621        // 2. If the argument is byval, RV is not sufficiently aligned, and
1622        //    we cannot force it to be sufficiently aligned.
1623        llvm::Value *Addr = RV.getAggregateAddr();
1624        unsigned Align = ArgInfo.getIndirectAlign();
1625        const llvm::TargetData *TD = &CGM.getTargetData();
1626        if ((!ArgInfo.getIndirectByVal() && I->NeedsCopy) ||
1627            (ArgInfo.getIndirectByVal() && TypeAlign < Align &&
1628             llvm::getOrEnforceKnownAlignment(Addr, Align, TD) < Align)) {
1629          // Create an aligned temporary, and copy to it.
1630          llvm::AllocaInst *AI = CreateMemTemp(I->Ty);
1631          if (Align > AI->getAlignment())
1632            AI->setAlignment(Align);
1633          Args.push_back(AI);
1634          EmitAggregateCopy(AI, Addr, I->Ty, RV.isVolatileQualified());
1635
1636          // Validate argument match.
1637          checkArgMatches(AI, IRArgNo, IRFuncTy);
1638        } else {
1639          // Skip the extra memcpy call.
1640          Args.push_back(Addr);
1641
1642          // Validate argument match.
1643          checkArgMatches(Addr, IRArgNo, IRFuncTy);
1644        }
1645      }
1646      break;
1647    }
1648
1649    case ABIArgInfo::Ignore:
1650      break;
1651
1652    case ABIArgInfo::Extend:
1653    case ABIArgInfo::Direct: {
1654      if (!isa<llvm::StructType>(ArgInfo.getCoerceToType()) &&
1655          ArgInfo.getCoerceToType() == ConvertType(info_it->type) &&
1656          ArgInfo.getDirectOffset() == 0) {
1657        llvm::Value *V;
1658        if (RV.isScalar())
1659          V = RV.getScalarVal();
1660        else
1661          V = Builder.CreateLoad(RV.getAggregateAddr());
1662
1663        // If the argument doesn't match, perform a bitcast to coerce it.  This
1664        // can happen due to trivial type mismatches.
1665        if (IRArgNo < IRFuncTy->getNumParams() &&
1666            V->getType() != IRFuncTy->getParamType(IRArgNo))
1667          V = Builder.CreateBitCast(V, IRFuncTy->getParamType(IRArgNo));
1668        Args.push_back(V);
1669
1670        checkArgMatches(V, IRArgNo, IRFuncTy);
1671        break;
1672      }
1673
1674      // FIXME: Avoid the conversion through memory if possible.
1675      llvm::Value *SrcPtr;
1676      if (RV.isScalar()) {
1677        SrcPtr = CreateMemTemp(I->Ty, "coerce");
1678        EmitStoreOfScalar(RV.getScalarVal(), SrcPtr, false, TypeAlign, I->Ty);
1679      } else if (RV.isComplex()) {
1680        SrcPtr = CreateMemTemp(I->Ty, "coerce");
1681        StoreComplexToAddr(RV.getComplexVal(), SrcPtr, false);
1682      } else
1683        SrcPtr = RV.getAggregateAddr();
1684
1685      // If the value is offset in memory, apply the offset now.
1686      if (unsigned Offs = ArgInfo.getDirectOffset()) {
1687        SrcPtr = Builder.CreateBitCast(SrcPtr, Builder.getInt8PtrTy());
1688        SrcPtr = Builder.CreateConstGEP1_32(SrcPtr, Offs);
1689        SrcPtr = Builder.CreateBitCast(SrcPtr,
1690                       llvm::PointerType::getUnqual(ArgInfo.getCoerceToType()));
1691
1692      }
1693
1694      // If the coerce-to type is a first class aggregate, we flatten it and
1695      // pass the elements. Either way is semantically identical, but fast-isel
1696      // and the optimizer generally likes scalar values better than FCAs.
1697      if (llvm::StructType *STy =
1698            dyn_cast<llvm::StructType>(ArgInfo.getCoerceToType())) {
1699        SrcPtr = Builder.CreateBitCast(SrcPtr,
1700                                       llvm::PointerType::getUnqual(STy));
1701        for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) {
1702          llvm::Value *EltPtr = Builder.CreateConstGEP2_32(SrcPtr, 0, i);
1703          llvm::LoadInst *LI = Builder.CreateLoad(EltPtr);
1704          // We don't know what we're loading from.
1705          LI->setAlignment(1);
1706          Args.push_back(LI);
1707
1708          // Validate argument match.
1709          checkArgMatches(LI, IRArgNo, IRFuncTy);
1710        }
1711      } else {
1712        // In the simple case, just pass the coerced loaded value.
1713        Args.push_back(CreateCoercedLoad(SrcPtr, ArgInfo.getCoerceToType(),
1714                                         *this));
1715
1716        // Validate argument match.
1717        checkArgMatches(Args.back(), IRArgNo, IRFuncTy);
1718      }
1719
1720      break;
1721    }
1722
1723    case ABIArgInfo::Expand:
1724      ExpandTypeToArgs(I->Ty, RV, Args, IRFuncTy);
1725      IRArgNo = Args.size();
1726      break;
1727    }
1728  }
1729
1730  // If the callee is a bitcast of a function to a varargs pointer to function
1731  // type, check to see if we can remove the bitcast.  This handles some cases
1732  // with unprototyped functions.
1733  if (llvm::ConstantExpr *CE = dyn_cast<llvm::ConstantExpr>(Callee))
1734    if (llvm::Function *CalleeF = dyn_cast<llvm::Function>(CE->getOperand(0))) {
1735      llvm::PointerType *CurPT=cast<llvm::PointerType>(Callee->getType());
1736      llvm::FunctionType *CurFT =
1737        cast<llvm::FunctionType>(CurPT->getElementType());
1738      llvm::FunctionType *ActualFT = CalleeF->getFunctionType();
1739
1740      if (CE->getOpcode() == llvm::Instruction::BitCast &&
1741          ActualFT->getReturnType() == CurFT->getReturnType() &&
1742          ActualFT->getNumParams() == CurFT->getNumParams() &&
1743          ActualFT->getNumParams() == Args.size() &&
1744          (CurFT->isVarArg() || !ActualFT->isVarArg())) {
1745        bool ArgsMatch = true;
1746        for (unsigned i = 0, e = ActualFT->getNumParams(); i != e; ++i)
1747          if (ActualFT->getParamType(i) != CurFT->getParamType(i)) {
1748            ArgsMatch = false;
1749            break;
1750          }
1751
1752        // Strip the cast if we can get away with it.  This is a nice cleanup,
1753        // but also allows us to inline the function at -O0 if it is marked
1754        // always_inline.
1755        if (ArgsMatch)
1756          Callee = CalleeF;
1757      }
1758    }
1759
1760  unsigned CallingConv;
1761  CodeGen::AttributeListType AttributeList;
1762  CGM.ConstructAttributeList(CallInfo, TargetDecl, AttributeList, CallingConv);
1763  llvm::AttrListPtr Attrs = llvm::AttrListPtr::get(AttributeList.begin(),
1764                                                   AttributeList.end());
1765
1766  llvm::BasicBlock *InvokeDest = 0;
1767  if (!(Attrs.getFnAttributes() & llvm::Attribute::NoUnwind))
1768    InvokeDest = getInvokeDest();
1769
1770  llvm::CallSite CS;
1771  if (!InvokeDest) {
1772    CS = Builder.CreateCall(Callee, Args);
1773  } else {
1774    llvm::BasicBlock *Cont = createBasicBlock("invoke.cont");
1775    CS = Builder.CreateInvoke(Callee, Cont, InvokeDest, Args);
1776    EmitBlock(Cont);
1777  }
1778  if (callOrInvoke)
1779    *callOrInvoke = CS.getInstruction();
1780
1781  CS.setAttributes(Attrs);
1782  CS.setCallingConv(static_cast<llvm::CallingConv::ID>(CallingConv));
1783
1784  // If the call doesn't return, finish the basic block and clear the
1785  // insertion point; this allows the rest of IRgen to discard
1786  // unreachable code.
1787  if (CS.doesNotReturn()) {
1788    Builder.CreateUnreachable();
1789    Builder.ClearInsertionPoint();
1790
1791    // FIXME: For now, emit a dummy basic block because expr emitters in
1792    // generally are not ready to handle emitting expressions at unreachable
1793    // points.
1794    EnsureInsertPoint();
1795
1796    // Return a reasonable RValue.
1797    return GetUndefRValue(RetTy);
1798  }
1799
1800  llvm::Instruction *CI = CS.getInstruction();
1801  if (Builder.isNamePreserving() && !CI->getType()->isVoidTy())
1802    CI->setName("call");
1803
1804  // Emit any writebacks immediately.  Arguably this should happen
1805  // after any return-value munging.
1806  if (CallArgs.hasWritebacks())
1807    emitWritebacks(*this, CallArgs);
1808
1809  switch (RetAI.getKind()) {
1810  case ABIArgInfo::Indirect: {
1811    unsigned Alignment = getContext().getTypeAlignInChars(RetTy).getQuantity();
1812    if (RetTy->isAnyComplexType())
1813      return RValue::getComplex(LoadComplexFromAddr(Args[0], false));
1814    if (CodeGenFunction::hasAggregateLLVMType(RetTy))
1815      return RValue::getAggregate(Args[0]);
1816    return RValue::get(EmitLoadOfScalar(Args[0], false, Alignment, RetTy));
1817  }
1818
1819  case ABIArgInfo::Ignore:
1820    // If we are ignoring an argument that had a result, make sure to
1821    // construct the appropriate return value for our caller.
1822    return GetUndefRValue(RetTy);
1823
1824  case ABIArgInfo::Extend:
1825  case ABIArgInfo::Direct: {
1826    llvm::Type *RetIRTy = ConvertType(RetTy);
1827    if (RetAI.getCoerceToType() == RetIRTy && RetAI.getDirectOffset() == 0) {
1828      if (RetTy->isAnyComplexType()) {
1829        llvm::Value *Real = Builder.CreateExtractValue(CI, 0);
1830        llvm::Value *Imag = Builder.CreateExtractValue(CI, 1);
1831        return RValue::getComplex(std::make_pair(Real, Imag));
1832      }
1833      if (CodeGenFunction::hasAggregateLLVMType(RetTy)) {
1834        llvm::Value *DestPtr = ReturnValue.getValue();
1835        bool DestIsVolatile = ReturnValue.isVolatile();
1836
1837        if (!DestPtr) {
1838          DestPtr = CreateMemTemp(RetTy, "agg.tmp");
1839          DestIsVolatile = false;
1840        }
1841        BuildAggStore(*this, CI, DestPtr, DestIsVolatile, false);
1842        return RValue::getAggregate(DestPtr);
1843      }
1844
1845      // If the argument doesn't match, perform a bitcast to coerce it.  This
1846      // can happen due to trivial type mismatches.
1847      llvm::Value *V = CI;
1848      if (V->getType() != RetIRTy)
1849        V = Builder.CreateBitCast(V, RetIRTy);
1850      return RValue::get(V);
1851    }
1852
1853    llvm::Value *DestPtr = ReturnValue.getValue();
1854    bool DestIsVolatile = ReturnValue.isVolatile();
1855
1856    if (!DestPtr) {
1857      DestPtr = CreateMemTemp(RetTy, "coerce");
1858      DestIsVolatile = false;
1859    }
1860
1861    // If the value is offset in memory, apply the offset now.
1862    llvm::Value *StorePtr = DestPtr;
1863    if (unsigned Offs = RetAI.getDirectOffset()) {
1864      StorePtr = Builder.CreateBitCast(StorePtr, Builder.getInt8PtrTy());
1865      StorePtr = Builder.CreateConstGEP1_32(StorePtr, Offs);
1866      StorePtr = Builder.CreateBitCast(StorePtr,
1867                         llvm::PointerType::getUnqual(RetAI.getCoerceToType()));
1868    }
1869    CreateCoercedStore(CI, StorePtr, DestIsVolatile, *this);
1870
1871    unsigned Alignment = getContext().getTypeAlignInChars(RetTy).getQuantity();
1872    if (RetTy->isAnyComplexType())
1873      return RValue::getComplex(LoadComplexFromAddr(DestPtr, false));
1874    if (CodeGenFunction::hasAggregateLLVMType(RetTy))
1875      return RValue::getAggregate(DestPtr);
1876    return RValue::get(EmitLoadOfScalar(DestPtr, false, Alignment, RetTy));
1877  }
1878
1879  case ABIArgInfo::Expand:
1880    llvm_unreachable("Invalid ABI kind for return argument");
1881  }
1882
1883  llvm_unreachable("Unhandled ABIArgInfo::Kind");
1884}
1885
1886/* VarArg handling */
1887
1888llvm::Value *CodeGenFunction::EmitVAArg(llvm::Value *VAListAddr, QualType Ty) {
1889  return CGM.getTypes().getABIInfo().EmitVAArg(VAListAddr, Ty, *this);
1890}
1891