CGCall.cpp revision ce93399f26f23735b8e291321f18ad54f64cb58a 
1//===----- CGCall.h - Encapsulate calling convention details ----*- C++ -*-===//
2//
3//                     The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10// These classes wrap the information about a call or function
11// definition used to handle ABI compliancy.
12//
13//===----------------------------------------------------------------------===//
14
15#include "CGCall.h"
16#include "ABIInfo.h"
17#include "CodeGenFunction.h"
18#include "CodeGenModule.h"
19#include "clang/Basic/TargetInfo.h"
20#include "clang/AST/Decl.h"
21#include "clang/AST/DeclCXX.h"
22#include "clang/AST/DeclObjC.h"
23#include "clang/Frontend/CodeGenOptions.h"
24#include "llvm/Attributes.h"
25#include "llvm/Support/CallSite.h"
26#include "llvm/Target/TargetData.h"
27using namespace clang;
28using namespace CodeGen;
29
30/***/
31
32static unsigned ClangCallConvToLLVMCallConv(CallingConv CC) {
33  switch (CC) {
34  default: return llvm::CallingConv::C;
35  case CC_X86StdCall: return llvm::CallingConv::X86_StdCall;
36  case CC_X86FastCall: return llvm::CallingConv::X86_FastCall;
37  case CC_X86ThisCall: return llvm::CallingConv::X86_ThisCall;
38  }
39}
40
41/// Derives the 'this' type for codegen purposes, i.e. ignoring method
42/// qualification.
43/// FIXME: address space qualification?
44static CanQualType GetThisType(ASTContext &Context, const CXXRecordDecl *RD) {
45  QualType RecTy = Context.getTagDeclType(RD)->getCanonicalTypeInternal();
46  return Context.getPointerType(CanQualType::CreateUnsafe(RecTy));
47}
48
49/// Returns the canonical formal type of the given C++ method.
50static CanQual<FunctionProtoType> GetFormalType(const CXXMethodDecl *MD) {
51  return MD->getType()->getCanonicalTypeUnqualified()
52           .getAs<FunctionProtoType>();
53}
54
55/// Returns the "extra-canonicalized" return type, which discards
56/// qualifiers on the return type.  Codegen doesn't care about them,
57/// and it makes ABI code a little easier to be able to assume that
58/// all parameter and return types are top-level unqualified.
59static CanQualType GetReturnType(QualType RetTy) {
60  return RetTy->getCanonicalTypeUnqualified().getUnqualifiedType();
61}
62
63const CGFunctionInfo &
64CodeGenTypes::getFunctionInfo(CanQual<FunctionNoProtoType> FTNP) {
65  return getFunctionInfo(FTNP->getResultType().getUnqualifiedType(),
66                         llvm::SmallVector<CanQualType, 16>(),
67                         FTNP->getExtInfo());
68}
69
70/// \param Args - contains any initial parameters besides those
71///   in the formal type
72static const CGFunctionInfo &getFunctionInfo(CodeGenTypes &CGT,
73                                  llvm::SmallVectorImpl<CanQualType> &ArgTys,
74                                             CanQual<FunctionProtoType> FTP) {
75  // FIXME: Kill copy.
76  for (unsigned i = 0, e = FTP->getNumArgs(); i != e; ++i)
77    ArgTys.push_back(FTP->getArgType(i));
78  CanQualType ResTy = FTP->getResultType().getUnqualifiedType();
79  return CGT.getFunctionInfo(ResTy, ArgTys, FTP->getExtInfo());
80}
81
82const CGFunctionInfo &
83CodeGenTypes::getFunctionInfo(CanQual<FunctionProtoType> FTP) {
84  llvm::SmallVector<CanQualType, 16> ArgTys;
85  return ::getFunctionInfo(*this, ArgTys, FTP);
86}
87
88static CallingConv getCallingConventionForDecl(const Decl *D) {
89  // Set the appropriate calling convention for the Function.
90  if (D->hasAttr<StdCallAttr>())
91    return CC_X86StdCall;
92
93  if (D->hasAttr<FastCallAttr>())
94    return CC_X86FastCall;
95
96  if (D->hasAttr<ThisCallAttr>())
97    return CC_X86ThisCall;
98
99  return CC_C;
100}
101
102const CGFunctionInfo &CodeGenTypes::getFunctionInfo(const CXXRecordDecl *RD,
103                                                 const FunctionProtoType *FTP) {
104  llvm::SmallVector<CanQualType, 16> ArgTys;
105
106  // Add the 'this' pointer.
107  ArgTys.push_back(GetThisType(Context, RD));
108
109  return ::getFunctionInfo(*this, ArgTys,
110              FTP->getCanonicalTypeUnqualified().getAs<FunctionProtoType>());
111}
112
113const CGFunctionInfo &CodeGenTypes::getFunctionInfo(const CXXMethodDecl *MD) {
114  llvm::SmallVector<CanQualType, 16> ArgTys;
115
116  // Add the 'this' pointer unless this is a static method.
117  if (MD->isInstance())
118    ArgTys.push_back(GetThisType(Context, MD->getParent()));
119
120  return ::getFunctionInfo(*this, ArgTys, GetFormalType(MD));
121}
122
123const CGFunctionInfo &CodeGenTypes::getFunctionInfo(const CXXConstructorDecl *D,
124                                                    CXXCtorType Type) {
125  llvm::SmallVector<CanQualType, 16> ArgTys;
126
127  // Add the 'this' pointer.
128  ArgTys.push_back(GetThisType(Context, D->getParent()));
129
130  // Check if we need to add a VTT parameter (which has type void **).
131  if (Type == Ctor_Base && D->getParent()->getNumVBases() != 0)
132    ArgTys.push_back(Context.getPointerType(Context.VoidPtrTy));
133
134  return ::getFunctionInfo(*this, ArgTys, GetFormalType(D));
135}
136
137const CGFunctionInfo &CodeGenTypes::getFunctionInfo(const CXXDestructorDecl *D,
138                                                    CXXDtorType Type) {
139  llvm::SmallVector<CanQualType, 16> ArgTys;
140
141  // Add the 'this' pointer.
142  ArgTys.push_back(GetThisType(Context, D->getParent()));
143
144  // Check if we need to add a VTT parameter (which has type void **).
145  if (Type == Dtor_Base && D->getParent()->getNumVBases() != 0)
146    ArgTys.push_back(Context.getPointerType(Context.VoidPtrTy));
147
148  return ::getFunctionInfo(*this, ArgTys, GetFormalType(D));
149}
150
151const CGFunctionInfo &CodeGenTypes::getFunctionInfo(const FunctionDecl *FD) {
152  if (const CXXMethodDecl *MD = dyn_cast<CXXMethodDecl>(FD))
153    if (MD->isInstance())
154      return getFunctionInfo(MD);
155
156  CanQualType FTy = FD->getType()->getCanonicalTypeUnqualified();
157  assert(isa<FunctionType>(FTy));
158  if (isa<FunctionNoProtoType>(FTy))
159    return getFunctionInfo(FTy.getAs<FunctionNoProtoType>());
160  assert(isa<FunctionProtoType>(FTy));
161  return getFunctionInfo(FTy.getAs<FunctionProtoType>());
162}
163
164const CGFunctionInfo &CodeGenTypes::getFunctionInfo(const ObjCMethodDecl *MD) {
165  llvm::SmallVector<CanQualType, 16> ArgTys;
166  ArgTys.push_back(Context.getCanonicalParamType(MD->getSelfDecl()->getType()));
167  ArgTys.push_back(Context.getCanonicalParamType(Context.getObjCSelType()));
168  // FIXME: Kill copy?
169  for (ObjCMethodDecl::param_iterator i = MD->param_begin(),
170         e = MD->param_end(); i != e; ++i) {
171    ArgTys.push_back(Context.getCanonicalParamType((*i)->getType()));
172  }
173  return getFunctionInfo(GetReturnType(MD->getResultType()),
174                         ArgTys,
175                         FunctionType::ExtInfo(
176                             /*NoReturn*/ false,
177                             /*RegParm*/ 0,
178                             getCallingConventionForDecl(MD)));
179}
180
181const CGFunctionInfo &CodeGenTypes::getFunctionInfo(GlobalDecl GD) {
182  // FIXME: Do we need to handle ObjCMethodDecl?
183  const FunctionDecl *FD = cast<FunctionDecl>(GD.getDecl());
184
185  if (const CXXConstructorDecl *CD = dyn_cast<CXXConstructorDecl>(FD))
186    return getFunctionInfo(CD, GD.getCtorType());
187
188  if (const CXXDestructorDecl *DD = dyn_cast<CXXDestructorDecl>(FD))
189    return getFunctionInfo(DD, GD.getDtorType());
190
191  return getFunctionInfo(FD);
192}
193
194const CGFunctionInfo &CodeGenTypes::getFunctionInfo(QualType ResTy,
195                                                    const CallArgList &Args,
196                                            const FunctionType::ExtInfo &Info) {
197  // FIXME: Kill copy.
198  llvm::SmallVector<CanQualType, 16> ArgTys;
199  for (CallArgList::const_iterator i = Args.begin(), e = Args.end();
200       i != e; ++i)
201    ArgTys.push_back(Context.getCanonicalParamType(i->second));
202  return getFunctionInfo(GetReturnType(ResTy), ArgTys, Info);
203}
204
205const CGFunctionInfo &CodeGenTypes::getFunctionInfo(QualType ResTy,
206                                                    const FunctionArgList &Args,
207                                            const FunctionType::ExtInfo &Info) {
208  // FIXME: Kill copy.
209  llvm::SmallVector<CanQualType, 16> ArgTys;
210  for (FunctionArgList::const_iterator i = Args.begin(), e = Args.end();
211       i != e; ++i)
212    ArgTys.push_back(Context.getCanonicalParamType(i->second));
213  return getFunctionInfo(GetReturnType(ResTy), ArgTys, Info);
214}
215
216const CGFunctionInfo &CodeGenTypes::getFunctionInfo(CanQualType ResTy,
217                           const llvm::SmallVectorImpl<CanQualType> &ArgTys,
218                                            const FunctionType::ExtInfo &Info) {
219#ifndef NDEBUG
220  for (llvm::SmallVectorImpl<CanQualType>::const_iterator
221         I = ArgTys.begin(), E = ArgTys.end(); I != E; ++I)
222    assert(I->isCanonicalAsParam());
223#endif
224
225  unsigned CC = ClangCallConvToLLVMCallConv(Info.getCC());
226
227  // Lookup or create unique function info.
228  llvm::FoldingSetNodeID ID;
229  CGFunctionInfo::Profile(ID, Info, ResTy,
230                          ArgTys.begin(), ArgTys.end());
231
232  void *InsertPos = 0;
233  CGFunctionInfo *FI = FunctionInfos.FindNodeOrInsertPos(ID, InsertPos);
234  if (FI)
235    return *FI;
236
237  // Construct the function info.
238  FI = new CGFunctionInfo(CC, Info.getNoReturn(), Info.getRegParm(), ResTy,
239                          ArgTys);
240  FunctionInfos.InsertNode(FI, InsertPos);
241
242  // ABI lowering wants to know what our preferred type for the argument is in
243  // various situations, pass it in.
244  llvm::SmallVector<const llvm::Type *, 8> PreferredArgTypes;
245  for (llvm::SmallVectorImpl<CanQualType>::const_iterator
246       I = ArgTys.begin(), E = ArgTys.end(); I != E; ++I)
247    PreferredArgTypes.push_back(ConvertType(*I));
248
249  // Compute ABI information.
250  getABIInfo().computeInfo(*FI, getContext(), TheModule.getContext(),
251                           PreferredArgTypes.data(), PreferredArgTypes.size());
252
253  return *FI;
254}
255
256CGFunctionInfo::CGFunctionInfo(unsigned _CallingConvention,
257                               bool _NoReturn,
258                               unsigned _RegParm,
259                               CanQualType ResTy,
260                               const llvm::SmallVectorImpl<CanQualType> &ArgTys)
261  : CallingConvention(_CallingConvention),
262    EffectiveCallingConvention(_CallingConvention),
263    NoReturn(_NoReturn), RegParm(_RegParm)
264{
265  NumArgs = ArgTys.size();
266
267  // FIXME: Coallocate with the CGFunctionInfo object.
268  Args = new ArgInfo[1 + NumArgs];
269  Args[0].type = ResTy;
270  for (unsigned i = 0; i < NumArgs; ++i)
271    Args[1 + i].type = ArgTys[i];
272}
273
274/***/
275
276void CodeGenTypes::GetExpandedTypes(QualType Ty,
277                                    std::vector<const llvm::Type*> &ArgTys) {
278  const RecordType *RT = Ty->getAsStructureType();
279  assert(RT && "Can only expand structure types.");
280  const RecordDecl *RD = RT->getDecl();
281  assert(!RD->hasFlexibleArrayMember() &&
282         "Cannot expand structure with flexible array.");
283
284  for (RecordDecl::field_iterator i = RD->field_begin(), e = RD->field_end();
285         i != e; ++i) {
286    const FieldDecl *FD = *i;
287    assert(!FD->isBitField() &&
288           "Cannot expand structure with bit-field members.");
289
290    QualType FT = FD->getType();
291    if (CodeGenFunction::hasAggregateLLVMType(FT)) {
292      GetExpandedTypes(FT, ArgTys);
293    } else {
294      ArgTys.push_back(ConvertType(FT));
295    }
296  }
297}
298
299llvm::Function::arg_iterator
300CodeGenFunction::ExpandTypeFromArgs(QualType Ty, LValue LV,
301                                    llvm::Function::arg_iterator AI) {
302  const RecordType *RT = Ty->getAsStructureType();
303  assert(RT && "Can only expand structure types.");
304
305  RecordDecl *RD = RT->getDecl();
306  assert(LV.isSimple() &&
307         "Unexpected non-simple lvalue during struct expansion.");
308  llvm::Value *Addr = LV.getAddress();
309  for (RecordDecl::field_iterator i = RD->field_begin(), e = RD->field_end();
310         i != e; ++i) {
311    FieldDecl *FD = *i;
312    QualType FT = FD->getType();
313
314    // FIXME: What are the right qualifiers here?
315    LValue LV = EmitLValueForField(Addr, FD, 0);
316    if (CodeGenFunction::hasAggregateLLVMType(FT)) {
317      AI = ExpandTypeFromArgs(FT, LV, AI);
318    } else {
319      EmitStoreThroughLValue(RValue::get(AI), LV, FT);
320      ++AI;
321    }
322  }
323
324  return AI;
325}
326
327void
328CodeGenFunction::ExpandTypeToArgs(QualType Ty, RValue RV,
329                                  llvm::SmallVector<llvm::Value*, 16> &Args) {
330  const RecordType *RT = Ty->getAsStructureType();
331  assert(RT && "Can only expand structure types.");
332
333  RecordDecl *RD = RT->getDecl();
334  assert(RV.isAggregate() && "Unexpected rvalue during struct expansion");
335  llvm::Value *Addr = RV.getAggregateAddr();
336  for (RecordDecl::field_iterator i = RD->field_begin(), e = RD->field_end();
337         i != e; ++i) {
338    FieldDecl *FD = *i;
339    QualType FT = FD->getType();
340
341    // FIXME: What are the right qualifiers here?
342    LValue LV = EmitLValueForField(Addr, FD, 0);
343    if (CodeGenFunction::hasAggregateLLVMType(FT)) {
344      ExpandTypeToArgs(FT, RValue::getAggregate(LV.getAddress()), Args);
345    } else {
346      RValue RV = EmitLoadOfLValue(LV, FT);
347      assert(RV.isScalar() &&
348             "Unexpected non-scalar rvalue during struct expansion.");
349      Args.push_back(RV.getScalarVal());
350    }
351  }
352}
353
354/// EnterStructPointerForCoercedAccess - Given a struct pointer that we are
355/// accessing some number of bytes out of it, try to gep into the struct to get
356/// at its inner goodness.  Dive as deep as possible without entering an element
357/// with an in-memory size smaller than DstSize.
358static llvm::Value *
359EnterStructPointerForCoercedAccess(llvm::Value *SrcPtr,
360                                   const llvm::StructType *SrcSTy,
361                                   uint64_t DstSize, CodeGenFunction &CGF) {
362  // We can't dive into a zero-element struct.
363  if (SrcSTy->getNumElements() == 0) return SrcPtr;
364
365  const llvm::Type *FirstElt = SrcSTy->getElementType(0);
366
367  // If the first elt is at least as large as what we're looking for, or if the
368  // first element is the same size as the whole struct, we can enter it.
369  uint64_t FirstEltSize =
370    CGF.CGM.getTargetData().getTypeAllocSize(FirstElt);
371  if (FirstEltSize < DstSize &&
372      FirstEltSize < CGF.CGM.getTargetData().getTypeAllocSize(SrcSTy))
373    return SrcPtr;
374
375  // GEP into the first element.
376  SrcPtr = CGF.Builder.CreateConstGEP2_32(SrcPtr, 0, 0, "coerce.dive");
377
378  // If the first element is a struct, recurse.
379  const llvm::Type *SrcTy =
380    cast<llvm::PointerType>(SrcPtr->getType())->getElementType();
381  if (const llvm::StructType *SrcSTy = dyn_cast<llvm::StructType>(SrcTy))
382    return EnterStructPointerForCoercedAccess(SrcPtr, SrcSTy, DstSize, CGF);
383
384  return SrcPtr;
385}
386
387/// CoerceIntOrPtrToIntOrPtr - Convert a value Val to the specific Ty where both
388/// are either integers or pointers.  This does a truncation of the value if it
389/// is too large or a zero extension if it is too small.
390static llvm::Value *CoerceIntOrPtrToIntOrPtr(llvm::Value *Val,
391                                             const llvm::Type *Ty,
392                                             CodeGenFunction &CGF) {
393  if (Val->getType() == Ty)
394    return Val;
395
396  if (isa<llvm::PointerType>(Val->getType())) {
397    // If this is Pointer->Pointer avoid conversion to and from int.
398    if (isa<llvm::PointerType>(Ty))
399      return CGF.Builder.CreateBitCast(Val, Ty, "coerce.val");
400
401    // Convert the pointer to an integer so we can play with its width.
402    Val = CGF.Builder.CreatePtrToInt(Val, CGF.IntPtrTy, "coerce.val.pi");
403  }
404
405  const llvm::Type *DestIntTy = Ty;
406  if (isa<llvm::PointerType>(DestIntTy))
407    DestIntTy = CGF.IntPtrTy;
408
409  if (Val->getType() != DestIntTy)
410    Val = CGF.Builder.CreateIntCast(Val, DestIntTy, false, "coerce.val.ii");
411
412  if (isa<llvm::PointerType>(Ty))
413    Val = CGF.Builder.CreateIntToPtr(Val, Ty, "coerce.val.ip");
414  return Val;
415}
416
417
418
419/// CreateCoercedLoad - Create a load from \arg SrcPtr interpreted as
420/// a pointer to an object of type \arg Ty.
421///
422/// This safely handles the case when the src type is smaller than the
423/// destination type; in this situation the values of bits which not
424/// present in the src are undefined.
425static llvm::Value *CreateCoercedLoad(llvm::Value *SrcPtr,
426                                      const llvm::Type *Ty,
427                                      CodeGenFunction &CGF) {
428  const llvm::Type *SrcTy =
429    cast<llvm::PointerType>(SrcPtr->getType())->getElementType();
430
431  // If SrcTy and Ty are the same, just do a load.
432  if (SrcTy == Ty)
433    return CGF.Builder.CreateLoad(SrcPtr);
434
435  uint64_t DstSize = CGF.CGM.getTargetData().getTypeAllocSize(Ty);
436
437  if (const llvm::StructType *SrcSTy = dyn_cast<llvm::StructType>(SrcTy)) {
438    SrcPtr = EnterStructPointerForCoercedAccess(SrcPtr, SrcSTy, DstSize, CGF);
439    SrcTy = cast<llvm::PointerType>(SrcPtr->getType())->getElementType();
440  }
441
442  uint64_t SrcSize = CGF.CGM.getTargetData().getTypeAllocSize(SrcTy);
443
444  // If the source and destination are integer or pointer types, just do an
445  // extension or truncation to the desired type.
446  if ((isa<llvm::IntegerType>(Ty) || isa<llvm::PointerType>(Ty)) &&
447      (isa<llvm::IntegerType>(SrcTy) || isa<llvm::PointerType>(SrcTy))) {
448    llvm::LoadInst *Load = CGF.Builder.CreateLoad(SrcPtr);
449    return CoerceIntOrPtrToIntOrPtr(Load, Ty, CGF);
450  }
451
452  // If load is legal, just bitcast the src pointer.
453  if (SrcSize >= DstSize) {
454    // Generally SrcSize is never greater than DstSize, since this means we are
455    // losing bits. However, this can happen in cases where the structure has
456    // additional padding, for example due to a user specified alignment.
457    //
458    // FIXME: Assert that we aren't truncating non-padding bits when have access
459    // to that information.
460    llvm::Value *Casted =
461      CGF.Builder.CreateBitCast(SrcPtr, llvm::PointerType::getUnqual(Ty));
462    llvm::LoadInst *Load = CGF.Builder.CreateLoad(Casted);
463    // FIXME: Use better alignment / avoid requiring aligned load.
464    Load->setAlignment(1);
465    return Load;
466  }
467
468  // Otherwise do coercion through memory. This is stupid, but
469  // simple.
470  llvm::Value *Tmp = CGF.CreateTempAlloca(Ty);
471  llvm::Value *Casted =
472    CGF.Builder.CreateBitCast(Tmp, llvm::PointerType::getUnqual(SrcTy));
473  llvm::StoreInst *Store =
474    CGF.Builder.CreateStore(CGF.Builder.CreateLoad(SrcPtr), Casted);
475  // FIXME: Use better alignment / avoid requiring aligned store.
476  Store->setAlignment(1);
477  return CGF.Builder.CreateLoad(Tmp);
478}
479
480/// CreateCoercedStore - Create a store to \arg DstPtr from \arg Src,
481/// where the source and destination may have different types.
482///
483/// This safely handles the case when the src type is larger than the
484/// destination type; the upper bits of the src will be lost.
485static void CreateCoercedStore(llvm::Value *Src,
486                               llvm::Value *DstPtr,
487                               bool DstIsVolatile,
488                               CodeGenFunction &CGF) {
489  const llvm::Type *SrcTy = Src->getType();
490  const llvm::Type *DstTy =
491    cast<llvm::PointerType>(DstPtr->getType())->getElementType();
492  if (SrcTy == DstTy) {
493    CGF.Builder.CreateStore(Src, DstPtr, DstIsVolatile);
494    return;
495  }
496
497  uint64_t SrcSize = CGF.CGM.getTargetData().getTypeAllocSize(SrcTy);
498
499  if (const llvm::StructType *DstSTy = dyn_cast<llvm::StructType>(DstTy)) {
500    DstPtr = EnterStructPointerForCoercedAccess(DstPtr, DstSTy, SrcSize, CGF);
501    DstTy = cast<llvm::PointerType>(DstPtr->getType())->getElementType();
502  }
503
504  // If the source and destination are integer or pointer types, just do an
505  // extension or truncation to the desired type.
506  if ((isa<llvm::IntegerType>(SrcTy) || isa<llvm::PointerType>(SrcTy)) &&
507      (isa<llvm::IntegerType>(DstTy) || isa<llvm::PointerType>(DstTy))) {
508    Src = CoerceIntOrPtrToIntOrPtr(Src, DstTy, CGF);
509    CGF.Builder.CreateStore(Src, DstPtr, DstIsVolatile);
510    return;
511  }
512
513  uint64_t DstSize = CGF.CGM.getTargetData().getTypeAllocSize(DstTy);
514
515  // If store is legal, just bitcast the src pointer.
516  if (SrcSize <= DstSize) {
517    llvm::Value *Casted =
518      CGF.Builder.CreateBitCast(DstPtr, llvm::PointerType::getUnqual(SrcTy));
519    // FIXME: Use better alignment / avoid requiring aligned store.
520    CGF.Builder.CreateStore(Src, Casted, DstIsVolatile)->setAlignment(1);
521  } else {
522    // Otherwise do coercion through memory. This is stupid, but
523    // simple.
524
525    // Generally SrcSize is never greater than DstSize, since this means we are
526    // losing bits. However, this can happen in cases where the structure has
527    // additional padding, for example due to a user specified alignment.
528    //
529    // FIXME: Assert that we aren't truncating non-padding bits when have access
530    // to that information.
531    llvm::Value *Tmp = CGF.CreateTempAlloca(SrcTy);
532    CGF.Builder.CreateStore(Src, Tmp);
533    llvm::Value *Casted =
534      CGF.Builder.CreateBitCast(Tmp, llvm::PointerType::getUnqual(DstTy));
535    llvm::LoadInst *Load = CGF.Builder.CreateLoad(Casted);
536    // FIXME: Use better alignment / avoid requiring aligned load.
537    Load->setAlignment(1);
538    CGF.Builder.CreateStore(Load, DstPtr, DstIsVolatile);
539  }
540}
541
542/***/
543
544bool CodeGenModule::ReturnTypeUsesSret(const CGFunctionInfo &FI) {
545  return FI.getReturnInfo().isIndirect();
546}
547
548const llvm::FunctionType *CodeGenTypes::GetFunctionType(GlobalDecl GD) {
549  const CGFunctionInfo &FI = getFunctionInfo(GD);
550
551  // For definition purposes, don't consider a K&R function variadic.
552  bool Variadic = false;
553  if (const FunctionProtoType *FPT =
554        cast<FunctionDecl>(GD.getDecl())->getType()->getAs<FunctionProtoType>())
555    Variadic = FPT->isVariadic();
556
557  return GetFunctionType(FI, Variadic);
558}
559
560const llvm::FunctionType *
561CodeGenTypes::GetFunctionType(const CGFunctionInfo &FI, bool IsVariadic) {
562  std::vector<const llvm::Type*> ArgTys;
563
564  const llvm::Type *ResultType = 0;
565
566  QualType RetTy = FI.getReturnType();
567  const ABIArgInfo &RetAI = FI.getReturnInfo();
568  switch (RetAI.getKind()) {
569  case ABIArgInfo::Expand:
570    assert(0 && "Invalid ABI kind for return argument");
571
572  case ABIArgInfo::Extend:
573  case ABIArgInfo::Direct:
574    ResultType = ConvertType(RetTy);
575    break;
576
577  case ABIArgInfo::Indirect: {
578    assert(!RetAI.getIndirectAlign() && "Align unused on indirect return.");
579    ResultType = llvm::Type::getVoidTy(getLLVMContext());
580    const llvm::Type *STy = ConvertType(RetTy);
581    ArgTys.push_back(llvm::PointerType::get(STy, RetTy.getAddressSpace()));
582    break;
583  }
584
585  case ABIArgInfo::Ignore:
586    ResultType = llvm::Type::getVoidTy(getLLVMContext());
587    break;
588
589  case ABIArgInfo::Coerce:
590    ResultType = RetAI.getCoerceToType();
591    break;
592  }
593
594  for (CGFunctionInfo::const_arg_iterator it = FI.arg_begin(),
595         ie = FI.arg_end(); it != ie; ++it) {
596    const ABIArgInfo &AI = it->info;
597
598    switch (AI.getKind()) {
599    case ABIArgInfo::Ignore:
600      break;
601
602    case ABIArgInfo::Coerce: {
603      // If the coerce-to type is a first class aggregate, flatten it.  Either
604      // way is semantically identical, but fast-isel and the optimizer
605      // generally likes scalar values better than FCAs.
606      const llvm::Type *ArgTy = AI.getCoerceToType();
607      if (const llvm::StructType *STy = dyn_cast<llvm::StructType>(ArgTy)) {
608        for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i)
609          ArgTys.push_back(STy->getElementType(i));
610      } else {
611        ArgTys.push_back(ArgTy);
612      }
613      break;
614    }
615
616    case ABIArgInfo::Indirect: {
617      // indirect arguments are always on the stack, which is addr space #0.
618      const llvm::Type *LTy = ConvertTypeForMem(it->type);
619      ArgTys.push_back(llvm::PointerType::getUnqual(LTy));
620      break;
621    }
622
623    case ABIArgInfo::Extend:
624    case ABIArgInfo::Direct:
625      ArgTys.push_back(ConvertType(it->type));
626      break;
627
628    case ABIArgInfo::Expand:
629      GetExpandedTypes(it->type, ArgTys);
630      break;
631    }
632  }
633
634  return llvm::FunctionType::get(ResultType, ArgTys, IsVariadic);
635}
636
637const llvm::Type *
638CodeGenTypes::GetFunctionTypeForVTable(const CXXMethodDecl *MD) {
639  const FunctionProtoType *FPT = MD->getType()->getAs<FunctionProtoType>();
640
641  if (!VerifyFuncTypeComplete(FPT))
642    return GetFunctionType(getFunctionInfo(MD), FPT->isVariadic());
643
644  return llvm::OpaqueType::get(getLLVMContext());
645}
646
647void CodeGenModule::ConstructAttributeList(const CGFunctionInfo &FI,
648                                           const Decl *TargetDecl,
649                                           AttributeListType &PAL,
650                                           unsigned &CallingConv) {
651  unsigned FuncAttrs = 0;
652  unsigned RetAttrs = 0;
653
654  CallingConv = FI.getEffectiveCallingConvention();
655
656  if (FI.isNoReturn())
657    FuncAttrs |= llvm::Attribute::NoReturn;
658
659  // FIXME: handle sseregparm someday...
660  if (TargetDecl) {
661    if (TargetDecl->hasAttr<NoThrowAttr>())
662      FuncAttrs |= llvm::Attribute::NoUnwind;
663    if (TargetDecl->hasAttr<NoReturnAttr>())
664      FuncAttrs |= llvm::Attribute::NoReturn;
665    if (TargetDecl->hasAttr<ConstAttr>())
666      FuncAttrs |= llvm::Attribute::ReadNone;
667    else if (TargetDecl->hasAttr<PureAttr>())
668      FuncAttrs |= llvm::Attribute::ReadOnly;
669    if (TargetDecl->hasAttr<MallocAttr>())
670      RetAttrs |= llvm::Attribute::NoAlias;
671  }
672
673  if (CodeGenOpts.OptimizeSize)
674    FuncAttrs |= llvm::Attribute::OptimizeForSize;
675  if (CodeGenOpts.DisableRedZone)
676    FuncAttrs |= llvm::Attribute::NoRedZone;
677  if (CodeGenOpts.NoImplicitFloat)
678    FuncAttrs |= llvm::Attribute::NoImplicitFloat;
679
680  QualType RetTy = FI.getReturnType();
681  unsigned Index = 1;
682  const ABIArgInfo &RetAI = FI.getReturnInfo();
683  switch (RetAI.getKind()) {
684  case ABIArgInfo::Extend:
685   if (RetTy->isSignedIntegerType()) {
686     RetAttrs |= llvm::Attribute::SExt;
687   } else if (RetTy->isUnsignedIntegerType()) {
688     RetAttrs |= llvm::Attribute::ZExt;
689   }
690   // FALLTHROUGH
691  case ABIArgInfo::Direct:
692    break;
693
694  case ABIArgInfo::Indirect:
695    PAL.push_back(llvm::AttributeWithIndex::get(Index,
696                                                llvm::Attribute::StructRet));
697    ++Index;
698    // sret disables readnone and readonly
699    FuncAttrs &= ~(llvm::Attribute::ReadOnly |
700                   llvm::Attribute::ReadNone);
701    break;
702
703  case ABIArgInfo::Ignore:
704  case ABIArgInfo::Coerce:
705    break;
706
707  case ABIArgInfo::Expand:
708    assert(0 && "Invalid ABI kind for return argument");
709  }
710
711  if (RetAttrs)
712    PAL.push_back(llvm::AttributeWithIndex::get(0, RetAttrs));
713
714  // FIXME: we need to honour command line settings also...
715  // FIXME: RegParm should be reduced in case of nested functions and/or global
716  // register variable.
717  signed RegParm = FI.getRegParm();
718
719  unsigned PointerWidth = getContext().Target.getPointerWidth(0);
720  for (CGFunctionInfo::const_arg_iterator it = FI.arg_begin(),
721         ie = FI.arg_end(); it != ie; ++it) {
722    QualType ParamType = it->type;
723    const ABIArgInfo &AI = it->info;
724    unsigned Attributes = 0;
725
726    // 'restrict' -> 'noalias' is done in EmitFunctionProlog when we
727    // have the corresponding parameter variable.  It doesn't make
728    // sense to do it here because parameters are so fucked up.
729
730    switch (AI.getKind()) {
731    case ABIArgInfo::Coerce:
732      if (const llvm::StructType *STy =
733          dyn_cast<llvm::StructType>(AI.getCoerceToType()))
734        Index += STy->getNumElements();
735      else
736        ++Index;
737      continue;  // Skip index increment.
738
739    case ABIArgInfo::Indirect:
740      if (AI.getIndirectByVal())
741        Attributes |= llvm::Attribute::ByVal;
742
743      Attributes |=
744        llvm::Attribute::constructAlignmentFromInt(AI.getIndirectAlign());
745      // byval disables readnone and readonly.
746      FuncAttrs &= ~(llvm::Attribute::ReadOnly |
747                     llvm::Attribute::ReadNone);
748      break;
749
750    case ABIArgInfo::Extend:
751     if (ParamType->isSignedIntegerType()) {
752       Attributes |= llvm::Attribute::SExt;
753     } else if (ParamType->isUnsignedIntegerType()) {
754       Attributes |= llvm::Attribute::ZExt;
755     }
756     // FALLS THROUGH
757    case ABIArgInfo::Direct:
758      if (RegParm > 0 &&
759          (ParamType->isIntegerType() || ParamType->isPointerType())) {
760        RegParm -=
761          (Context.getTypeSize(ParamType) + PointerWidth - 1) / PointerWidth;
762        if (RegParm >= 0)
763          Attributes |= llvm::Attribute::InReg;
764      }
765      // FIXME: handle sseregparm someday...
766      break;
767
768    case ABIArgInfo::Ignore:
769      // Skip increment, no matching LLVM parameter.
770      continue;
771
772    case ABIArgInfo::Expand: {
773      std::vector<const llvm::Type*> Tys;
774      // FIXME: This is rather inefficient. Do we ever actually need to do
775      // anything here? The result should be just reconstructed on the other
776      // side, so extension should be a non-issue.
777      getTypes().GetExpandedTypes(ParamType, Tys);
778      Index += Tys.size();
779      continue;
780    }
781    }
782
783    if (Attributes)
784      PAL.push_back(llvm::AttributeWithIndex::get(Index, Attributes));
785    ++Index;
786  }
787  if (FuncAttrs)
788    PAL.push_back(llvm::AttributeWithIndex::get(~0, FuncAttrs));
789}
790
791void CodeGenFunction::EmitFunctionProlog(const CGFunctionInfo &FI,
792                                         llvm::Function *Fn,
793                                         const FunctionArgList &Args) {
794  // If this is an implicit-return-zero function, go ahead and
795  // initialize the return value.  TODO: it might be nice to have
796  // a more general mechanism for this that didn't require synthesized
797  // return statements.
798  if (const FunctionDecl* FD = dyn_cast_or_null<FunctionDecl>(CurFuncDecl)) {
799    if (FD->hasImplicitReturnZero()) {
800      QualType RetTy = FD->getResultType().getUnqualifiedType();
801      const llvm::Type* LLVMTy = CGM.getTypes().ConvertType(RetTy);
802      llvm::Constant* Zero = llvm::Constant::getNullValue(LLVMTy);
803      Builder.CreateStore(Zero, ReturnValue);
804    }
805  }
806
807  // FIXME: We no longer need the types from FunctionArgList; lift up and
808  // simplify.
809
810  // Emit allocs for param decls.  Give the LLVM Argument nodes names.
811  llvm::Function::arg_iterator AI = Fn->arg_begin();
812
813  // Name the struct return argument.
814  if (CGM.ReturnTypeUsesSret(FI)) {
815    AI->setName("agg.result");
816    ++AI;
817  }
818
819  assert(FI.arg_size() == Args.size() &&
820         "Mismatch between function signature & arguments.");
821  CGFunctionInfo::const_arg_iterator info_it = FI.arg_begin();
822  for (FunctionArgList::const_iterator i = Args.begin(), e = Args.end();
823       i != e; ++i, ++info_it) {
824    const VarDecl *Arg = i->first;
825    QualType Ty = info_it->type;
826    const ABIArgInfo &ArgI = info_it->info;
827
828    switch (ArgI.getKind()) {
829    case ABIArgInfo::Indirect: {
830      llvm::Value *V = AI;
831      if (hasAggregateLLVMType(Ty)) {
832        // Do nothing, aggregates and complex variables are accessed by
833        // reference.
834      } else {
835        // Load scalar value from indirect argument.
836        V = EmitLoadOfScalar(V, false, Ty);
837        if (!getContext().typesAreCompatible(Ty, Arg->getType())) {
838          // This must be a promotion, for something like
839          // "void a(x) short x; {..."
840          V = EmitScalarConversion(V, Ty, Arg->getType());
841        }
842      }
843      EmitParmDecl(*Arg, V);
844      break;
845    }
846
847    case ABIArgInfo::Extend:
848    case ABIArgInfo::Direct: {
849      assert(AI != Fn->arg_end() && "Argument mismatch!");
850      llvm::Value *V = AI;
851      if (hasAggregateLLVMType(Ty)) {
852        // Create a temporary alloca to hold the argument; the rest of
853        // codegen expects to access aggregates & complex values by
854        // reference.
855        V = CreateMemTemp(Ty);
856        Builder.CreateStore(AI, V);
857      } else {
858        if (Arg->getType().isRestrictQualified())
859          AI->addAttr(llvm::Attribute::NoAlias);
860
861        if (!getContext().typesAreCompatible(Ty, Arg->getType())) {
862          // This must be a promotion, for something like
863          // "void a(x) short x; {..."
864          V = EmitScalarConversion(V, Ty, Arg->getType());
865        }
866      }
867      EmitParmDecl(*Arg, V);
868      break;
869    }
870
871    case ABIArgInfo::Expand: {
872      // If this structure was expanded into multiple arguments then
873      // we need to create a temporary and reconstruct it from the
874      // arguments.
875      llvm::Value *Temp = CreateMemTemp(Ty, Arg->getName() + ".addr");
876      // FIXME: What are the right qualifiers here?
877      llvm::Function::arg_iterator End =
878        ExpandTypeFromArgs(Ty, LValue::MakeAddr(Temp, Qualifiers()), AI);
879      EmitParmDecl(*Arg, Temp);
880
881      // Name the arguments used in expansion and increment AI.
882      unsigned Index = 0;
883      for (; AI != End; ++AI, ++Index)
884        AI->setName(Arg->getName() + "." + llvm::Twine(Index));
885      continue;
886    }
887
888    case ABIArgInfo::Ignore:
889      // Initialize the local variable appropriately.
890      if (hasAggregateLLVMType(Ty)) {
891        EmitParmDecl(*Arg, CreateMemTemp(Ty));
892      } else {
893        EmitParmDecl(*Arg, llvm::UndefValue::get(ConvertType(Arg->getType())));
894      }
895
896      // Skip increment, no matching LLVM parameter.
897      continue;
898
899    case ABIArgInfo::Coerce: {
900      // FIXME: This is very wasteful; EmitParmDecl is just going to drop the
901      // result in a new alloca anyway, so we could just store into that
902      // directly if we broke the abstraction down more.
903      llvm::Value *V = CreateMemTemp(Ty, "coerce");
904
905      // If the coerce-to type is a first class aggregate, we flatten it and
906      // pass the elements. Either way is semantically identical, but fast-isel
907      // and the optimizer generally likes scalar values better than FCAs.
908      if (const llvm::StructType *STy =
909            dyn_cast<llvm::StructType>(ArgI.getCoerceToType())) {
910        // If the argument and alloca types match up, we don't have to build the
911        // FCA at all, emit a series of GEPs and stores, which is better for
912        // fast isel.
913        if (STy == cast<llvm::PointerType>(V->getType())->getElementType()) {
914          for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) {
915            assert(AI != Fn->arg_end() && "Argument mismatch!");
916            AI->setName(Arg->getName() + ".coerce" + llvm::Twine(i));
917            llvm::Value *EltPtr = Builder.CreateConstGEP2_32(V, 0, i);
918            Builder.CreateStore(AI++, EltPtr);
919          }
920        } else {
921          // Reconstruct the FCA here so we can do a coerced store.
922          llvm::Value *FormalArg = llvm::UndefValue::get(STy);
923          for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) {
924            assert(AI != Fn->arg_end() && "Argument mismatch!");
925            AI->setName(Arg->getName() + ".coerce" + llvm::Twine(i));
926            FormalArg = Builder.CreateInsertValue(FormalArg, AI++, i);
927          }
928          CreateCoercedStore(FormalArg, V, /*DestIsVolatile=*/false, *this);
929        }
930      } else {
931        // Simple case, just do a coerced store of the argument into the alloca.
932        assert(AI != Fn->arg_end() && "Argument mismatch!");
933        AI->setName(Arg->getName() + ".coerce");
934        CreateCoercedStore(AI++, V, /*DestIsVolatile=*/false, *this);
935      }
936
937
938      // Match to what EmitParmDecl is expecting for this type.
939      if (!CodeGenFunction::hasAggregateLLVMType(Ty)) {
940        V = EmitLoadOfScalar(V, false, Ty);
941        if (!getContext().typesAreCompatible(Ty, Arg->getType())) {
942          // This must be a promotion, for something like
943          // "void a(x) short x; {..."
944          V = EmitScalarConversion(V, Ty, Arg->getType());
945        }
946      }
947      EmitParmDecl(*Arg, V);
948      continue;  // Skip ++AI increment, already done.
949    }
950    }
951
952    ++AI;
953  }
954  assert(AI == Fn->arg_end() && "Argument mismatch!");
955}
956
957void CodeGenFunction::EmitFunctionEpilog(const CGFunctionInfo &FI) {
958  // Functions with no result always return void.
959  if (ReturnValue == 0) {
960    Builder.CreateRetVoid();
961    return;
962  }
963
964  llvm::Value *RV = 0;
965  QualType RetTy = FI.getReturnType();
966  const ABIArgInfo &RetAI = FI.getReturnInfo();
967
968  switch (RetAI.getKind()) {
969  case ABIArgInfo::Indirect:
970    if (RetTy->isAnyComplexType()) {
971      ComplexPairTy RT = LoadComplexFromAddr(ReturnValue, false);
972      StoreComplexToAddr(RT, CurFn->arg_begin(), false);
973    } else if (CodeGenFunction::hasAggregateLLVMType(RetTy)) {
974      // Do nothing; aggregrates get evaluated directly into the destination.
975    } else {
976      EmitStoreOfScalar(Builder.CreateLoad(ReturnValue), CurFn->arg_begin(),
977                        false, RetTy);
978    }
979    break;
980
981  case ABIArgInfo::Extend:
982  case ABIArgInfo::Direct: {
983    // The internal return value temp always will have pointer-to-return-type
984    // type, just do a load.
985
986    // If the instruction right before the insertion point is a store to the
987    // return value, we can elide the load, zap the store, and usually zap the
988    // alloca.
989    llvm::BasicBlock *InsertBB = Builder.GetInsertBlock();
990    llvm::StoreInst *SI = 0;
991    if (InsertBB->empty() ||
992        !(SI = dyn_cast<llvm::StoreInst>(&InsertBB->back())) ||
993        SI->getPointerOperand() != ReturnValue || SI->isVolatile()) {
994      RV = Builder.CreateLoad(ReturnValue);
995    } else {
996      // Get the stored value and nuke the now-dead store.
997      RV = SI->getValueOperand();
998      SI->eraseFromParent();
999
1000      // If that was the only use of the return value, nuke it as well now.
1001      if (ReturnValue->use_empty() && isa<llvm::AllocaInst>(ReturnValue)) {
1002        cast<llvm::AllocaInst>(ReturnValue)->eraseFromParent();
1003        ReturnValue = 0;
1004      }
1005    }
1006    break;
1007  }
1008  case ABIArgInfo::Ignore:
1009    break;
1010
1011  case ABIArgInfo::Coerce:
1012    RV = CreateCoercedLoad(ReturnValue, RetAI.getCoerceToType(), *this);
1013    break;
1014
1015  case ABIArgInfo::Expand:
1016    assert(0 && "Invalid ABI kind for return argument");
1017  }
1018
1019  if (RV)
1020    Builder.CreateRet(RV);
1021  else
1022    Builder.CreateRetVoid();
1023}
1024
1025RValue CodeGenFunction::EmitDelegateCallArg(const VarDecl *Param) {
1026  // StartFunction converted the ABI-lowered parameter(s) into a
1027  // local alloca.  We need to turn that into an r-value suitable
1028  // for EmitCall.
1029  llvm::Value *Local = GetAddrOfLocalVar(Param);
1030
1031  QualType ArgType = Param->getType();
1032
1033  // For the most part, we just need to load the alloca, except:
1034  // 1) aggregate r-values are actually pointers to temporaries, and
1035  // 2) references to aggregates are pointers directly to the aggregate.
1036  // I don't know why references to non-aggregates are different here.
1037  if (const ReferenceType *RefType = ArgType->getAs<ReferenceType>()) {
1038    if (hasAggregateLLVMType(RefType->getPointeeType()))
1039      return RValue::getAggregate(Local);
1040
1041    // Locals which are references to scalars are represented
1042    // with allocas holding the pointer.
1043    return RValue::get(Builder.CreateLoad(Local));
1044  }
1045
1046  if (ArgType->isAnyComplexType())
1047    return RValue::getComplex(LoadComplexFromAddr(Local, /*volatile*/ false));
1048
1049  if (hasAggregateLLVMType(ArgType))
1050    return RValue::getAggregate(Local);
1051
1052  return RValue::get(EmitLoadOfScalar(Local, false, ArgType));
1053}
1054
1055RValue CodeGenFunction::EmitCallArg(const Expr *E, QualType ArgType) {
1056  if (ArgType->isReferenceType())
1057    return EmitReferenceBindingToExpr(E, /*InitializedDecl=*/0);
1058
1059  return EmitAnyExprToTemp(E);
1060}
1061
1062RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
1063                                 llvm::Value *Callee,
1064                                 ReturnValueSlot ReturnValue,
1065                                 const CallArgList &CallArgs,
1066                                 const Decl *TargetDecl,
1067                                 llvm::Instruction **callOrInvoke) {
1068  // FIXME: We no longer need the types from CallArgs; lift up and simplify.
1069  llvm::SmallVector<llvm::Value*, 16> Args;
1070
1071  // Handle struct-return functions by passing a pointer to the
1072  // location that we would like to return into.
1073  QualType RetTy = CallInfo.getReturnType();
1074  const ABIArgInfo &RetAI = CallInfo.getReturnInfo();
1075
1076
1077  // If the call returns a temporary with struct return, create a temporary
1078  // alloca to hold the result, unless one is given to us.
1079  if (CGM.ReturnTypeUsesSret(CallInfo)) {
1080    llvm::Value *Value = ReturnValue.getValue();
1081    if (!Value)
1082      Value = CreateMemTemp(RetTy);
1083    Args.push_back(Value);
1084  }
1085
1086  assert(CallInfo.arg_size() == CallArgs.size() &&
1087         "Mismatch between function signature & arguments.");
1088  CGFunctionInfo::const_arg_iterator info_it = CallInfo.arg_begin();
1089  for (CallArgList::const_iterator I = CallArgs.begin(), E = CallArgs.end();
1090       I != E; ++I, ++info_it) {
1091    const ABIArgInfo &ArgInfo = info_it->info;
1092    RValue RV = I->first;
1093
1094    switch (ArgInfo.getKind()) {
1095    case ABIArgInfo::Indirect:
1096      if (RV.isScalar() || RV.isComplex()) {
1097        // Make a temporary alloca to pass the argument.
1098        Args.push_back(CreateMemTemp(I->second));
1099        if (RV.isScalar())
1100          EmitStoreOfScalar(RV.getScalarVal(), Args.back(), false, I->second);
1101        else
1102          StoreComplexToAddr(RV.getComplexVal(), Args.back(), false);
1103      } else {
1104        Args.push_back(RV.getAggregateAddr());
1105      }
1106      break;
1107
1108    case ABIArgInfo::Extend:
1109    case ABIArgInfo::Direct:
1110      if (RV.isScalar()) {
1111        Args.push_back(RV.getScalarVal());
1112      } else if (RV.isComplex()) {
1113        llvm::Value *Tmp = llvm::UndefValue::get(ConvertType(I->second));
1114        Tmp = Builder.CreateInsertValue(Tmp, RV.getComplexVal().first, 0);
1115        Tmp = Builder.CreateInsertValue(Tmp, RV.getComplexVal().second, 1);
1116        Args.push_back(Tmp);
1117      } else {
1118        Args.push_back(Builder.CreateLoad(RV.getAggregateAddr()));
1119      }
1120      break;
1121
1122    case ABIArgInfo::Ignore:
1123      break;
1124
1125    case ABIArgInfo::Coerce: {
1126      // FIXME: Avoid the conversion through memory if possible.
1127      llvm::Value *SrcPtr;
1128      if (RV.isScalar()) {
1129        SrcPtr = CreateMemTemp(I->second, "coerce");
1130        EmitStoreOfScalar(RV.getScalarVal(), SrcPtr, false, I->second);
1131      } else if (RV.isComplex()) {
1132        SrcPtr = CreateMemTemp(I->second, "coerce");
1133        StoreComplexToAddr(RV.getComplexVal(), SrcPtr, false);
1134      } else
1135        SrcPtr = RV.getAggregateAddr();
1136
1137      // If the coerce-to type is a first class aggregate, we flatten it and
1138      // pass the elements. Either way is semantically identical, but fast-isel
1139      // and the optimizer generally likes scalar values better than FCAs.
1140      if (const llvm::StructType *STy =
1141            dyn_cast<llvm::StructType>(ArgInfo.getCoerceToType())) {
1142        // If the argument and alloca types match up, we don't have to build the
1143        // FCA at all, emit a series of GEPs and loads, which is better for
1144        // fast isel.
1145        if (STy ==cast<llvm::PointerType>(SrcPtr->getType())->getElementType()){
1146          for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) {
1147            llvm::Value *EltPtr = Builder.CreateConstGEP2_32(SrcPtr, 0, i);
1148            Args.push_back(Builder.CreateLoad(EltPtr));
1149          }
1150        } else {
1151          // Otherwise, do a coerced load the entire FCA and handle the pieces.
1152          llvm::Value *SrcVal =
1153            CreateCoercedLoad(SrcPtr, ArgInfo.getCoerceToType(), *this);
1154
1155          // Extract the elements of the value to pass in.
1156          for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i)
1157            Args.push_back(Builder.CreateExtractValue(SrcVal, i));
1158        }
1159      } else {
1160        // In the simple case, just pass the coerced loaded value.
1161        Args.push_back(CreateCoercedLoad(SrcPtr, ArgInfo.getCoerceToType(),
1162                                         *this));
1163      }
1164
1165      break;
1166    }
1167
1168    case ABIArgInfo::Expand:
1169      ExpandTypeToArgs(I->second, RV, Args);
1170      break;
1171    }
1172  }
1173
1174  // If the callee is a bitcast of a function to a varargs pointer to function
1175  // type, check to see if we can remove the bitcast.  This handles some cases
1176  // with unprototyped functions.
1177  if (llvm::ConstantExpr *CE = dyn_cast<llvm::ConstantExpr>(Callee))
1178    if (llvm::Function *CalleeF = dyn_cast<llvm::Function>(CE->getOperand(0))) {
1179      const llvm::PointerType *CurPT=cast<llvm::PointerType>(Callee->getType());
1180      const llvm::FunctionType *CurFT =
1181        cast<llvm::FunctionType>(CurPT->getElementType());
1182      const llvm::FunctionType *ActualFT = CalleeF->getFunctionType();
1183
1184      if (CE->getOpcode() == llvm::Instruction::BitCast &&
1185          ActualFT->getReturnType() == CurFT->getReturnType() &&
1186          ActualFT->getNumParams() == CurFT->getNumParams() &&
1187          ActualFT->getNumParams() == Args.size()) {
1188        bool ArgsMatch = true;
1189        for (unsigned i = 0, e = ActualFT->getNumParams(); i != e; ++i)
1190          if (ActualFT->getParamType(i) != CurFT->getParamType(i)) {
1191            ArgsMatch = false;
1192            break;
1193          }
1194
1195        // Strip the cast if we can get away with it.  This is a nice cleanup,
1196        // but also allows us to inline the function at -O0 if it is marked
1197        // always_inline.
1198        if (ArgsMatch)
1199          Callee = CalleeF;
1200      }
1201    }
1202
1203
1204  llvm::BasicBlock *InvokeDest = getInvokeDest();
1205  unsigned CallingConv;
1206  CodeGen::AttributeListType AttributeList;
1207  CGM.ConstructAttributeList(CallInfo, TargetDecl, AttributeList, CallingConv);
1208  llvm::AttrListPtr Attrs = llvm::AttrListPtr::get(AttributeList.begin(),
1209                                                   AttributeList.end());
1210
1211  llvm::CallSite CS;
1212  if (!InvokeDest || (Attrs.getFnAttributes() & llvm::Attribute::NoUnwind)) {
1213    CS = Builder.CreateCall(Callee, Args.data(), Args.data()+Args.size());
1214  } else {
1215    llvm::BasicBlock *Cont = createBasicBlock("invoke.cont");
1216    CS = Builder.CreateInvoke(Callee, Cont, InvokeDest,
1217                              Args.data(), Args.data()+Args.size());
1218    EmitBlock(Cont);
1219  }
1220  if (callOrInvoke)
1221    *callOrInvoke = CS.getInstruction();
1222
1223  CS.setAttributes(Attrs);
1224  CS.setCallingConv(static_cast<llvm::CallingConv::ID>(CallingConv));
1225
1226  // If the call doesn't return, finish the basic block and clear the
1227  // insertion point; this allows the rest of IRgen to discard
1228  // unreachable code.
1229  if (CS.doesNotReturn()) {
1230    Builder.CreateUnreachable();
1231    Builder.ClearInsertionPoint();
1232
1233    // FIXME: For now, emit a dummy basic block because expr emitters in
1234    // generally are not ready to handle emitting expressions at unreachable
1235    // points.
1236    EnsureInsertPoint();
1237
1238    // Return a reasonable RValue.
1239    return GetUndefRValue(RetTy);
1240  }
1241
1242  llvm::Instruction *CI = CS.getInstruction();
1243  if (Builder.isNamePreserving() && !CI->getType()->isVoidTy())
1244    CI->setName("call");
1245
1246  switch (RetAI.getKind()) {
1247  case ABIArgInfo::Indirect:
1248    if (RetTy->isAnyComplexType())
1249      return RValue::getComplex(LoadComplexFromAddr(Args[0], false));
1250    if (CodeGenFunction::hasAggregateLLVMType(RetTy))
1251      return RValue::getAggregate(Args[0]);
1252    return RValue::get(EmitLoadOfScalar(Args[0], false, RetTy));
1253
1254  case ABIArgInfo::Extend:
1255  case ABIArgInfo::Direct:
1256    if (RetTy->isAnyComplexType()) {
1257      llvm::Value *Real = Builder.CreateExtractValue(CI, 0);
1258      llvm::Value *Imag = Builder.CreateExtractValue(CI, 1);
1259      return RValue::getComplex(std::make_pair(Real, Imag));
1260    }
1261    if (CodeGenFunction::hasAggregateLLVMType(RetTy)) {
1262      llvm::Value *DestPtr = ReturnValue.getValue();
1263      bool DestIsVolatile = ReturnValue.isVolatile();
1264
1265      if (!DestPtr) {
1266        DestPtr = CreateMemTemp(RetTy, "agg.tmp");
1267        DestIsVolatile = false;
1268      }
1269      Builder.CreateStore(CI, DestPtr, DestIsVolatile);
1270      return RValue::getAggregate(DestPtr);
1271    }
1272    return RValue::get(CI);
1273
1274  case ABIArgInfo::Ignore:
1275    // If we are ignoring an argument that had a result, make sure to
1276    // construct the appropriate return value for our caller.
1277    return GetUndefRValue(RetTy);
1278
1279  case ABIArgInfo::Coerce: {
1280    llvm::Value *DestPtr = ReturnValue.getValue();
1281    bool DestIsVolatile = ReturnValue.isVolatile();
1282
1283    if (!DestPtr) {
1284      DestPtr = CreateMemTemp(RetTy, "coerce");
1285      DestIsVolatile = false;
1286    }
1287
1288    CreateCoercedStore(CI, DestPtr, DestIsVolatile, *this);
1289    if (RetTy->isAnyComplexType())
1290      return RValue::getComplex(LoadComplexFromAddr(DestPtr, false));
1291    if (CodeGenFunction::hasAggregateLLVMType(RetTy))
1292      return RValue::getAggregate(DestPtr);
1293    return RValue::get(EmitLoadOfScalar(DestPtr, false, RetTy));
1294  }
1295
1296  case ABIArgInfo::Expand:
1297    assert(0 && "Invalid ABI kind for return argument");
1298  }
1299
1300  assert(0 && "Unhandled ABIArgInfo::Kind");
1301  return RValue::get(0);
1302}
1303
1304/* VarArg handling */
1305
1306llvm::Value *CodeGenFunction::EmitVAArg(llvm::Value *VAListAddr, QualType Ty) {
1307  return CGM.getTypes().getABIInfo().EmitVAArg(VAListAddr, Ty, *this);
1308}
1309