CGCall.cpp revision 24a9f6e11d222f2d9feaf5f9605c1a66006f7061 
1//===----- CGCall.h - Encapsulate calling convention details ----*- C++ -*-===//
2//
3//                     The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10// These classes wrap the information about a call or function
11// definition used to handle ABI compliancy.
12//
13//===----------------------------------------------------------------------===//
14
15#include "CGCall.h"
16#include "CodeGenFunction.h"
17#include "CodeGenModule.h"
18#include "clang/Basic/TargetInfo.h"
19#include "clang/AST/ASTContext.h"
20#include "clang/AST/Decl.h"
21#include "clang/AST/DeclObjC.h"
22#include "clang/AST/RecordLayout.h"
23#include "llvm/ADT/StringExtras.h"
24#include "llvm/Attributes.h"
25#include "llvm/Support/CommandLine.h"
26#include "llvm/Support/MathExtras.h"
27#include "llvm/Support/raw_ostream.h"
28#include "llvm/Target/TargetData.h"
29
30#include "ABIInfo.h"
31
32using namespace clang;
33using namespace CodeGen;
34
35/***/
36
37// FIXME: Use iterator and sidestep silly type array creation.
38
39const
40CGFunctionInfo &CodeGenTypes::getFunctionInfo(const FunctionTypeNoProto *FTNP) {
41  return getFunctionInfo(FTNP->getResultType(),
42                         llvm::SmallVector<QualType, 16>());
43}
44
45const
46CGFunctionInfo &CodeGenTypes::getFunctionInfo(const FunctionTypeProto *FTP) {
47  llvm::SmallVector<QualType, 16> ArgTys;
48  // FIXME: Kill copy.
49  for (unsigned i = 0, e = FTP->getNumArgs(); i != e; ++i)
50    ArgTys.push_back(FTP->getArgType(i));
51  return getFunctionInfo(FTP->getResultType(), ArgTys);
52}
53
54const CGFunctionInfo &CodeGenTypes::getFunctionInfo(const FunctionDecl *FD) {
55  const FunctionType *FTy = FD->getType()->getAsFunctionType();
56  if (const FunctionTypeProto *FTP = dyn_cast<FunctionTypeProto>(FTy))
57    return getFunctionInfo(FTP);
58  return getFunctionInfo(cast<FunctionTypeNoProto>(FTy));
59}
60
61const CGFunctionInfo &CodeGenTypes::getFunctionInfo(const ObjCMethodDecl *MD) {
62  llvm::SmallVector<QualType, 16> ArgTys;
63  ArgTys.push_back(MD->getSelfDecl()->getType());
64  ArgTys.push_back(Context.getObjCSelType());
65  // FIXME: Kill copy?
66  for (ObjCMethodDecl::param_iterator i = MD->param_begin(),
67         e = MD->param_end(); i != e; ++i)
68    ArgTys.push_back((*i)->getType());
69  return getFunctionInfo(MD->getResultType(), ArgTys);
70}
71
72const CGFunctionInfo &CodeGenTypes::getFunctionInfo(QualType ResTy,
73                                                    const CallArgList &Args) {
74  // FIXME: Kill copy.
75  llvm::SmallVector<QualType, 16> ArgTys;
76  for (CallArgList::const_iterator i = Args.begin(), e = Args.end();
77       i != e; ++i)
78    ArgTys.push_back(i->second);
79  return getFunctionInfo(ResTy, ArgTys);
80}
81
82const CGFunctionInfo &CodeGenTypes::getFunctionInfo(QualType ResTy,
83                                                  const FunctionArgList &Args) {
84  // FIXME: Kill copy.
85  llvm::SmallVector<QualType, 16> ArgTys;
86  for (FunctionArgList::const_iterator i = Args.begin(), e = Args.end();
87       i != e; ++i)
88    ArgTys.push_back(i->second);
89  return getFunctionInfo(ResTy, ArgTys);
90}
91
92const CGFunctionInfo &CodeGenTypes::getFunctionInfo(QualType ResTy,
93                               const llvm::SmallVector<QualType, 16> &ArgTys) {
94  // Lookup or create unique function info.
95  llvm::FoldingSetNodeID ID;
96  CGFunctionInfo::Profile(ID, ResTy, ArgTys.begin(), ArgTys.end());
97
98  void *InsertPos = 0;
99  CGFunctionInfo *FI = FunctionInfos.FindNodeOrInsertPos(ID, InsertPos);
100  if (FI)
101    return *FI;
102
103  // Construct the function info.
104  FI = new CGFunctionInfo(ResTy, ArgTys);
105  FunctionInfos.InsertNode(FI, InsertPos);
106
107  // Compute ABI information.
108  getABIInfo().computeInfo(*FI, getContext());
109
110  return *FI;
111}
112
113/***/
114
115ABIInfo::~ABIInfo() {}
116
117void ABIArgInfo::dump() const {
118  fprintf(stderr, "(ABIArgInfo Kind=");
119  switch (TheKind) {
120  case Direct:
121    fprintf(stderr, "Direct");
122    break;
123  case Ignore:
124    fprintf(stderr, "Ignore");
125    break;
126  case Coerce:
127    fprintf(stderr, "Coerce Type=");
128    getCoerceToType()->print(llvm::errs());
129    // FIXME: This is ridiculous.
130    llvm::errs().flush();
131    break;
132  case Indirect:
133    fprintf(stderr, "Indirect Align=%d", getIndirectAlign());
134    break;
135  case Expand:
136    fprintf(stderr, "Expand");
137    break;
138  }
139  fprintf(stderr, ")\n");
140}
141
142/***/
143
144/// isEmptyStruct - Return true iff a structure has no non-empty
145/// members. Note that a structure with a flexible array member is not
146/// considered empty.
147static bool isEmptyStruct(QualType T) {
148  const RecordType *RT = T->getAsStructureType();
149  if (!RT)
150    return 0;
151  const RecordDecl *RD = RT->getDecl();
152  if (RD->hasFlexibleArrayMember())
153    return false;
154  for (RecordDecl::field_iterator i = RD->field_begin(),
155         e = RD->field_end(); i != e; ++i) {
156    const FieldDecl *FD = *i;
157    if (!isEmptyStruct(FD->getType()))
158      return false;
159  }
160  return true;
161}
162
163/// isSingleElementStruct - Determine if a structure is a "single
164/// element struct", i.e. it has exactly one non-empty field or
165/// exactly one field which is itself a single element
166/// struct. Structures with flexible array members are never
167/// considered single element structs.
168///
169/// \return The field declaration for the single non-empty field, if
170/// it exists.
171static const FieldDecl *isSingleElementStruct(QualType T) {
172  const RecordType *RT = T->getAsStructureType();
173  if (!RT)
174    return 0;
175
176  const RecordDecl *RD = RT->getDecl();
177  if (RD->hasFlexibleArrayMember())
178    return 0;
179
180  const FieldDecl *Found = 0;
181  for (RecordDecl::field_iterator i = RD->field_begin(),
182         e = RD->field_end(); i != e; ++i) {
183    const FieldDecl *FD = *i;
184    QualType FT = FD->getType();
185
186    if (isEmptyStruct(FT)) {
187      // Ignore
188    } else if (Found) {
189      return 0;
190    } else if (!CodeGenFunction::hasAggregateLLVMType(FT)) {
191      Found = FD;
192    } else {
193      Found = isSingleElementStruct(FT);
194      if (!Found)
195        return 0;
196    }
197  }
198
199  return Found;
200}
201
202static bool is32Or64BitBasicType(QualType Ty, ASTContext &Context) {
203  if (!Ty->getAsBuiltinType() && !Ty->isPointerType())
204    return false;
205
206  uint64_t Size = Context.getTypeSize(Ty);
207  return Size == 32 || Size == 64;
208}
209
210static bool areAllFields32Or64BitBasicType(const RecordDecl *RD,
211                                           ASTContext &Context) {
212  for (RecordDecl::field_iterator i = RD->field_begin(),
213         e = RD->field_end(); i != e; ++i) {
214    const FieldDecl *FD = *i;
215
216    if (!is32Or64BitBasicType(FD->getType(), Context))
217      return false;
218
219    // If this is a bit-field we need to make sure it is still a
220    // 32-bit or 64-bit type.
221    if (Expr *BW = FD->getBitWidth()) {
222      unsigned Width = BW->getIntegerConstantExprValue(Context).getZExtValue();
223      if (Width <= 16)
224        return false;
225    }
226  }
227  return true;
228}
229
230namespace {
231/// DefaultABIInfo - The default implementation for ABI specific
232/// details. This implementation provides information which results in
233/// self-consistent and sensible LLVM IR generation, but does not
234/// conform to any particular ABI.
235class DefaultABIInfo : public ABIInfo {
236  ABIArgInfo classifyReturnType(QualType RetTy,
237                                ASTContext &Context) const;
238
239  ABIArgInfo classifyArgumentType(QualType RetTy,
240                                  ASTContext &Context) const;
241
242  virtual void computeInfo(CGFunctionInfo &FI, ASTContext &Context) const {
243    FI.getReturnInfo() = classifyReturnType(FI.getReturnType(), Context);
244    for (CGFunctionInfo::arg_iterator it = FI.arg_begin(), ie = FI.arg_end();
245         it != ie; ++it)
246      it->info = classifyArgumentType(it->type, Context);
247  }
248
249  virtual llvm::Value *EmitVAArg(llvm::Value *VAListAddr, QualType Ty,
250                                 CodeGenFunction &CGF) const;
251};
252
253/// X86_32ABIInfo - The X86-32 ABI information.
254class X86_32ABIInfo : public ABIInfo {
255public:
256  ABIArgInfo classifyReturnType(QualType RetTy,
257                                ASTContext &Context) const;
258
259  ABIArgInfo classifyArgumentType(QualType RetTy,
260                                  ASTContext &Context) const;
261
262  virtual void computeInfo(CGFunctionInfo &FI, ASTContext &Context) const {
263    FI.getReturnInfo() = classifyReturnType(FI.getReturnType(), Context);
264    for (CGFunctionInfo::arg_iterator it = FI.arg_begin(), ie = FI.arg_end();
265         it != ie; ++it)
266      it->info = classifyArgumentType(it->type, Context);
267  }
268
269  virtual llvm::Value *EmitVAArg(llvm::Value *VAListAddr, QualType Ty,
270                                 CodeGenFunction &CGF) const;
271};
272}
273
274ABIArgInfo X86_32ABIInfo::classifyReturnType(QualType RetTy,
275                                            ASTContext &Context) const {
276  if (RetTy->isVoidType()) {
277    return ABIArgInfo::getIgnore();
278  } else if (CodeGenFunction::hasAggregateLLVMType(RetTy)) {
279    // Classify "single element" structs as their element type.
280    const FieldDecl *SeltFD = isSingleElementStruct(RetTy);
281    if (SeltFD) {
282      QualType SeltTy = SeltFD->getType()->getDesugaredType();
283      if (const BuiltinType *BT = SeltTy->getAsBuiltinType()) {
284        // FIXME: This is gross, it would be nice if we could just
285        // pass back SeltTy and have clients deal with it. Is it worth
286        // supporting coerce to both LLVM and clang Types?
287        if (BT->isIntegerType()) {
288          uint64_t Size = Context.getTypeSize(SeltTy);
289          return ABIArgInfo::getCoerce(llvm::IntegerType::get((unsigned) Size));
290        } else if (BT->getKind() == BuiltinType::Float) {
291          return ABIArgInfo::getCoerce(llvm::Type::FloatTy);
292        } else if (BT->getKind() == BuiltinType::Double) {
293          return ABIArgInfo::getCoerce(llvm::Type::DoubleTy);
294        }
295      } else if (SeltTy->isPointerType()) {
296        // FIXME: It would be really nice if this could come out as
297        // the proper pointer type.
298        llvm::Type *PtrTy =
299          llvm::PointerType::getUnqual(llvm::Type::Int8Ty);
300        return ABIArgInfo::getCoerce(PtrTy);
301      }
302    }
303
304    uint64_t Size = Context.getTypeSize(RetTy);
305    if (Size == 8) {
306      return ABIArgInfo::getCoerce(llvm::Type::Int8Ty);
307    } else if (Size == 16) {
308      return ABIArgInfo::getCoerce(llvm::Type::Int16Ty);
309    } else if (Size == 32) {
310      return ABIArgInfo::getCoerce(llvm::Type::Int32Ty);
311    } else if (Size == 64) {
312      return ABIArgInfo::getCoerce(llvm::Type::Int64Ty);
313    } else {
314      return ABIArgInfo::getIndirect(0);
315    }
316  } else {
317    return ABIArgInfo::getDirect();
318  }
319}
320
321ABIArgInfo X86_32ABIInfo::classifyArgumentType(QualType Ty,
322                                               ASTContext &Context) const {
323  // FIXME: Set alignment on indirect arguments.
324  if (CodeGenFunction::hasAggregateLLVMType(Ty)) {
325    // Structures with flexible arrays are always indirect.
326    if (const RecordType *RT = Ty->getAsStructureType())
327      if (RT->getDecl()->hasFlexibleArrayMember())
328        return ABIArgInfo::getIndirect(0);
329
330    // Ignore empty structs.
331    uint64_t Size = Context.getTypeSize(Ty);
332    if (Ty->isStructureType() && Size == 0)
333      return ABIArgInfo::getIgnore();
334
335    // Expand structs with size <= 128-bits which consist only of
336    // basic types (int, long long, float, double, xxx*). This is
337    // non-recursive and does not ignore empty fields.
338    if (const RecordType *RT = Ty->getAsStructureType()) {
339      if (Context.getTypeSize(Ty) <= 4*32 &&
340          areAllFields32Or64BitBasicType(RT->getDecl(), Context))
341        return ABIArgInfo::getExpand();
342    }
343
344    return ABIArgInfo::getIndirect(0);
345  } else {
346    return ABIArgInfo::getDirect();
347  }
348}
349
350llvm::Value *X86_32ABIInfo::EmitVAArg(llvm::Value *VAListAddr, QualType Ty,
351                                      CodeGenFunction &CGF) const {
352  const llvm::Type *BP = llvm::PointerType::getUnqual(llvm::Type::Int8Ty);
353  const llvm::Type *BPP = llvm::PointerType::getUnqual(BP);
354
355  CGBuilderTy &Builder = CGF.Builder;
356  llvm::Value *VAListAddrAsBPP = Builder.CreateBitCast(VAListAddr, BPP,
357                                                       "ap");
358  llvm::Value *Addr = Builder.CreateLoad(VAListAddrAsBPP, "ap.cur");
359  llvm::Type *PTy =
360    llvm::PointerType::getUnqual(CGF.ConvertType(Ty));
361  llvm::Value *AddrTyped = Builder.CreateBitCast(Addr, PTy);
362
363  uint64_t Offset =
364    llvm::RoundUpToAlignment(CGF.getContext().getTypeSize(Ty) / 8, 4);
365  llvm::Value *NextAddr =
366    Builder.CreateGEP(Addr,
367                      llvm::ConstantInt::get(llvm::Type::Int32Ty, Offset),
368                      "ap.next");
369  Builder.CreateStore(NextAddr, VAListAddrAsBPP);
370
371  return AddrTyped;
372}
373
374namespace {
375/// X86_64ABIInfo - The X86_64 ABI information.
376class X86_64ABIInfo : public ABIInfo {
377  enum Class {
378    Integer = 0,
379    SSE,
380    SSEUp,
381    X87,
382    X87Up,
383    ComplexX87,
384    NoClass,
385    Memory
386  };
387
388  /// merge - Implement the X86_64 ABI merging algorithm.
389  ///
390  /// Merge an accumulating classification \arg Accum with a field
391  /// classification \arg Field.
392  ///
393  /// \param Accum - The accumulating classification. This should
394  /// always be either NoClass or the result of a previous merge
395  /// call. In addition, this should never be Memory (the caller
396  /// should just return Memory for the aggregate).
397  Class merge(Class Accum, Class Field) const;
398
399  /// classify - Determine the x86_64 register classes in which the
400  /// given type T should be passed.
401  ///
402  /// \param Lo - The classification for the parts of the type
403  /// residing in the low word of the containing object.
404  ///
405  /// \param Hi - The classification for the parts of the type
406  /// residing in the high word of the containing object.
407  ///
408  /// \param OffsetBase - The bit offset of this type in the
409  /// containing object.  Some parameters are classified different
410  /// depending on whether they straddle an eightbyte boundary.
411  ///
412  /// If a word is unused its result will be NoClass; if a type should
413  /// be passed in Memory then at least the classification of \arg Lo
414  /// will be Memory.
415  ///
416  /// The \arg Lo class will be NoClass iff the argument is ignored.
417  ///
418  /// If the \arg Lo class is ComplexX87, then the \arg Hi class will
419  /// also be ComplexX87.
420  void classify(QualType T, ASTContext &Context, uint64_t OffsetBase,
421                Class &Lo, Class &Hi) const;
422
423  /// getCoerceResult - Given a source type \arg Ty and an LLVM type
424  /// to coerce to, chose the best way to pass Ty in the same place
425  /// that \arg CoerceTo would be passed, but while keeping the
426  /// emitted code as simple as possible.
427  ///
428  /// FIXME: Note, this should be cleaned up to just take an
429  /// enumeration of all the ways we might want to pass things,
430  /// instead of constructing an LLVM type. This makes this code more
431  /// explicit, and it makes it clearer that we are also doing this
432  /// for correctness in the case of passing scalar types.
433  ABIArgInfo getCoerceResult(QualType Ty,
434                             const llvm::Type *CoerceTo,
435                             ASTContext &Context) const;
436
437  ABIArgInfo classifyReturnType(QualType RetTy,
438                                ASTContext &Context) const;
439
440  ABIArgInfo classifyArgumentType(QualType Ty,
441                                  ASTContext &Context,
442                                  unsigned &neededInt,
443                                  unsigned &neededSSE) const;
444
445public:
446  virtual void computeInfo(CGFunctionInfo &FI, ASTContext &Context) const;
447
448  virtual llvm::Value *EmitVAArg(llvm::Value *VAListAddr, QualType Ty,
449                                 CodeGenFunction &CGF) const;
450};
451}
452
453X86_64ABIInfo::Class X86_64ABIInfo::merge(Class Accum,
454                                          Class Field) const {
455  // AMD64-ABI 3.2.3p2: Rule 4. Each field of an object is
456  // classified recursively so that always two fields are
457  // considered. The resulting class is calculated according to
458  // the classes of the fields in the eightbyte:
459  //
460  // (a) If both classes are equal, this is the resulting class.
461  //
462  // (b) If one of the classes is NO_CLASS, the resulting class is
463  // the other class.
464  //
465  // (c) If one of the classes is MEMORY, the result is the MEMORY
466  // class.
467  //
468  // (d) If one of the classes is INTEGER, the result is the
469  // INTEGER.
470  //
471  // (e) If one of the classes is X87, X87UP, COMPLEX_X87 class,
472  // MEMORY is used as class.
473  //
474  // (f) Otherwise class SSE is used.
475  assert((Accum == NoClass || Accum == Integer ||
476          Accum == SSE || Accum == SSEUp) &&
477         "Invalid accumulated classification during merge.");
478  if (Accum == Field || Field == NoClass)
479    return Accum;
480  else if (Field == Memory)
481    return Memory;
482  else if (Accum == NoClass)
483    return Field;
484  else if (Accum == Integer || Field == Integer)
485    return Integer;
486  else if (Field == X87 || Field == X87Up || Field == ComplexX87)
487    return Memory;
488  else
489    return SSE;
490}
491
492void X86_64ABIInfo::classify(QualType Ty,
493                             ASTContext &Context,
494                             uint64_t OffsetBase,
495                             Class &Lo, Class &Hi) const {
496  // FIXME: This code can be simplified by introducing a simple value
497  // class for Class pairs with appropriate constructor methods for
498  // the various situations.
499
500  // FIXME: Some of the split computations are wrong; unaligned
501  // vectors shouldn't be passed in registers for example, so there is
502  // no chance they can straddle an eightbyte. Verify & simplify.
503
504  Lo = Hi = NoClass;
505
506  Class &Current = OffsetBase < 64 ? Lo : Hi;
507  Current = Memory;
508
509  if (const BuiltinType *BT = Ty->getAsBuiltinType()) {
510    BuiltinType::Kind k = BT->getKind();
511
512    if (k == BuiltinType::Void) {
513      Current = NoClass;
514    } else if (k >= BuiltinType::Bool && k <= BuiltinType::LongLong) {
515      Current = Integer;
516    } else if (k == BuiltinType::Float || k == BuiltinType::Double) {
517      Current = SSE;
518    } else if (k == BuiltinType::LongDouble) {
519      Lo = X87;
520      Hi = X87Up;
521    }
522    // FIXME: _Decimal32 and _Decimal64 are SSE.
523    // FIXME: _float128 and _Decimal128 are (SSE, SSEUp).
524    // FIXME: __int128 is (Integer, Integer).
525  } else if (const EnumType *ET = Ty->getAsEnumType()) {
526    // Classify the underlying integer type.
527    classify(ET->getDecl()->getIntegerType(), Context, OffsetBase, Lo, Hi);
528  } else if (Ty->isPointerType() || Ty->isReferenceType() ||
529             Ty->isBlockPointerType() || Ty->isObjCQualifiedIdType() ||
530             Ty->isObjCQualifiedInterfaceType()) {
531    Current = Integer;
532  } else if (const VectorType *VT = Ty->getAsVectorType()) {
533    uint64_t Size = Context.getTypeSize(VT);
534    if (Size == 32) {
535      // gcc passes all <4 x char>, <2 x short>, <1 x int>, <1 x
536      // float> as integer.
537      Current = Integer;
538
539      // If this type crosses an eightbyte boundary, it should be
540      // split.
541      uint64_t EB_Real = (OffsetBase) / 64;
542      uint64_t EB_Imag = (OffsetBase + Size - 1) / 64;
543      if (EB_Real != EB_Imag)
544        Hi = Lo;
545    } else if (Size == 64) {
546      // gcc passes <1 x double> in memory. :(
547      if (VT->getElementType()->isSpecificBuiltinType(BuiltinType::Double))
548        return;
549
550      // gcc passes <1 x long long> as INTEGER.
551      if (VT->getElementType()->isSpecificBuiltinType(BuiltinType::LongLong))
552        Current = Integer;
553      else
554        Current = SSE;
555
556      // If this type crosses an eightbyte boundary, it should be
557      // split.
558      if (OffsetBase && OffsetBase != 64)
559        Hi = Lo;
560    } else if (Size == 128) {
561      Lo = SSE;
562      Hi = SSEUp;
563    }
564  } else if (const ComplexType *CT = Ty->getAsComplexType()) {
565    QualType ET = Context.getCanonicalType(CT->getElementType());
566
567    uint64_t Size = Context.getTypeSize(Ty);
568    if (ET->isIntegralType()) {
569      if (Size <= 64)
570        Current = Integer;
571      else if (Size <= 128)
572        Lo = Hi = Integer;
573    } else if (ET == Context.FloatTy)
574      Current = SSE;
575    else if (ET == Context.DoubleTy)
576      Lo = Hi = SSE;
577    else if (ET == Context.LongDoubleTy)
578      Current = ComplexX87;
579
580    // If this complex type crosses an eightbyte boundary then it
581    // should be split.
582    uint64_t EB_Real = (OffsetBase) / 64;
583    uint64_t EB_Imag = (OffsetBase + Context.getTypeSize(ET)) / 64;
584    if (Hi == NoClass && EB_Real != EB_Imag)
585      Hi = Lo;
586  } else if (const ConstantArrayType *AT = Context.getAsConstantArrayType(Ty)) {
587    // Arrays are treated like structures.
588
589    uint64_t Size = Context.getTypeSize(Ty);
590
591    // AMD64-ABI 3.2.3p2: Rule 1. If the size of an object is larger
592    // than two eightbytes, ..., it has class MEMORY.
593    if (Size > 128)
594      return;
595
596    // AMD64-ABI 3.2.3p2: Rule 1. If ..., or it contains unaligned
597    // fields, it has class MEMORY.
598    //
599    // Only need to check alignment of array base.
600    if (OffsetBase % Context.getTypeAlign(AT->getElementType()))
601      return;
602
603    // Otherwise implement simplified merge. We could be smarter about
604    // this, but it isn't worth it and would be harder to verify.
605    Current = NoClass;
606    uint64_t EltSize = Context.getTypeSize(AT->getElementType());
607    uint64_t ArraySize = AT->getSize().getZExtValue();
608    for (uint64_t i=0, Offset=OffsetBase; i<ArraySize; ++i, Offset += EltSize) {
609      Class FieldLo, FieldHi;
610      classify(AT->getElementType(), Context, Offset, FieldLo, FieldHi);
611      Lo = merge(Lo, FieldLo);
612      Hi = merge(Hi, FieldHi);
613      if (Lo == Memory || Hi == Memory)
614        break;
615    }
616
617    // Do post merger cleanup (see below). Only case we worry about is Memory.
618    if (Hi == Memory)
619      Lo = Memory;
620    assert((Hi != SSEUp || Lo == SSE) && "Invalid SSEUp array classification.");
621  } else if (const RecordType *RT = Ty->getAsRecordType()) {
622    uint64_t Size = Context.getTypeSize(Ty);
623
624    // AMD64-ABI 3.2.3p2: Rule 1. If the size of an object is larger
625    // than two eightbytes, ..., it has class MEMORY.
626    if (Size > 128)
627      return;
628
629    const RecordDecl *RD = RT->getDecl();
630
631    // Assume variable sized types are passed in memory.
632    if (RD->hasFlexibleArrayMember())
633      return;
634
635    const ASTRecordLayout &Layout = Context.getASTRecordLayout(RD);
636
637    // Reset Lo class, this will be recomputed.
638    Current = NoClass;
639    unsigned idx = 0;
640    for (RecordDecl::field_iterator i = RD->field_begin(),
641           e = RD->field_end(); i != e; ++i, ++idx) {
642      uint64_t Offset = OffsetBase + Layout.getFieldOffset(idx);
643      bool BitField = i->isBitField();
644
645      // AMD64-ABI 3.2.3p2: Rule 1. If ..., or it contains unaligned
646      // fields, it has class MEMORY.
647      //
648      // Note, skip this test for bitfields, see below.
649      if (!BitField && Offset % Context.getTypeAlign(i->getType())) {
650        Lo = Memory;
651        return;
652      }
653
654      // Classify this field.
655      //
656      // AMD64-ABI 3.2.3p2: Rule 3. If the size of the aggregate
657      // exceeds a single eightbyte, each is classified
658      // separately. Each eightbyte gets initialized to class
659      // NO_CLASS.
660      Class FieldLo, FieldHi;
661
662      // Bitfields require special handling, they do not force the
663      // structure to be passed in memory even if unaligned, and
664      // therefore they can straddle an eightbyte.
665      if (BitField) {
666        uint64_t Offset = OffsetBase + Layout.getFieldOffset(idx);
667        uint64_t Size =
668          i->getBitWidth()->getIntegerConstantExprValue(Context).getZExtValue();
669
670        uint64_t EB_Lo = Offset / 64;
671        uint64_t EB_Hi = (Offset + Size - 1) / 64;
672        FieldLo = FieldHi = NoClass;
673        if (EB_Lo) {
674          assert(EB_Hi == EB_Lo && "Invalid classification, type > 16 bytes.");
675          FieldLo = NoClass;
676          FieldHi = Integer;
677        } else {
678          FieldLo = Integer;
679          FieldHi = EB_Hi ? Integer : NoClass;
680        }
681      } else
682        classify(i->getType(), Context, Offset, FieldLo, FieldHi);
683      Lo = merge(Lo, FieldLo);
684      Hi = merge(Hi, FieldHi);
685      if (Lo == Memory || Hi == Memory)
686        break;
687    }
688
689    // AMD64-ABI 3.2.3p2: Rule 5. Then a post merger cleanup is done:
690    //
691    // (a) If one of the classes is MEMORY, the whole argument is
692    // passed in memory.
693    //
694    // (b) If SSEUP is not preceeded by SSE, it is converted to SSE.
695
696    // The first of these conditions is guaranteed by how we implement
697    // the merge (just bail).
698    //
699    // The second condition occurs in the case of unions; for example
700    // union { _Complex double; unsigned; }.
701    if (Hi == Memory)
702      Lo = Memory;
703    if (Hi == SSEUp && Lo != SSE)
704      Hi = SSE;
705  }
706}
707
708ABIArgInfo X86_64ABIInfo::getCoerceResult(QualType Ty,
709                                          const llvm::Type *CoerceTo,
710                                          ASTContext &Context) const {
711  if (CoerceTo == llvm::Type::Int64Ty) {
712    // Integer and pointer types will end up in a general purpose
713    // register.
714    if (Ty->isIntegralType() || Ty->isPointerType())
715      return ABIArgInfo::getDirect();
716
717  } else if (CoerceTo == llvm::Type::DoubleTy) {
718    // FIXME: It would probably be better to make CGFunctionInfo only
719    // map using canonical types than to canonize here.
720    QualType CTy = Context.getCanonicalType(Ty);
721
722    // Float and double end up in a single SSE reg.
723    if (CTy == Context.FloatTy || CTy == Context.DoubleTy)
724      return ABIArgInfo::getDirect();
725
726  }
727
728  return ABIArgInfo::getCoerce(CoerceTo);
729}
730
731ABIArgInfo X86_64ABIInfo::classifyReturnType(QualType RetTy,
732                                            ASTContext &Context) const {
733  // AMD64-ABI 3.2.3p4: Rule 1. Classify the return type with the
734  // classification algorithm.
735  X86_64ABIInfo::Class Lo, Hi;
736  classify(RetTy, Context, 0, Lo, Hi);
737
738  // Check some invariants.
739  assert((Hi != Memory || Lo == Memory) && "Invalid memory classification.");
740  assert((Lo != NoClass || Hi == NoClass) && "Invalid null classification.");
741  assert((Hi != SSEUp || Lo == SSE) && "Invalid SSEUp classification.");
742
743  const llvm::Type *ResType = 0;
744  switch (Lo) {
745  case NoClass:
746    return ABIArgInfo::getIgnore();
747
748  case SSEUp:
749  case X87Up:
750    assert(0 && "Invalid classification for lo word.");
751
752    // AMD64-ABI 3.2.3p4: Rule 2. Types of class memory are returned via
753    // hidden argument.
754  case Memory:
755    return ABIArgInfo::getIndirect(0);
756
757    // AMD64-ABI 3.2.3p4: Rule 3. If the class is INTEGER, the next
758    // available register of the sequence %rax, %rdx is used.
759  case Integer:
760    ResType = llvm::Type::Int64Ty; break;
761
762    // AMD64-ABI 3.2.3p4: Rule 4. If the class is SSE, the next
763    // available SSE register of the sequence %xmm0, %xmm1 is used.
764  case SSE:
765    ResType = llvm::Type::DoubleTy; break;
766
767    // AMD64-ABI 3.2.3p4: Rule 6. If the class is X87, the value is
768    // returned on the X87 stack in %st0 as 80-bit x87 number.
769  case X87:
770    ResType = llvm::Type::X86_FP80Ty; break;
771
772    // AMD64-ABI 3.2.3p4: Rule 8. If the class is COMPLEX_X87, the real
773    // part of the value is returned in %st0 and the imaginary part in
774    // %st1.
775  case ComplexX87:
776    assert(Hi == ComplexX87 && "Unexpected ComplexX87 classification.");
777    ResType = llvm::StructType::get(llvm::Type::X86_FP80Ty,
778                                    llvm::Type::X86_FP80Ty,
779                                    NULL);
780    break;
781  }
782
783  switch (Hi) {
784    // Memory was handled previously and X87 should
785    // never occur as a hi class.
786  case Memory:
787  case X87:
788    assert(0 && "Invalid classification for hi word.");
789
790  case ComplexX87: // Previously handled.
791  case NoClass: break;
792
793  case Integer:
794    ResType = llvm::StructType::get(ResType, llvm::Type::Int64Ty, NULL);
795    break;
796  case SSE:
797    ResType = llvm::StructType::get(ResType, llvm::Type::DoubleTy, NULL);
798    break;
799
800    // AMD64-ABI 3.2.3p4: Rule 5. If the class is SSEUP, the eightbyte
801    // is passed in the upper half of the last used SSE register.
802    //
803    // SSEUP should always be preceeded by SSE, just widen.
804  case SSEUp:
805    assert(Lo == SSE && "Unexpected SSEUp classification.");
806    ResType = llvm::VectorType::get(llvm::Type::DoubleTy, 2);
807    break;
808
809    // AMD64-ABI 3.2.3p4: Rule 7. If the class is X87UP, the value is
810    // returned together with the previous X87 value in %st0.
811    //
812    // X87UP should always be preceeded by X87, so we don't need to do
813    // anything here.
814  case X87Up:
815    assert(Lo == X87 && "Unexpected X87Up classification.");
816    break;
817  }
818
819  return getCoerceResult(RetTy, ResType, Context);
820}
821
822ABIArgInfo X86_64ABIInfo::classifyArgumentType(QualType Ty, ASTContext &Context,
823                                               unsigned &neededInt,
824                                               unsigned &neededSSE) const {
825  X86_64ABIInfo::Class Lo, Hi;
826  classify(Ty, Context, 0, Lo, Hi);
827
828  // Check some invariants.
829  // FIXME: Enforce these by construction.
830  assert((Hi != Memory || Lo == Memory) && "Invalid memory classification.");
831  assert((Lo != NoClass || Hi == NoClass) && "Invalid null classification.");
832  assert((Hi != SSEUp || Lo == SSE) && "Invalid SSEUp classification.");
833
834  neededInt = 0;
835  neededSSE = 0;
836  const llvm::Type *ResType = 0;
837  switch (Lo) {
838  case NoClass:
839    return ABIArgInfo::getIgnore();
840
841    // AMD64-ABI 3.2.3p3: Rule 1. If the class is MEMORY, pass the argument
842    // on the stack.
843  case Memory:
844
845    // AMD64-ABI 3.2.3p3: Rule 5. If the class is X87, X87UP or
846    // COMPLEX_X87, it is passed in memory.
847  case X87:
848  case ComplexX87:
849    return ABIArgInfo::getIndirect(0);
850
851  case SSEUp:
852  case X87Up:
853    assert(0 && "Invalid classification for lo word.");
854
855    // AMD64-ABI 3.2.3p3: Rule 2. If the class is INTEGER, the next
856    // available register of the sequence %rdi, %rsi, %rdx, %rcx, %r8
857    // and %r9 is used.
858  case Integer:
859    ++neededInt;
860    ResType = llvm::Type::Int64Ty;
861    break;
862
863    // AMD64-ABI 3.2.3p3: Rule 3. If the class is SSE, the next
864    // available SSE register is used, the registers are taken in the
865    // order from %xmm0 to %xmm7.
866  case SSE:
867    ++neededSSE;
868    ResType = llvm::Type::DoubleTy;
869    break;
870  }
871
872  switch (Hi) {
873    // Memory was handled previously, ComplexX87 and X87 should
874    // never occur as hi classes, and X87Up must be preceed by X87,
875    // which is passed in memory.
876  case Memory:
877  case X87:
878  case X87Up:
879  case ComplexX87:
880    assert(0 && "Invalid classification for hi word.");
881
882  case NoClass: break;
883  case Integer:
884    ResType = llvm::StructType::get(ResType, llvm::Type::Int64Ty, NULL);
885    ++neededInt;
886    break;
887  case SSE:
888    ResType = llvm::StructType::get(ResType, llvm::Type::DoubleTy, NULL);
889    ++neededSSE;
890    break;
891
892    // AMD64-ABI 3.2.3p3: Rule 4. If the class is SSEUP, the
893    // eightbyte is passed in the upper half of the last used SSE
894    // register.
895  case SSEUp:
896    assert(Lo == SSE && "Unexpected SSEUp classification.");
897    ResType = llvm::VectorType::get(llvm::Type::DoubleTy, 2);
898    break;
899  }
900
901  return getCoerceResult(Ty, ResType, Context);
902}
903
904void X86_64ABIInfo::computeInfo(CGFunctionInfo &FI, ASTContext &Context) const {
905  FI.getReturnInfo() = classifyReturnType(FI.getReturnType(), Context);
906
907  // Keep track of the number of assigned registers.
908  unsigned freeIntRegs = 6, freeSSERegs = 8;
909
910  // AMD64-ABI 3.2.3p3: Once arguments are classified, the registers
911  // get assigned (in left-to-right order) for passing as follows...
912  for (CGFunctionInfo::arg_iterator it = FI.arg_begin(), ie = FI.arg_end();
913       it != ie; ++it) {
914    unsigned neededInt, neededSSE;
915    it->info = classifyArgumentType(it->type, Context, neededInt, neededSSE);
916
917    // AMD64-ABI 3.2.3p3: If there are no registers available for any
918    // eightbyte of an argument, the whole argument is passed on the
919    // stack. If registers have already been assigned for some
920    // eightbytes of such an argument, the assignments get reverted.
921    if (freeIntRegs >= neededInt && freeSSERegs >= neededSSE) {
922      freeIntRegs -= neededInt;
923      freeSSERegs -= neededSSE;
924    } else {
925      it->info = ABIArgInfo::getIndirect(0);
926    }
927  }
928}
929
930static llvm::Value *EmitVAArgFromMemory(llvm::Value *VAListAddr,
931                                        QualType Ty,
932                                        CodeGenFunction &CGF) {
933  llvm::Value *overflow_arg_area_p =
934    CGF.Builder.CreateStructGEP(VAListAddr, 2, "overflow_arg_area_p");
935  llvm::Value *overflow_arg_area =
936    CGF.Builder.CreateLoad(overflow_arg_area_p, "overflow_arg_area");
937
938  // AMD64-ABI 3.5.7p5: Step 7. Align l->overflow_arg_area upwards to a 16
939  // byte boundary if alignment needed by type exceeds 8 byte boundary.
940  uint64_t Align = CGF.getContext().getTypeAlign(Ty) / 8;
941  if (Align > 8) {
942    // Note that we follow the ABI & gcc here, even though the type
943    // could in theory have an alignment greater than 16. This case
944    // shouldn't ever matter in practice.
945
946    // overflow_arg_area = (overflow_arg_area + 15) & ~15;
947    llvm::Value *Offset = llvm::ConstantInt::get(llvm::Type::Int32Ty, 15);
948    overflow_arg_area = CGF.Builder.CreateGEP(overflow_arg_area, Offset);
949    llvm::Value *AsInt = CGF.Builder.CreatePtrToInt(overflow_arg_area,
950                                                    llvm::Type::Int64Ty);
951    llvm::Value *Mask = llvm::ConstantInt::get(llvm::Type::Int64Ty, ~15LL);
952    overflow_arg_area =
953      CGF.Builder.CreateIntToPtr(CGF.Builder.CreateAnd(AsInt, Mask),
954                                 overflow_arg_area->getType(),
955                                 "overflow_arg_area.align");
956  }
957
958  // AMD64-ABI 3.5.7p5: Step 8. Fetch type from l->overflow_arg_area.
959  const llvm::Type *LTy = CGF.ConvertTypeForMem(Ty);
960  llvm::Value *Res =
961    CGF.Builder.CreateBitCast(overflow_arg_area,
962                              llvm::PointerType::getUnqual(LTy));
963
964  // AMD64-ABI 3.5.7p5: Step 9. Set l->overflow_arg_area to:
965  // l->overflow_arg_area + sizeof(type).
966  // AMD64-ABI 3.5.7p5: Step 10. Align l->overflow_arg_area upwards to
967  // an 8 byte boundary.
968
969  uint64_t SizeInBytes = (CGF.getContext().getTypeSize(Ty) + 7) / 8;
970  llvm::Value *Offset = llvm::ConstantInt::get(llvm::Type::Int32Ty,
971                                               (SizeInBytes + 7)  & ~7);
972  overflow_arg_area = CGF.Builder.CreateGEP(overflow_arg_area, Offset,
973                                            "overflow_arg_area.next");
974  CGF.Builder.CreateStore(overflow_arg_area, overflow_arg_area_p);
975
976  // AMD64-ABI 3.5.7p5: Step 11. Return the fetched type.
977  return Res;
978}
979
980llvm::Value *X86_64ABIInfo::EmitVAArg(llvm::Value *VAListAddr, QualType Ty,
981                                      CodeGenFunction &CGF) const {
982  // Assume that va_list type is correct; should be pointer to LLVM type:
983  // struct {
984  //   i32 gp_offset;
985  //   i32 fp_offset;
986  //   i8* overflow_arg_area;
987  //   i8* reg_save_area;
988  // };
989  unsigned neededInt, neededSSE;
990  ABIArgInfo AI = classifyArgumentType(Ty, CGF.getContext(),
991                                       neededInt, neededSSE);
992
993  // AMD64-ABI 3.5.7p5: Step 1. Determine whether type may be passed
994  // in the registers. If not go to step 7.
995  if (!neededInt && !neededSSE)
996    return EmitVAArgFromMemory(VAListAddr, Ty, CGF);
997
998  // AMD64-ABI 3.5.7p5: Step 2. Compute num_gp to hold the number of
999  // general purpose registers needed to pass type and num_fp to hold
1000  // the number of floating point registers needed.
1001
1002  // AMD64-ABI 3.5.7p5: Step 3. Verify whether arguments fit into
1003  // registers. In the case: l->gp_offset > 48 - num_gp * 8 or
1004  // l->fp_offset > 304 - num_fp * 16 go to step 7.
1005  //
1006  // NOTE: 304 is a typo, there are (6 * 8 + 8 * 16) = 176 bytes of
1007  // register save space).
1008
1009  llvm::Value *InRegs = 0;
1010  llvm::Value *gp_offset_p = 0, *gp_offset = 0;
1011  llvm::Value *fp_offset_p = 0, *fp_offset = 0;
1012  if (neededInt) {
1013    gp_offset_p = CGF.Builder.CreateStructGEP(VAListAddr, 0, "gp_offset_p");
1014    gp_offset = CGF.Builder.CreateLoad(gp_offset_p, "gp_offset");
1015    InRegs =
1016      CGF.Builder.CreateICmpULE(gp_offset,
1017                                llvm::ConstantInt::get(llvm::Type::Int32Ty,
1018                                                       48 - neededInt * 8),
1019                                "fits_in_gp");
1020  }
1021
1022  if (neededSSE) {
1023    fp_offset_p = CGF.Builder.CreateStructGEP(VAListAddr, 1, "fp_offset_p");
1024    fp_offset = CGF.Builder.CreateLoad(fp_offset_p, "fp_offset");
1025    llvm::Value *FitsInFP =
1026      CGF.Builder.CreateICmpULE(fp_offset,
1027                                llvm::ConstantInt::get(llvm::Type::Int32Ty,
1028                                                       176 - neededSSE * 16),
1029                                "fits_in_fp");
1030    InRegs = InRegs ? CGF.Builder.CreateAnd(InRegs, FitsInFP) : FitsInFP;
1031  }
1032
1033  llvm::BasicBlock *InRegBlock = CGF.createBasicBlock("vaarg.in_reg");
1034  llvm::BasicBlock *InMemBlock = CGF.createBasicBlock("vaarg.in_mem");
1035  llvm::BasicBlock *ContBlock = CGF.createBasicBlock("vaarg.end");
1036  CGF.Builder.CreateCondBr(InRegs, InRegBlock, InMemBlock);
1037
1038  // Emit code to load the value if it was passed in registers.
1039
1040  CGF.EmitBlock(InRegBlock);
1041
1042  // AMD64-ABI 3.5.7p5: Step 4. Fetch type from l->reg_save_area with
1043  // an offset of l->gp_offset and/or l->fp_offset. This may require
1044  // copying to a temporary location in case the parameter is passed
1045  // in different register classes or requires an alignment greater
1046  // than 8 for general purpose registers and 16 for XMM registers.
1047  //
1048  // FIXME: This really results in shameful code when we end up
1049  // needing to collect arguments from different places; often what
1050  // should result in a simple assembling of a structure from
1051  // scattered addresses has many more loads than necessary. Can we
1052  // clean this up?
1053  const llvm::Type *LTy = CGF.ConvertTypeForMem(Ty);
1054  llvm::Value *RegAddr =
1055    CGF.Builder.CreateLoad(CGF.Builder.CreateStructGEP(VAListAddr, 3),
1056                           "reg_save_area");
1057  if (neededInt && neededSSE) {
1058    // FIXME: Cleanup.
1059    assert(AI.isCoerce() && "Unexpected ABI info for mixed regs");
1060    const llvm::StructType *ST = cast<llvm::StructType>(AI.getCoerceToType());
1061    llvm::Value *Tmp = CGF.CreateTempAlloca(ST);
1062    assert(ST->getNumElements() == 2 && "Unexpected ABI info for mixed regs");
1063    const llvm::Type *TyLo = ST->getElementType(0);
1064    const llvm::Type *TyHi = ST->getElementType(1);
1065    assert((TyLo->isFloatingPoint() ^ TyHi->isFloatingPoint()) &&
1066           "Unexpected ABI info for mixed regs");
1067    const llvm::Type *PTyLo = llvm::PointerType::getUnqual(TyLo);
1068    const llvm::Type *PTyHi = llvm::PointerType::getUnqual(TyHi);
1069    llvm::Value *GPAddr = CGF.Builder.CreateGEP(RegAddr, gp_offset);
1070    llvm::Value *FPAddr = CGF.Builder.CreateGEP(RegAddr, fp_offset);
1071    llvm::Value *RegLoAddr = TyLo->isFloatingPoint() ? FPAddr : GPAddr;
1072    llvm::Value *RegHiAddr = TyLo->isFloatingPoint() ? GPAddr : FPAddr;
1073    llvm::Value *V =
1074      CGF.Builder.CreateLoad(CGF.Builder.CreateBitCast(RegLoAddr, PTyLo));
1075    CGF.Builder.CreateStore(V, CGF.Builder.CreateStructGEP(Tmp, 0));
1076    V = CGF.Builder.CreateLoad(CGF.Builder.CreateBitCast(RegHiAddr, PTyHi));
1077    CGF.Builder.CreateStore(V, CGF.Builder.CreateStructGEP(Tmp, 1));
1078
1079    RegAddr = CGF.Builder.CreateBitCast(Tmp, llvm::PointerType::getUnqual(LTy));
1080  } else if (neededInt) {
1081    RegAddr = CGF.Builder.CreateGEP(RegAddr, gp_offset);
1082    RegAddr = CGF.Builder.CreateBitCast(RegAddr,
1083                                        llvm::PointerType::getUnqual(LTy));
1084  } else {
1085    if (neededSSE == 1) {
1086      RegAddr = CGF.Builder.CreateGEP(RegAddr, fp_offset);
1087      RegAddr = CGF.Builder.CreateBitCast(RegAddr,
1088                                          llvm::PointerType::getUnqual(LTy));
1089    } else {
1090      assert(neededSSE == 2 && "Invalid number of needed registers!");
1091      // SSE registers are spaced 16 bytes apart in the register save
1092      // area, we need to collect the two eightbytes together.
1093      llvm::Value *RegAddrLo = CGF.Builder.CreateGEP(RegAddr, fp_offset);
1094      llvm::Value *RegAddrHi =
1095        CGF.Builder.CreateGEP(RegAddrLo,
1096                              llvm::ConstantInt::get(llvm::Type::Int32Ty, 16));
1097      const llvm::Type *DblPtrTy =
1098        llvm::PointerType::getUnqual(llvm::Type::DoubleTy);
1099      const llvm::StructType *ST = llvm::StructType::get(llvm::Type::DoubleTy,
1100                                                         llvm::Type::DoubleTy,
1101                                                         NULL);
1102      llvm::Value *V, *Tmp = CGF.CreateTempAlloca(ST);
1103      V = CGF.Builder.CreateLoad(CGF.Builder.CreateBitCast(RegAddrLo,
1104                                                           DblPtrTy));
1105      CGF.Builder.CreateStore(V, CGF.Builder.CreateStructGEP(Tmp, 0));
1106      V = CGF.Builder.CreateLoad(CGF.Builder.CreateBitCast(RegAddrHi,
1107                                                           DblPtrTy));
1108      CGF.Builder.CreateStore(V, CGF.Builder.CreateStructGEP(Tmp, 1));
1109      RegAddr = CGF.Builder.CreateBitCast(Tmp,
1110                                          llvm::PointerType::getUnqual(LTy));
1111    }
1112  }
1113
1114  // AMD64-ABI 3.5.7p5: Step 5. Set:
1115  // l->gp_offset = l->gp_offset + num_gp * 8
1116  // l->fp_offset = l->fp_offset + num_fp * 16.
1117  if (neededInt) {
1118    llvm::Value *Offset = llvm::ConstantInt::get(llvm::Type::Int32Ty,
1119                                                 neededInt * 8);
1120    CGF.Builder.CreateStore(CGF.Builder.CreateAdd(gp_offset, Offset),
1121                            gp_offset_p);
1122  }
1123  if (neededSSE) {
1124    llvm::Value *Offset = llvm::ConstantInt::get(llvm::Type::Int32Ty,
1125                                                 neededSSE * 16);
1126    CGF.Builder.CreateStore(CGF.Builder.CreateAdd(fp_offset, Offset),
1127                            fp_offset_p);
1128  }
1129  CGF.EmitBranch(ContBlock);
1130
1131  // Emit code to load the value if it was passed in memory.
1132
1133  CGF.EmitBlock(InMemBlock);
1134  llvm::Value *MemAddr = EmitVAArgFromMemory(VAListAddr, Ty, CGF);
1135
1136  // Return the appropriate result.
1137
1138  CGF.EmitBlock(ContBlock);
1139  llvm::PHINode *ResAddr = CGF.Builder.CreatePHI(RegAddr->getType(),
1140                                                 "vaarg.addr");
1141  ResAddr->reserveOperandSpace(2);
1142  ResAddr->addIncoming(RegAddr, InRegBlock);
1143  ResAddr->addIncoming(MemAddr, InMemBlock);
1144
1145  return ResAddr;
1146}
1147
1148ABIArgInfo DefaultABIInfo::classifyReturnType(QualType RetTy,
1149                                              ASTContext &Context) const {
1150  if (RetTy->isVoidType()) {
1151    return ABIArgInfo::getIgnore();
1152  } else if (CodeGenFunction::hasAggregateLLVMType(RetTy)) {
1153    return ABIArgInfo::getIndirect(0);
1154  } else {
1155    return ABIArgInfo::getDirect();
1156  }
1157}
1158
1159ABIArgInfo DefaultABIInfo::classifyArgumentType(QualType Ty,
1160                                                ASTContext &Context) const {
1161  if (CodeGenFunction::hasAggregateLLVMType(Ty)) {
1162    return ABIArgInfo::getIndirect(0);
1163  } else {
1164    return ABIArgInfo::getDirect();
1165  }
1166}
1167
1168llvm::Value *DefaultABIInfo::EmitVAArg(llvm::Value *VAListAddr, QualType Ty,
1169                                       CodeGenFunction &CGF) const {
1170  return 0;
1171}
1172
1173const ABIInfo &CodeGenTypes::getABIInfo() const {
1174  if (TheABIInfo)
1175    return *TheABIInfo;
1176
1177  // For now we just cache this in the CodeGenTypes and don't bother
1178  // to free it.
1179  const char *TargetPrefix = getContext().Target.getTargetPrefix();
1180  if (strcmp(TargetPrefix, "x86") == 0) {
1181    switch (getContext().Target.getPointerWidth(0)) {
1182    case 32:
1183      return *(TheABIInfo = new X86_32ABIInfo());
1184    case 64:
1185      return *(TheABIInfo = new X86_64ABIInfo());
1186    }
1187  }
1188
1189  return *(TheABIInfo = new DefaultABIInfo);
1190}
1191
1192/***/
1193
1194CGFunctionInfo::CGFunctionInfo(QualType ResTy,
1195                               const llvm::SmallVector<QualType, 16> &ArgTys) {
1196  NumArgs = ArgTys.size();
1197  Args = new ArgInfo[1 + NumArgs];
1198  Args[0].type = ResTy;
1199  for (unsigned i = 0; i < NumArgs; ++i)
1200    Args[1 + i].type = ArgTys[i];
1201}
1202
1203/***/
1204
1205void CodeGenTypes::GetExpandedTypes(QualType Ty,
1206                                    std::vector<const llvm::Type*> &ArgTys) {
1207  const RecordType *RT = Ty->getAsStructureType();
1208  assert(RT && "Can only expand structure types.");
1209  const RecordDecl *RD = RT->getDecl();
1210  assert(!RD->hasFlexibleArrayMember() &&
1211         "Cannot expand structure with flexible array.");
1212
1213  for (RecordDecl::field_iterator i = RD->field_begin(),
1214         e = RD->field_end(); i != e; ++i) {
1215    const FieldDecl *FD = *i;
1216    assert(!FD->isBitField() &&
1217           "Cannot expand structure with bit-field members.");
1218
1219    QualType FT = FD->getType();
1220    if (CodeGenFunction::hasAggregateLLVMType(FT)) {
1221      GetExpandedTypes(FT, ArgTys);
1222    } else {
1223      ArgTys.push_back(ConvertType(FT));
1224    }
1225  }
1226}
1227
1228llvm::Function::arg_iterator
1229CodeGenFunction::ExpandTypeFromArgs(QualType Ty, LValue LV,
1230                                    llvm::Function::arg_iterator AI) {
1231  const RecordType *RT = Ty->getAsStructureType();
1232  assert(RT && "Can only expand structure types.");
1233
1234  RecordDecl *RD = RT->getDecl();
1235  assert(LV.isSimple() &&
1236         "Unexpected non-simple lvalue during struct expansion.");
1237  llvm::Value *Addr = LV.getAddress();
1238  for (RecordDecl::field_iterator i = RD->field_begin(),
1239         e = RD->field_end(); i != e; ++i) {
1240    FieldDecl *FD = *i;
1241    QualType FT = FD->getType();
1242
1243    // FIXME: What are the right qualifiers here?
1244    LValue LV = EmitLValueForField(Addr, FD, false, 0);
1245    if (CodeGenFunction::hasAggregateLLVMType(FT)) {
1246      AI = ExpandTypeFromArgs(FT, LV, AI);
1247    } else {
1248      EmitStoreThroughLValue(RValue::get(AI), LV, FT);
1249      ++AI;
1250    }
1251  }
1252
1253  return AI;
1254}
1255
1256void
1257CodeGenFunction::ExpandTypeToArgs(QualType Ty, RValue RV,
1258                                  llvm::SmallVector<llvm::Value*, 16> &Args) {
1259  const RecordType *RT = Ty->getAsStructureType();
1260  assert(RT && "Can only expand structure types.");
1261
1262  RecordDecl *RD = RT->getDecl();
1263  assert(RV.isAggregate() && "Unexpected rvalue during struct expansion");
1264  llvm::Value *Addr = RV.getAggregateAddr();
1265  for (RecordDecl::field_iterator i = RD->field_begin(),
1266         e = RD->field_end(); i != e; ++i) {
1267    FieldDecl *FD = *i;
1268    QualType FT = FD->getType();
1269
1270    // FIXME: What are the right qualifiers here?
1271    LValue LV = EmitLValueForField(Addr, FD, false, 0);
1272    if (CodeGenFunction::hasAggregateLLVMType(FT)) {
1273      ExpandTypeToArgs(FT, RValue::getAggregate(LV.getAddress()), Args);
1274    } else {
1275      RValue RV = EmitLoadOfLValue(LV, FT);
1276      assert(RV.isScalar() &&
1277             "Unexpected non-scalar rvalue during struct expansion.");
1278      Args.push_back(RV.getScalarVal());
1279    }
1280  }
1281}
1282
1283/// CreateCoercedLoad - Create a load from \arg SrcPtr interpreted as
1284/// a pointer to an object of type \arg Ty.
1285///
1286/// This safely handles the case when the src type is smaller than the
1287/// destination type; in this situation the values of bits which not
1288/// present in the src are undefined.
1289static llvm::Value *CreateCoercedLoad(llvm::Value *SrcPtr,
1290                                      const llvm::Type *Ty,
1291                                      CodeGenFunction &CGF) {
1292  const llvm::Type *SrcTy =
1293    cast<llvm::PointerType>(SrcPtr->getType())->getElementType();
1294  uint64_t SrcSize = CGF.CGM.getTargetData().getTypePaddedSize(SrcTy);
1295  uint64_t DstSize = CGF.CGM.getTargetData().getTypePaddedSize(Ty);
1296
1297  // If load is legal, just bitcast the src pointer.
1298  if (SrcSize == DstSize) {
1299    llvm::Value *Casted =
1300      CGF.Builder.CreateBitCast(SrcPtr, llvm::PointerType::getUnqual(Ty));
1301    llvm::LoadInst *Load = CGF.Builder.CreateLoad(Casted);
1302    // FIXME: Use better alignment / avoid requiring aligned load.
1303    Load->setAlignment(1);
1304    return Load;
1305  } else {
1306    assert(SrcSize < DstSize && "Coercion is losing source bits!");
1307
1308    // Otherwise do coercion through memory. This is stupid, but
1309    // simple.
1310    llvm::Value *Tmp = CGF.CreateTempAlloca(Ty);
1311    llvm::Value *Casted =
1312      CGF.Builder.CreateBitCast(Tmp, llvm::PointerType::getUnqual(SrcTy));
1313    llvm::StoreInst *Store =
1314      CGF.Builder.CreateStore(CGF.Builder.CreateLoad(SrcPtr), Casted);
1315    // FIXME: Use better alignment / avoid requiring aligned store.
1316    Store->setAlignment(1);
1317    return CGF.Builder.CreateLoad(Tmp);
1318  }
1319}
1320
1321/// CreateCoercedStore - Create a store to \arg DstPtr from \arg Src,
1322/// where the source and destination may have different types.
1323///
1324/// This safely handles the case when the src type is larger than the
1325/// destination type; the upper bits of the src will be lost.
1326static void CreateCoercedStore(llvm::Value *Src,
1327                               llvm::Value *DstPtr,
1328                               CodeGenFunction &CGF) {
1329  const llvm::Type *SrcTy = Src->getType();
1330  const llvm::Type *DstTy =
1331    cast<llvm::PointerType>(DstPtr->getType())->getElementType();
1332
1333  uint64_t SrcSize = CGF.CGM.getTargetData().getTypePaddedSize(SrcTy);
1334  uint64_t DstSize = CGF.CGM.getTargetData().getTypePaddedSize(DstTy);
1335
1336  // If store is legal, just bitcast the src pointer.
1337  if (SrcSize == DstSize) {
1338    llvm::Value *Casted =
1339      CGF.Builder.CreateBitCast(DstPtr, llvm::PointerType::getUnqual(SrcTy));
1340    // FIXME: Use better alignment / avoid requiring aligned store.
1341    CGF.Builder.CreateStore(Src, Casted)->setAlignment(1);
1342  } else {
1343    assert(SrcSize > DstSize && "Coercion is missing bits!");
1344
1345    // Otherwise do coercion through memory. This is stupid, but
1346    // simple.
1347    llvm::Value *Tmp = CGF.CreateTempAlloca(SrcTy);
1348    CGF.Builder.CreateStore(Src, Tmp);
1349    llvm::Value *Casted =
1350      CGF.Builder.CreateBitCast(Tmp, llvm::PointerType::getUnqual(DstTy));
1351    llvm::LoadInst *Load = CGF.Builder.CreateLoad(Casted);
1352    // FIXME: Use better alignment / avoid requiring aligned load.
1353    Load->setAlignment(1);
1354    CGF.Builder.CreateStore(Load, DstPtr);
1355  }
1356}
1357
1358/***/
1359
1360bool CodeGenModule::ReturnTypeUsesSret(const CGFunctionInfo &FI) {
1361  return FI.getReturnInfo().isIndirect();
1362}
1363
1364const llvm::FunctionType *
1365CodeGenTypes::GetFunctionType(const CGFunctionInfo &FI, bool IsVariadic) {
1366  std::vector<const llvm::Type*> ArgTys;
1367
1368  const llvm::Type *ResultType = 0;
1369
1370  QualType RetTy = FI.getReturnType();
1371  const ABIArgInfo &RetAI = FI.getReturnInfo();
1372  switch (RetAI.getKind()) {
1373  case ABIArgInfo::Expand:
1374    assert(0 && "Invalid ABI kind for return argument");
1375
1376  case ABIArgInfo::Direct:
1377    ResultType = ConvertType(RetTy);
1378    break;
1379
1380  case ABIArgInfo::Indirect: {
1381    assert(!RetAI.getIndirectAlign() && "Align unused on indirect return.");
1382    ResultType = llvm::Type::VoidTy;
1383    const llvm::Type *STy = ConvertType(RetTy);
1384    ArgTys.push_back(llvm::PointerType::get(STy, RetTy.getAddressSpace()));
1385    break;
1386  }
1387
1388  case ABIArgInfo::Ignore:
1389    ResultType = llvm::Type::VoidTy;
1390    break;
1391
1392  case ABIArgInfo::Coerce:
1393    ResultType = RetAI.getCoerceToType();
1394    break;
1395  }
1396
1397  for (CGFunctionInfo::const_arg_iterator it = FI.arg_begin(),
1398         ie = FI.arg_end(); it != ie; ++it) {
1399    const ABIArgInfo &AI = it->info;
1400
1401    switch (AI.getKind()) {
1402    case ABIArgInfo::Ignore:
1403      break;
1404
1405    case ABIArgInfo::Coerce:
1406      ArgTys.push_back(AI.getCoerceToType());
1407      break;
1408
1409    case ABIArgInfo::Indirect: {
1410      // indirect arguments are always on the stack, which is addr space #0.
1411      const llvm::Type *LTy = ConvertTypeForMem(it->type);
1412      ArgTys.push_back(llvm::PointerType::getUnqual(LTy));
1413      break;
1414    }
1415
1416    case ABIArgInfo::Direct:
1417      ArgTys.push_back(ConvertType(it->type));
1418      break;
1419
1420    case ABIArgInfo::Expand:
1421      GetExpandedTypes(it->type, ArgTys);
1422      break;
1423    }
1424  }
1425
1426  return llvm::FunctionType::get(ResultType, ArgTys, IsVariadic);
1427}
1428
1429void CodeGenModule::ConstructAttributeList(const CGFunctionInfo &FI,
1430                                           const Decl *TargetDecl,
1431                                           AttributeListType &PAL) {
1432  unsigned FuncAttrs = 0;
1433  unsigned RetAttrs = 0;
1434
1435  if (TargetDecl) {
1436    if (TargetDecl->getAttr<NoThrowAttr>())
1437      FuncAttrs |= llvm::Attribute::NoUnwind;
1438    if (TargetDecl->getAttr<NoReturnAttr>())
1439      FuncAttrs |= llvm::Attribute::NoReturn;
1440    if (TargetDecl->getAttr<PureAttr>())
1441      FuncAttrs |= llvm::Attribute::ReadOnly;
1442    if (TargetDecl->getAttr<ConstAttr>())
1443      FuncAttrs |= llvm::Attribute::ReadNone;
1444  }
1445
1446  QualType RetTy = FI.getReturnType();
1447  unsigned Index = 1;
1448  const ABIArgInfo &RetAI = FI.getReturnInfo();
1449  switch (RetAI.getKind()) {
1450  case ABIArgInfo::Direct:
1451    if (RetTy->isPromotableIntegerType()) {
1452      if (RetTy->isSignedIntegerType()) {
1453        RetAttrs |= llvm::Attribute::SExt;
1454      } else if (RetTy->isUnsignedIntegerType()) {
1455        RetAttrs |= llvm::Attribute::ZExt;
1456      }
1457    }
1458    break;
1459
1460  case ABIArgInfo::Indirect:
1461    PAL.push_back(llvm::AttributeWithIndex::get(Index,
1462                                                llvm::Attribute::StructRet |
1463                                                llvm::Attribute::NoAlias));
1464    ++Index;
1465    break;
1466
1467  case ABIArgInfo::Ignore:
1468  case ABIArgInfo::Coerce:
1469    break;
1470
1471  case ABIArgInfo::Expand:
1472    assert(0 && "Invalid ABI kind for return argument");
1473  }
1474
1475  if (RetAttrs)
1476    PAL.push_back(llvm::AttributeWithIndex::get(0, RetAttrs));
1477  for (CGFunctionInfo::const_arg_iterator it = FI.arg_begin(),
1478         ie = FI.arg_end(); it != ie; ++it) {
1479    QualType ParamType = it->type;
1480    const ABIArgInfo &AI = it->info;
1481    unsigned Attributes = 0;
1482
1483    switch (AI.getKind()) {
1484    case ABIArgInfo::Coerce:
1485      break;
1486
1487    case ABIArgInfo::Indirect:
1488      Attributes |= llvm::Attribute::ByVal;
1489      Attributes |=
1490        llvm::Attribute::constructAlignmentFromInt(AI.getIndirectAlign());
1491      break;
1492
1493    case ABIArgInfo::Direct:
1494      if (ParamType->isPromotableIntegerType()) {
1495        if (ParamType->isSignedIntegerType()) {
1496          Attributes |= llvm::Attribute::SExt;
1497        } else if (ParamType->isUnsignedIntegerType()) {
1498          Attributes |= llvm::Attribute::ZExt;
1499        }
1500      }
1501      break;
1502
1503    case ABIArgInfo::Ignore:
1504      // Skip increment, no matching LLVM parameter.
1505      continue;
1506
1507    case ABIArgInfo::Expand: {
1508      std::vector<const llvm::Type*> Tys;
1509      // FIXME: This is rather inefficient. Do we ever actually need
1510      // to do anything here? The result should be just reconstructed
1511      // on the other side, so extension should be a non-issue.
1512      getTypes().GetExpandedTypes(ParamType, Tys);
1513      Index += Tys.size();
1514      continue;
1515    }
1516    }
1517
1518    if (Attributes)
1519      PAL.push_back(llvm::AttributeWithIndex::get(Index, Attributes));
1520    ++Index;
1521  }
1522  if (FuncAttrs)
1523    PAL.push_back(llvm::AttributeWithIndex::get(~0, FuncAttrs));
1524}
1525
1526void CodeGenFunction::EmitFunctionProlog(const CGFunctionInfo &FI,
1527                                         llvm::Function *Fn,
1528                                         const FunctionArgList &Args) {
1529  // FIXME: We no longer need the types from FunctionArgList; lift up
1530  // and simplify.
1531
1532  // Emit allocs for param decls.  Give the LLVM Argument nodes names.
1533  llvm::Function::arg_iterator AI = Fn->arg_begin();
1534
1535  // Name the struct return argument.
1536  if (CGM.ReturnTypeUsesSret(FI)) {
1537    AI->setName("agg.result");
1538    ++AI;
1539  }
1540
1541  assert(FI.arg_size() == Args.size() &&
1542         "Mismatch between function signature & arguments.");
1543  CGFunctionInfo::const_arg_iterator info_it = FI.arg_begin();
1544  for (FunctionArgList::const_iterator i = Args.begin(), e = Args.end();
1545       i != e; ++i, ++info_it) {
1546    const VarDecl *Arg = i->first;
1547    QualType Ty = info_it->type;
1548    const ABIArgInfo &ArgI = info_it->info;
1549
1550    switch (ArgI.getKind()) {
1551    case ABIArgInfo::Indirect: {
1552      llvm::Value* V = AI;
1553      if (hasAggregateLLVMType(Ty)) {
1554        // Do nothing, aggregates and complex variables are accessed by
1555        // reference.
1556      } else {
1557        // Load scalar value from indirect argument.
1558        V = EmitLoadOfScalar(V, false, Ty);
1559        if (!getContext().typesAreCompatible(Ty, Arg->getType())) {
1560          // This must be a promotion, for something like
1561          // "void a(x) short x; {..."
1562          V = EmitScalarConversion(V, Ty, Arg->getType());
1563        }
1564      }
1565      EmitParmDecl(*Arg, V);
1566      break;
1567    }
1568
1569    case ABIArgInfo::Direct: {
1570      assert(AI != Fn->arg_end() && "Argument mismatch!");
1571      llvm::Value* V = AI;
1572      if (hasAggregateLLVMType(Ty)) {
1573        // Create a temporary alloca to hold the argument; the rest of
1574        // codegen expects to access aggregates & complex values by
1575        // reference.
1576        V = CreateTempAlloca(ConvertTypeForMem(Ty));
1577        Builder.CreateStore(AI, V);
1578      } else {
1579        if (!getContext().typesAreCompatible(Ty, Arg->getType())) {
1580          // This must be a promotion, for something like
1581          // "void a(x) short x; {..."
1582          V = EmitScalarConversion(V, Ty, Arg->getType());
1583        }
1584      }
1585      EmitParmDecl(*Arg, V);
1586      break;
1587    }
1588
1589    case ABIArgInfo::Expand: {
1590      // If this structure was expanded into multiple arguments then
1591      // we need to create a temporary and reconstruct it from the
1592      // arguments.
1593      std::string Name = Arg->getNameAsString();
1594      llvm::Value *Temp = CreateTempAlloca(ConvertTypeForMem(Ty),
1595                                           (Name + ".addr").c_str());
1596      // FIXME: What are the right qualifiers here?
1597      llvm::Function::arg_iterator End =
1598        ExpandTypeFromArgs(Ty, LValue::MakeAddr(Temp,0), AI);
1599      EmitParmDecl(*Arg, Temp);
1600
1601      // Name the arguments used in expansion and increment AI.
1602      unsigned Index = 0;
1603      for (; AI != End; ++AI, ++Index)
1604        AI->setName(Name + "." + llvm::utostr(Index));
1605      continue;
1606    }
1607
1608    case ABIArgInfo::Ignore:
1609      // Initialize the local variable appropriately.
1610      if (hasAggregateLLVMType(Ty)) {
1611        EmitParmDecl(*Arg, CreateTempAlloca(ConvertTypeForMem(Ty)));
1612      } else {
1613        EmitParmDecl(*Arg, llvm::UndefValue::get(ConvertType(Arg->getType())));
1614      }
1615
1616      // Skip increment, no matching LLVM parameter.
1617      continue;
1618
1619    case ABIArgInfo::Coerce: {
1620      assert(AI != Fn->arg_end() && "Argument mismatch!");
1621      // FIXME: This is very wasteful; EmitParmDecl is just going to
1622      // drop the result in a new alloca anyway, so we could just
1623      // store into that directly if we broke the abstraction down
1624      // more.
1625      llvm::Value *V = CreateTempAlloca(ConvertTypeForMem(Ty), "coerce");
1626      CreateCoercedStore(AI, V, *this);
1627      // Match to what EmitParmDecl is expecting for this type.
1628      if (!CodeGenFunction::hasAggregateLLVMType(Ty)) {
1629        V = EmitLoadOfScalar(V, false, Ty);
1630        if (!getContext().typesAreCompatible(Ty, Arg->getType())) {
1631          // This must be a promotion, for something like
1632          // "void a(x) short x; {..."
1633          V = EmitScalarConversion(V, Ty, Arg->getType());
1634        }
1635      }
1636      EmitParmDecl(*Arg, V);
1637      break;
1638    }
1639    }
1640
1641    ++AI;
1642  }
1643  assert(AI == Fn->arg_end() && "Argument mismatch!");
1644}
1645
1646void CodeGenFunction::EmitFunctionEpilog(const CGFunctionInfo &FI,
1647                                         llvm::Value *ReturnValue) {
1648  llvm::Value *RV = 0;
1649
1650  // Functions with no result always return void.
1651  if (ReturnValue) {
1652    QualType RetTy = FI.getReturnType();
1653    const ABIArgInfo &RetAI = FI.getReturnInfo();
1654
1655    switch (RetAI.getKind()) {
1656    case ABIArgInfo::Indirect:
1657      if (RetTy->isAnyComplexType()) {
1658        ComplexPairTy RT = LoadComplexFromAddr(ReturnValue, false);
1659        StoreComplexToAddr(RT, CurFn->arg_begin(), false);
1660      } else if (CodeGenFunction::hasAggregateLLVMType(RetTy)) {
1661        EmitAggregateCopy(CurFn->arg_begin(), ReturnValue, RetTy);
1662      } else {
1663        EmitStoreOfScalar(Builder.CreateLoad(ReturnValue), CurFn->arg_begin(),
1664                          false);
1665      }
1666      break;
1667
1668    case ABIArgInfo::Direct:
1669      // The internal return value temp always will have
1670      // pointer-to-return-type type.
1671      RV = Builder.CreateLoad(ReturnValue);
1672      break;
1673
1674    case ABIArgInfo::Ignore:
1675      break;
1676
1677    case ABIArgInfo::Coerce:
1678      RV = CreateCoercedLoad(ReturnValue, RetAI.getCoerceToType(), *this);
1679      break;
1680
1681    case ABIArgInfo::Expand:
1682      assert(0 && "Invalid ABI kind for return argument");
1683    }
1684  }
1685
1686  if (RV) {
1687    Builder.CreateRet(RV);
1688  } else {
1689    Builder.CreateRetVoid();
1690  }
1691}
1692
1693RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
1694                                 llvm::Value *Callee,
1695                                 const CallArgList &CallArgs,
1696                                 const Decl *TargetDecl) {
1697  // FIXME: We no longer need the types from CallArgs; lift up and
1698  // simplify.
1699  llvm::SmallVector<llvm::Value*, 16> Args;
1700
1701  // Handle struct-return functions by passing a pointer to the
1702  // location that we would like to return into.
1703  QualType RetTy = CallInfo.getReturnType();
1704  const ABIArgInfo &RetAI = CallInfo.getReturnInfo();
1705  if (CGM.ReturnTypeUsesSret(CallInfo)) {
1706    // Create a temporary alloca to hold the result of the call. :(
1707    Args.push_back(CreateTempAlloca(ConvertTypeForMem(RetTy)));
1708  }
1709
1710  assert(CallInfo.arg_size() == CallArgs.size() &&
1711         "Mismatch between function signature & arguments.");
1712  CGFunctionInfo::const_arg_iterator info_it = CallInfo.arg_begin();
1713  for (CallArgList::const_iterator I = CallArgs.begin(), E = CallArgs.end();
1714       I != E; ++I, ++info_it) {
1715    const ABIArgInfo &ArgInfo = info_it->info;
1716    RValue RV = I->first;
1717
1718    switch (ArgInfo.getKind()) {
1719    case ABIArgInfo::Indirect:
1720      if (RV.isScalar() || RV.isComplex()) {
1721        // Make a temporary alloca to pass the argument.
1722        Args.push_back(CreateTempAlloca(ConvertTypeForMem(I->second)));
1723        if (RV.isScalar())
1724          EmitStoreOfScalar(RV.getScalarVal(), Args.back(), false);
1725        else
1726          StoreComplexToAddr(RV.getComplexVal(), Args.back(), false);
1727      } else {
1728        Args.push_back(RV.getAggregateAddr());
1729      }
1730      break;
1731
1732    case ABIArgInfo::Direct:
1733      if (RV.isScalar()) {
1734        Args.push_back(RV.getScalarVal());
1735      } else if (RV.isComplex()) {
1736        llvm::Value *Tmp = llvm::UndefValue::get(ConvertType(I->second));
1737        Tmp = Builder.CreateInsertValue(Tmp, RV.getComplexVal().first, 0);
1738        Tmp = Builder.CreateInsertValue(Tmp, RV.getComplexVal().second, 1);
1739        Args.push_back(Tmp);
1740      } else {
1741        Args.push_back(Builder.CreateLoad(RV.getAggregateAddr()));
1742      }
1743      break;
1744
1745    case ABIArgInfo::Ignore:
1746      break;
1747
1748    case ABIArgInfo::Coerce: {
1749      // FIXME: Avoid the conversion through memory if possible.
1750      llvm::Value *SrcPtr;
1751      if (RV.isScalar()) {
1752        SrcPtr = CreateTempAlloca(ConvertTypeForMem(I->second), "coerce");
1753        EmitStoreOfScalar(RV.getScalarVal(), SrcPtr, false);
1754      } else if (RV.isComplex()) {
1755        SrcPtr = CreateTempAlloca(ConvertTypeForMem(I->second), "coerce");
1756        StoreComplexToAddr(RV.getComplexVal(), SrcPtr, false);
1757      } else
1758        SrcPtr = RV.getAggregateAddr();
1759      Args.push_back(CreateCoercedLoad(SrcPtr, ArgInfo.getCoerceToType(),
1760                                       *this));
1761      break;
1762    }
1763
1764    case ABIArgInfo::Expand:
1765      ExpandTypeToArgs(I->second, RV, Args);
1766      break;
1767    }
1768  }
1769
1770  llvm::BasicBlock *InvokeDest = getInvokeDest();
1771  CodeGen::AttributeListType AttributeList;
1772  CGM.ConstructAttributeList(CallInfo, TargetDecl, AttributeList);
1773  llvm::AttrListPtr Attrs = llvm::AttrListPtr::get(AttributeList.begin(),
1774                                                   AttributeList.end());
1775
1776  llvm::Instruction *CI;
1777  if (!InvokeDest || Attrs.getFnAttributes() & (llvm::Attribute::NoUnwind ||
1778                                                llvm::Attribute::NoReturn)) {
1779    llvm::CallInst *CallInstr =
1780      Builder.CreateCall(Callee, &Args[0], &Args[0]+Args.size());
1781    CI = CallInstr;
1782
1783    CallInstr->setAttributes(Attrs);
1784    if (const llvm::Function *F = dyn_cast<llvm::Function>(Callee))
1785      CallInstr->setCallingConv(F->getCallingConv());
1786
1787    // If the call doesn't return, finish the basic block and clear the
1788    // insertion point; this allows the rest of IRgen to discard
1789    // unreachable code.
1790    if (CallInstr->doesNotReturn()) {
1791      Builder.CreateUnreachable();
1792      Builder.ClearInsertionPoint();
1793
1794      // FIXME: For now, emit a dummy basic block because expr
1795      // emitters in generally are not ready to handle emitting
1796      // expressions at unreachable points.
1797      EnsureInsertPoint();
1798
1799      // Return a reasonable RValue.
1800      return GetUndefRValue(RetTy);
1801    }
1802  } else {
1803    llvm::BasicBlock *Cont = createBasicBlock("invoke.cont");
1804    llvm::InvokeInst *InvokeInstr =
1805      Builder.CreateInvoke(Callee, Cont, InvokeDest,
1806                           &Args[0], &Args[0]+Args.size());
1807    CI = InvokeInstr;
1808
1809    InvokeInstr->setAttributes(Attrs);
1810    if (const llvm::Function *F = dyn_cast<llvm::Function>(Callee))
1811      InvokeInstr->setCallingConv(F->getCallingConv());
1812
1813    EmitBlock(Cont);
1814  }
1815
1816  if (CI->getType() != llvm::Type::VoidTy)
1817    CI->setName("call");
1818
1819  switch (RetAI.getKind()) {
1820  case ABIArgInfo::Indirect:
1821    if (RetTy->isAnyComplexType())
1822      return RValue::getComplex(LoadComplexFromAddr(Args[0], false));
1823    else if (CodeGenFunction::hasAggregateLLVMType(RetTy))
1824      return RValue::getAggregate(Args[0]);
1825    else
1826      return RValue::get(EmitLoadOfScalar(Args[0], false, RetTy));
1827
1828  case ABIArgInfo::Direct:
1829    if (RetTy->isAnyComplexType()) {
1830      llvm::Value *Real = Builder.CreateExtractValue(CI, 0);
1831      llvm::Value *Imag = Builder.CreateExtractValue(CI, 1);
1832      return RValue::getComplex(std::make_pair(Real, Imag));
1833    } else if (CodeGenFunction::hasAggregateLLVMType(RetTy)) {
1834      llvm::Value *V = CreateTempAlloca(ConvertTypeForMem(RetTy), "agg.tmp");
1835      Builder.CreateStore(CI, V);
1836      return RValue::getAggregate(V);
1837    } else
1838      return RValue::get(CI);
1839
1840  case ABIArgInfo::Ignore:
1841    // If we are ignoring an argument that had a result, make sure to
1842    // construct the appropriate return value for our caller.
1843    return GetUndefRValue(RetTy);
1844
1845  case ABIArgInfo::Coerce: {
1846    // FIXME: Avoid the conversion through memory if possible.
1847    llvm::Value *V = CreateTempAlloca(ConvertTypeForMem(RetTy), "coerce");
1848    CreateCoercedStore(CI, V, *this);
1849    if (RetTy->isAnyComplexType())
1850      return RValue::getComplex(LoadComplexFromAddr(V, false));
1851    else if (CodeGenFunction::hasAggregateLLVMType(RetTy))
1852      return RValue::getAggregate(V);
1853    else
1854      return RValue::get(EmitLoadOfScalar(V, false, RetTy));
1855  }
1856
1857  case ABIArgInfo::Expand:
1858    assert(0 && "Invalid ABI kind for return argument");
1859  }
1860
1861  assert(0 && "Unhandled ABIArgInfo::Kind");
1862  return RValue::get(0);
1863}
1864
1865/* VarArg handling */
1866
1867llvm::Value *CodeGenFunction::EmitVAArg(llvm::Value *VAListAddr, QualType Ty) {
1868  return CGM.getTypes().getABIInfo().EmitVAArg(VAListAddr, Ty, *this);
1869}
1870