X86ISelLowering.cpp revision 67f92a76497d1f50e0dd6279c81b45689bd463f5 
1//===-- X86ISelLowering.h - X86 DAG Lowering Interface ----------*- C++ -*-===//
2//
3//                     The LLVM Compiler Infrastructure
4//
5// This file was developed by Chris Lattner and is distributed under
6// the University of Illinois Open Source License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10// This file defines the interfaces that X86 uses to lower LLVM code into a
11// selection DAG.
12//
13//===----------------------------------------------------------------------===//
14
15#include "X86.h"
16#include "X86ISelLowering.h"
17#include "X86TargetMachine.h"
18#include "llvm/CallingConv.h"
19#include "llvm/Function.h"
20#include "llvm/CodeGen/MachineFrameInfo.h"
21#include "llvm/CodeGen/MachineFunction.h"
22#include "llvm/CodeGen/MachineInstrBuilder.h"
23#include "llvm/CodeGen/SelectionDAG.h"
24#include "llvm/CodeGen/SSARegMap.h"
25#include "llvm/Target/TargetOptions.h"
26using namespace llvm;
27
28// FIXME: temporary.
29#include "llvm/Support/CommandLine.h"
30static cl::opt<bool> EnableFastCC("enable-x86-fastcc", cl::Hidden,
31                                  cl::desc("Enable fastcc on X86"));
32
33X86TargetLowering::X86TargetLowering(TargetMachine &TM)
34  : TargetLowering(TM) {
35  // Set up the TargetLowering object.
36
37  // X86 is weird, it always uses i8 for shift amounts and setcc results.
38  setShiftAmountType(MVT::i8);
39  setSetCCResultType(MVT::i8);
40  setSetCCResultContents(ZeroOrOneSetCCResult);
41  setShiftAmountFlavor(Mask);   // shl X, 32 == shl X, 0
42
43  // Set up the register classes.
44  addRegisterClass(MVT::i8, X86::R8RegisterClass);
45  addRegisterClass(MVT::i16, X86::R16RegisterClass);
46  addRegisterClass(MVT::i32, X86::R32RegisterClass);
47
48  // Promote all UINT_TO_FP to larger SINT_TO_FP's, as X86 doesn't have this
49  // operation.
50  setOperationAction(ISD::UINT_TO_FP       , MVT::i1   , Promote);
51  setOperationAction(ISD::UINT_TO_FP       , MVT::i8   , Promote);
52  setOperationAction(ISD::UINT_TO_FP       , MVT::i16  , Promote);
53  setOperationAction(ISD::UINT_TO_FP       , MVT::i32  , Promote);
54
55  // Promote i1/i8 SINT_TO_FP to larger SINT_TO_FP's, as X86 doesn't have
56  // this operation.
57  setOperationAction(ISD::SINT_TO_FP       , MVT::i1   , Promote);
58  setOperationAction(ISD::SINT_TO_FP       , MVT::i8   , Promote);
59
60  if (!X86ScalarSSE) {
61    // We can handle SINT_TO_FP and FP_TO_SINT from/TO i64 even though i64
62    // isn't legal.
63    setOperationAction(ISD::SINT_TO_FP     , MVT::i64  , Custom);
64    setOperationAction(ISD::FP_TO_SINT     , MVT::i64  , Custom);
65    setOperationAction(ISD::FP_TO_SINT     , MVT::i32  , Custom);
66    setOperationAction(ISD::FP_TO_SINT     , MVT::i16  , Custom);
67  }
68
69  // Handle FP_TO_UINT by promoting the destination to a larger signed
70  // conversion.
71  setOperationAction(ISD::FP_TO_UINT       , MVT::i1   , Promote);
72  setOperationAction(ISD::FP_TO_UINT       , MVT::i8   , Promote);
73  setOperationAction(ISD::FP_TO_UINT       , MVT::i16  , Promote);
74
75  if (!X86ScalarSSE)
76    setOperationAction(ISD::FP_TO_UINT     , MVT::i32  , Promote);
77
78  // Promote i1/i8 FP_TO_SINT to larger FP_TO_SINTS's, as X86 doesn't have
79  // this operation.
80  setOperationAction(ISD::FP_TO_SINT       , MVT::i1   , Promote);
81  setOperationAction(ISD::FP_TO_SINT       , MVT::i8   , Promote);
82  setOperationAction(ISD::FP_TO_SINT       , MVT::i16  , Promote);
83
84  setOperationAction(ISD::BIT_CONVERT, MVT::f32, Expand);
85  setOperationAction(ISD::BIT_CONVERT, MVT::i32, Expand);
86
87  if (X86DAGIsel) {
88    setOperationAction(ISD::BRCOND         , MVT::Other, Custom);
89  }
90  setOperationAction(ISD::BRCONDTWOWAY     , MVT::Other, Expand);
91  setOperationAction(ISD::BRTWOWAY_CC      , MVT::Other, Expand);
92  setOperationAction(ISD::MEMMOVE          , MVT::Other, Expand);
93  setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i16  , Expand);
94  setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i8   , Expand);
95  setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i1   , Expand);
96  setOperationAction(ISD::FP_ROUND_INREG   , MVT::f32  , Expand);
97  setOperationAction(ISD::SEXTLOAD         , MVT::i1   , Expand);
98  setOperationAction(ISD::FREM             , MVT::f64  , Expand);
99  setOperationAction(ISD::CTPOP            , MVT::i8   , Expand);
100  setOperationAction(ISD::CTTZ             , MVT::i8   , Expand);
101  setOperationAction(ISD::CTLZ             , MVT::i8   , Expand);
102  setOperationAction(ISD::CTPOP            , MVT::i16  , Expand);
103  setOperationAction(ISD::CTTZ             , MVT::i16  , Expand);
104  setOperationAction(ISD::CTLZ             , MVT::i16  , Expand);
105  setOperationAction(ISD::CTPOP            , MVT::i32  , Expand);
106  setOperationAction(ISD::CTTZ             , MVT::i32  , Expand);
107  setOperationAction(ISD::CTLZ             , MVT::i32  , Expand);
108  setOperationAction(ISD::READCYCLECOUNTER , MVT::i64  , Custom);
109
110  setOperationAction(ISD::ROTL             , MVT::i8   , Expand);
111  setOperationAction(ISD::ROTR             , MVT::i8   , Expand);
112  setOperationAction(ISD::ROTL             , MVT::i16  , Expand);
113  setOperationAction(ISD::ROTR             , MVT::i16  , Expand);
114  setOperationAction(ISD::ROTL             , MVT::i32  , Expand);
115  setOperationAction(ISD::ROTR             , MVT::i32  , Expand);
116
117  setOperationAction(ISD::READIO           , MVT::i1   , Expand);
118  setOperationAction(ISD::READIO           , MVT::i8   , Expand);
119  setOperationAction(ISD::READIO           , MVT::i16  , Expand);
120  setOperationAction(ISD::READIO           , MVT::i32  , Expand);
121  setOperationAction(ISD::WRITEIO          , MVT::i1   , Expand);
122  setOperationAction(ISD::WRITEIO          , MVT::i8   , Expand);
123  setOperationAction(ISD::WRITEIO          , MVT::i16  , Expand);
124  setOperationAction(ISD::WRITEIO          , MVT::i32  , Expand);
125
126  // These should be promoted to a larger select which is supported.
127  setOperationAction(ISD::SELECT           , MVT::i1   , Promote);
128  setOperationAction(ISD::SELECT           , MVT::i8   , Promote);
129  if (X86DAGIsel) {
130    // X86 wants to expand cmov itself.
131    setOperationAction(ISD::SELECT         , MVT::i16  , Custom);
132    setOperationAction(ISD::SELECT         , MVT::i32  , Custom);
133    setOperationAction(ISD::SELECT         , MVT::f32  , Custom);
134    setOperationAction(ISD::SELECT         , MVT::f64  , Custom);
135    setOperationAction(ISD::SETCC          , MVT::i8   , Custom);
136    setOperationAction(ISD::SETCC          , MVT::i16  , Custom);
137    setOperationAction(ISD::SETCC          , MVT::i32  , Custom);
138    setOperationAction(ISD::SETCC          , MVT::f32  , Custom);
139    setOperationAction(ISD::SETCC          , MVT::f64  , Custom);
140    // X86 ret instruction may pop stack.
141    setOperationAction(ISD::RET            , MVT::Other, Custom);
142    // Darwin ABI issue.
143    setOperationAction(ISD::GlobalAddress  , MVT::i32  , Custom);
144    // 64-bit addm sub, shl, sra, srl (iff 32-bit x86)
145    setOperationAction(ISD::ADD_PARTS      , MVT::i32  , Custom);
146    setOperationAction(ISD::SUB_PARTS      , MVT::i32  , Custom);
147    setOperationAction(ISD::SHL_PARTS      , MVT::i32  , Custom);
148    setOperationAction(ISD::SRA_PARTS      , MVT::i32  , Custom);
149    setOperationAction(ISD::SRL_PARTS      , MVT::i32  , Custom);
150    // X86 wants to expand memset / memcpy itself.
151    setOperationAction(ISD::MEMSET         , MVT::Other, Custom);
152    setOperationAction(ISD::MEMCPY         , MVT::Other, Custom);
153  }
154
155  // We don't have line number support yet.
156  setOperationAction(ISD::LOCATION, MVT::Other, Expand);
157  setOperationAction(ISD::DEBUG_LOC, MVT::Other, Expand);
158  setOperationAction(ISD::DEBUG_LABEL, MVT::Other, Expand);
159
160  if (X86ScalarSSE) {
161    // Set up the FP register classes.
162    addRegisterClass(MVT::f32, X86::V4F4RegisterClass);
163    addRegisterClass(MVT::f64, X86::V2F8RegisterClass);
164
165    // SSE has no load+extend ops
166    setOperationAction(ISD::EXTLOAD,  MVT::f32, Expand);
167    setOperationAction(ISD::ZEXTLOAD, MVT::f32, Expand);
168
169    // SSE has no i16 to fp conversion, only i32
170    setOperationAction(ISD::SINT_TO_FP, MVT::i16, Promote);
171    setOperationAction(ISD::FP_TO_SINT, MVT::i16, Promote);
172
173    // Expand FP_TO_UINT into a select.
174    // FIXME: We would like to use a Custom expander here eventually to do
175    // the optimal thing for SSE vs. the default expansion in the legalizer.
176    setOperationAction(ISD::FP_TO_UINT       , MVT::i32  , Expand);
177
178    // We don't support sin/cos/sqrt/fmod
179    setOperationAction(ISD::FSIN , MVT::f64, Expand);
180    setOperationAction(ISD::FCOS , MVT::f64, Expand);
181    setOperationAction(ISD::FABS , MVT::f64, Expand);
182    setOperationAction(ISD::FNEG , MVT::f64, Expand);
183    setOperationAction(ISD::FREM , MVT::f64, Expand);
184    setOperationAction(ISD::FSIN , MVT::f32, Expand);
185    setOperationAction(ISD::FCOS , MVT::f32, Expand);
186    setOperationAction(ISD::FABS , MVT::f32, Expand);
187    setOperationAction(ISD::FNEG , MVT::f32, Expand);
188    setOperationAction(ISD::FREM , MVT::f32, Expand);
189
190    addLegalFPImmediate(+0.0); // xorps / xorpd
191  } else {
192    // Set up the FP register classes.
193    addRegisterClass(MVT::f64, X86::RFPRegisterClass);
194
195    if (!UnsafeFPMath) {
196      setOperationAction(ISD::FSIN           , MVT::f64  , Expand);
197      setOperationAction(ISD::FCOS           , MVT::f64  , Expand);
198    }
199
200    addLegalFPImmediate(+0.0); // FLD0
201    addLegalFPImmediate(+1.0); // FLD1
202    addLegalFPImmediate(-0.0); // FLD0/FCHS
203    addLegalFPImmediate(-1.0); // FLD1/FCHS
204  }
205  computeRegisterProperties();
206
207  maxStoresPerMemSet = 8; // For %llvm.memset -> sequence of stores
208  maxStoresPerMemCpy = 8; // For %llvm.memcpy -> sequence of stores
209  maxStoresPerMemMove = 8; // For %llvm.memmove -> sequence of stores
210  allowUnalignedMemoryAccesses = true; // x86 supports it!
211}
212
213std::vector<SDOperand>
214X86TargetLowering::LowerArguments(Function &F, SelectionDAG &DAG) {
215  if (F.getCallingConv() == CallingConv::Fast && EnableFastCC)
216    return LowerFastCCArguments(F, DAG);
217  return LowerCCCArguments(F, DAG);
218}
219
220std::pair<SDOperand, SDOperand>
221X86TargetLowering::LowerCallTo(SDOperand Chain, const Type *RetTy,
222                               bool isVarArg, unsigned CallingConv,
223                               bool isTailCall,
224                               SDOperand Callee, ArgListTy &Args,
225                               SelectionDAG &DAG) {
226  assert((!isVarArg || CallingConv == CallingConv::C) &&
227         "Only C takes varargs!");
228
229  // If the callee is a GlobalAddress node (quite common, every direct call is)
230  // turn it into a TargetGlobalAddress node so that legalize doesn't hack it.
231  if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee))
232    Callee = DAG.getTargetGlobalAddress(G->getGlobal(), getPointerTy());
233  else if (ExternalSymbolSDNode *S = dyn_cast<ExternalSymbolSDNode>(Callee))
234    Callee = DAG.getTargetExternalSymbol(S->getSymbol(), getPointerTy());
235
236  if (CallingConv == CallingConv::Fast && EnableFastCC)
237    return LowerFastCCCallTo(Chain, RetTy, isTailCall, Callee, Args, DAG);
238  return  LowerCCCCallTo(Chain, RetTy, isVarArg, isTailCall, Callee, Args, DAG);
239}
240
241SDOperand X86TargetLowering::LowerReturnTo(SDOperand Chain, SDOperand Op,
242                                           SelectionDAG &DAG) {
243  if (!X86DAGIsel)
244    return DAG.getNode(ISD::RET, MVT::Other, Chain, Op);
245
246  SDOperand Copy;
247  MVT::ValueType OpVT = Op.getValueType();
248  switch (OpVT) {
249    default: assert(0 && "Unknown type to return!");
250    case MVT::i32:
251      Copy = DAG.getCopyToReg(Chain, X86::EAX, Op, SDOperand());
252      break;
253    case MVT::i64: {
254      SDOperand Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, MVT::i32, Op,
255                                 DAG.getConstant(1, MVT::i32));
256      SDOperand Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, MVT::i32, Op,
257                                 DAG.getConstant(0, MVT::i32));
258      Copy = DAG.getCopyToReg(Chain, X86::EDX, Hi, SDOperand());
259      Copy = DAG.getCopyToReg(Copy,  X86::EAX, Lo, Copy.getValue(1));
260      break;
261    }
262    case MVT::f32:
263    case MVT::f64:
264      if (!X86ScalarSSE) {
265        if (OpVT == MVT::f32)
266          Op = DAG.getNode(ISD::FP_EXTEND, MVT::f64, Op);
267        std::vector<MVT::ValueType> Tys;
268        Tys.push_back(MVT::Other);
269        Tys.push_back(MVT::Flag);
270        std::vector<SDOperand> Ops;
271        Ops.push_back(Chain);
272        Ops.push_back(Op);
273        Copy = DAG.getNode(X86ISD::FP_SET_RESULT, Tys, Ops);
274      } else {
275        // Spill the value to memory and reload it into top of stack.
276        unsigned Size = MVT::getSizeInBits(OpVT)/8;
277        MachineFunction &MF = DAG.getMachineFunction();
278        int SSFI = MF.getFrameInfo()->CreateStackObject(Size, Size);
279        SDOperand StackSlot = DAG.getFrameIndex(SSFI, getPointerTy());
280        Chain = DAG.getNode(ISD::STORE, MVT::Other, Chain, Op,
281                            StackSlot, DAG.getSrcValue(NULL));
282        std::vector<MVT::ValueType> Tys;
283        Tys.push_back(MVT::f64);
284        Tys.push_back(MVT::Other);
285        std::vector<SDOperand> Ops;
286        Ops.push_back(Chain);
287        Ops.push_back(StackSlot);
288        Ops.push_back(DAG.getValueType(OpVT));
289        Copy = DAG.getNode(X86ISD::FLD, Tys, Ops);
290        Tys.clear();
291        Tys.push_back(MVT::Other);
292        Tys.push_back(MVT::Flag);
293        Ops.clear();
294        Ops.push_back(Copy.getValue(1));
295        Ops.push_back(Copy);
296        Copy = DAG.getNode(X86ISD::FP_SET_RESULT, Tys, Ops);
297      }
298      break;
299  }
300
301  return DAG.getNode(X86ISD::RET_FLAG, MVT::Other,
302                     Copy, DAG.getConstant(getBytesToPopOnReturn(), MVT::i16),
303                     Copy.getValue(1));
304}
305
306//===----------------------------------------------------------------------===//
307//                    C Calling Convention implementation
308//===----------------------------------------------------------------------===//
309
310std::vector<SDOperand>
311X86TargetLowering::LowerCCCArguments(Function &F, SelectionDAG &DAG) {
312  std::vector<SDOperand> ArgValues;
313
314  MachineFunction &MF = DAG.getMachineFunction();
315  MachineFrameInfo *MFI = MF.getFrameInfo();
316
317  // Add DAG nodes to load the arguments...  On entry to a function on the X86,
318  // the stack frame looks like this:
319  //
320  // [ESP] -- return address
321  // [ESP + 4] -- first argument (leftmost lexically)
322  // [ESP + 8] -- second argument, if first argument is four bytes in size
323  //    ...
324  //
325  unsigned ArgOffset = 0;   // Frame mechanisms handle retaddr slot
326  for (Function::arg_iterator I = F.arg_begin(), E = F.arg_end(); I != E; ++I) {
327    MVT::ValueType ObjectVT = getValueType(I->getType());
328    unsigned ArgIncrement = 4;
329    unsigned ObjSize;
330    switch (ObjectVT) {
331    default: assert(0 && "Unhandled argument type!");
332    case MVT::i1:
333    case MVT::i8:  ObjSize = 1;                break;
334    case MVT::i16: ObjSize = 2;                break;
335    case MVT::i32: ObjSize = 4;                break;
336    case MVT::i64: ObjSize = ArgIncrement = 8; break;
337    case MVT::f32: ObjSize = 4;                break;
338    case MVT::f64: ObjSize = ArgIncrement = 8; break;
339    }
340    // Create the frame index object for this incoming parameter...
341    int FI = MFI->CreateFixedObject(ObjSize, ArgOffset);
342
343    // Create the SelectionDAG nodes corresponding to a load from this parameter
344    SDOperand FIN = DAG.getFrameIndex(FI, MVT::i32);
345
346    // Don't codegen dead arguments.  FIXME: remove this check when we can nuke
347    // dead loads.
348    SDOperand ArgValue;
349    if (!I->use_empty())
350      ArgValue = DAG.getLoad(ObjectVT, DAG.getEntryNode(), FIN,
351                             DAG.getSrcValue(NULL));
352    else {
353      if (MVT::isInteger(ObjectVT))
354        ArgValue = DAG.getConstant(0, ObjectVT);
355      else
356        ArgValue = DAG.getConstantFP(0, ObjectVT);
357    }
358    ArgValues.push_back(ArgValue);
359
360    ArgOffset += ArgIncrement;   // Move on to the next argument...
361  }
362
363  // If the function takes variable number of arguments, make a frame index for
364  // the start of the first vararg value... for expansion of llvm.va_start.
365  if (F.isVarArg())
366    VarArgsFrameIndex = MFI->CreateFixedObject(1, ArgOffset);
367  ReturnAddrIndex = 0;     // No return address slot generated yet.
368  BytesToPopOnReturn = 0;  // Callee pops nothing.
369  BytesCallerReserves = ArgOffset;
370
371  // Finally, inform the code generator which regs we return values in.
372  switch (getValueType(F.getReturnType())) {
373  default: assert(0 && "Unknown type!");
374  case MVT::isVoid: break;
375  case MVT::i1:
376  case MVT::i8:
377  case MVT::i16:
378  case MVT::i32:
379    MF.addLiveOut(X86::EAX);
380    break;
381  case MVT::i64:
382    MF.addLiveOut(X86::EAX);
383    MF.addLiveOut(X86::EDX);
384    break;
385  case MVT::f32:
386  case MVT::f64:
387    MF.addLiveOut(X86::ST0);
388    break;
389  }
390  return ArgValues;
391}
392
393std::pair<SDOperand, SDOperand>
394X86TargetLowering::LowerCCCCallTo(SDOperand Chain, const Type *RetTy,
395                                  bool isVarArg, bool isTailCall,
396                                  SDOperand Callee, ArgListTy &Args,
397                                  SelectionDAG &DAG) {
398  // Count how many bytes are to be pushed on the stack.
399  unsigned NumBytes = 0;
400
401  if (Args.empty()) {
402    // Save zero bytes.
403    Chain = DAG.getNode(ISD::CALLSEQ_START, MVT::Other, Chain,
404                        DAG.getConstant(0, getPointerTy()));
405  } else {
406    for (unsigned i = 0, e = Args.size(); i != e; ++i)
407      switch (getValueType(Args[i].second)) {
408      default: assert(0 && "Unknown value type!");
409      case MVT::i1:
410      case MVT::i8:
411      case MVT::i16:
412      case MVT::i32:
413      case MVT::f32:
414        NumBytes += 4;
415        break;
416      case MVT::i64:
417      case MVT::f64:
418        NumBytes += 8;
419        break;
420      }
421
422    Chain = DAG.getNode(ISD::CALLSEQ_START, MVT::Other, Chain,
423                        DAG.getConstant(NumBytes, getPointerTy()));
424
425    // Arguments go on the stack in reverse order, as specified by the ABI.
426    unsigned ArgOffset = 0;
427    SDOperand StackPtr = DAG.getRegister(X86::ESP, MVT::i32);
428    std::vector<SDOperand> Stores;
429
430    for (unsigned i = 0, e = Args.size(); i != e; ++i) {
431      SDOperand PtrOff = DAG.getConstant(ArgOffset, getPointerTy());
432      PtrOff = DAG.getNode(ISD::ADD, MVT::i32, StackPtr, PtrOff);
433
434      switch (getValueType(Args[i].second)) {
435      default: assert(0 && "Unexpected ValueType for argument!");
436      case MVT::i1:
437      case MVT::i8:
438      case MVT::i16:
439        // Promote the integer to 32 bits.  If the input type is signed use a
440        // sign extend, otherwise use a zero extend.
441        if (Args[i].second->isSigned())
442          Args[i].first =DAG.getNode(ISD::SIGN_EXTEND, MVT::i32, Args[i].first);
443        else
444          Args[i].first =DAG.getNode(ISD::ZERO_EXTEND, MVT::i32, Args[i].first);
445
446        // FALL THROUGH
447      case MVT::i32:
448      case MVT::f32:
449        Stores.push_back(DAG.getNode(ISD::STORE, MVT::Other, Chain,
450                                     Args[i].first, PtrOff,
451                                     DAG.getSrcValue(NULL)));
452        ArgOffset += 4;
453        break;
454      case MVT::i64:
455      case MVT::f64:
456        Stores.push_back(DAG.getNode(ISD::STORE, MVT::Other, Chain,
457                                     Args[i].first, PtrOff,
458                                     DAG.getSrcValue(NULL)));
459        ArgOffset += 8;
460        break;
461      }
462    }
463    Chain = DAG.getNode(ISD::TokenFactor, MVT::Other, Stores);
464  }
465
466  std::vector<MVT::ValueType> RetVals;
467  MVT::ValueType RetTyVT = getValueType(RetTy);
468  RetVals.push_back(MVT::Other);
469
470  // The result values produced have to be legal.  Promote the result.
471  switch (RetTyVT) {
472  case MVT::isVoid: break;
473  default:
474    RetVals.push_back(RetTyVT);
475    break;
476  case MVT::i1:
477  case MVT::i8:
478  case MVT::i16:
479    RetVals.push_back(MVT::i32);
480    break;
481  case MVT::f32:
482    if (X86ScalarSSE)
483      RetVals.push_back(MVT::f32);
484    else
485      RetVals.push_back(MVT::f64);
486    break;
487  case MVT::i64:
488    RetVals.push_back(MVT::i32);
489    RetVals.push_back(MVT::i32);
490    break;
491  }
492
493  if (X86DAGIsel) {
494    std::vector<MVT::ValueType> NodeTys;
495    NodeTys.push_back(MVT::Other);   // Returns a chain
496    NodeTys.push_back(MVT::Flag);    // Returns a flag for retval copy to use.
497    std::vector<SDOperand> Ops;
498    Ops.push_back(Chain);
499    Ops.push_back(Callee);
500
501    // FIXME: Do not generate X86ISD::TAILCALL for now.
502    Chain = DAG.getNode(X86ISD::CALL, NodeTys, Ops);
503    SDOperand InFlag = Chain.getValue(1);
504
505    SDOperand RetVal;
506    if (RetTyVT != MVT::isVoid) {
507      switch (RetTyVT) {
508      default: assert(0 && "Unknown value type to return!");
509      case MVT::i1:
510      case MVT::i8:
511        RetVal = DAG.getCopyFromReg(Chain, X86::AL, MVT::i8, InFlag);
512        Chain = RetVal.getValue(1);
513        break;
514      case MVT::i16:
515        RetVal = DAG.getCopyFromReg(Chain, X86::AX, MVT::i16, InFlag);
516        Chain = RetVal.getValue(1);
517        break;
518      case MVT::i32:
519        RetVal = DAG.getCopyFromReg(Chain, X86::EAX, MVT::i32, InFlag);
520        Chain = RetVal.getValue(1);
521        break;
522      case MVT::i64: {
523        SDOperand Lo = DAG.getCopyFromReg(Chain, X86::EAX, MVT::i32, InFlag);
524        SDOperand Hi = DAG.getCopyFromReg(Lo.getValue(1), X86::EDX, MVT::i32,
525                                          Lo.getValue(2));
526        RetVal = DAG.getNode(ISD::BUILD_PAIR, MVT::i64, Lo, Hi);
527        Chain = Hi.getValue(1);
528        break;
529      }
530      case MVT::f32:
531      case MVT::f64: {
532        std::vector<MVT::ValueType> Tys;
533        Tys.push_back(MVT::f64);
534        Tys.push_back(MVT::Other);
535        std::vector<SDOperand> Ops;
536        Ops.push_back(Chain);
537        Ops.push_back(InFlag);
538        RetVal = DAG.getNode(X86ISD::FP_GET_RESULT, Tys, Ops);
539        Chain = RetVal.getValue(1);
540        if (X86ScalarSSE) {
541          unsigned Size = MVT::getSizeInBits(MVT::f64)/8;
542          MachineFunction &MF = DAG.getMachineFunction();
543          int SSFI = MF.getFrameInfo()->CreateStackObject(Size, Size);
544          SDOperand StackSlot = DAG.getFrameIndex(SSFI, getPointerTy());
545          Tys.clear();
546          Tys.push_back(MVT::Other);
547          Ops.clear();
548          Ops.push_back(Chain);
549          Ops.push_back(RetVal);
550          Ops.push_back(StackSlot);
551          Ops.push_back(DAG.getValueType(RetTyVT));
552          Chain = DAG.getNode(X86ISD::FST, Tys, Ops);
553          RetVal = DAG.getLoad(RetTyVT, Chain, StackSlot,
554                               DAG.getSrcValue(NULL));
555          Chain = RetVal.getValue(1);
556        } else if (RetTyVT == MVT::f32)
557          RetVal = DAG.getNode(ISD::FP_ROUND, MVT::f32, RetVal);
558        break;
559      }
560      }
561    }
562
563    Chain = DAG.getNode(ISD::CALLSEQ_END, MVT::Other, Chain,
564                        DAG.getConstant(NumBytes, getPointerTy()),
565                        DAG.getConstant(0, getPointerTy()));
566    return std::make_pair(RetVal, Chain);
567  } else {
568    std::vector<SDOperand> Ops;
569    Ops.push_back(Chain);
570    Ops.push_back(Callee);
571    Ops.push_back(DAG.getConstant(NumBytes, getPointerTy()));
572    Ops.push_back(DAG.getConstant(0, getPointerTy()));
573
574    SDOperand TheCall = DAG.getNode(isTailCall ? X86ISD::TAILCALL : X86ISD::CALL,
575                                    RetVals, Ops);
576
577    SDOperand ResultVal;
578    switch (RetTyVT) {
579    case MVT::isVoid: break;
580    default:
581      ResultVal = TheCall.getValue(1);
582      break;
583    case MVT::i1:
584    case MVT::i8:
585    case MVT::i16:
586      ResultVal = DAG.getNode(ISD::TRUNCATE, RetTyVT, TheCall.getValue(1));
587      break;
588    case MVT::f32:
589      // FIXME: we would really like to remember that this FP_ROUND operation is
590      // okay to eliminate if we allow excess FP precision.
591      ResultVal = DAG.getNode(ISD::FP_ROUND, MVT::f32, TheCall.getValue(1));
592      break;
593    case MVT::i64:
594      ResultVal = DAG.getNode(ISD::BUILD_PAIR, MVT::i64, TheCall.getValue(1),
595                              TheCall.getValue(2));
596      break;
597    }
598
599    Chain = DAG.getNode(ISD::CALLSEQ_END, MVT::Other, TheCall);
600    return std::make_pair(ResultVal, Chain);
601  }
602}
603
604SDOperand
605X86TargetLowering::LowerVAStart(SDOperand Chain, SDOperand VAListP,
606                                Value *VAListV, SelectionDAG &DAG) {
607  // vastart just stores the address of the VarArgsFrameIndex slot.
608  SDOperand FR = DAG.getFrameIndex(VarArgsFrameIndex, MVT::i32);
609  return DAG.getNode(ISD::STORE, MVT::Other, Chain, FR, VAListP,
610                     DAG.getSrcValue(VAListV));
611}
612
613
614std::pair<SDOperand,SDOperand>
615X86TargetLowering::LowerVAArg(SDOperand Chain, SDOperand VAListP,
616                              Value *VAListV, const Type *ArgTy,
617                              SelectionDAG &DAG) {
618  MVT::ValueType ArgVT = getValueType(ArgTy);
619  SDOperand Val = DAG.getLoad(MVT::i32, Chain,
620                              VAListP, DAG.getSrcValue(VAListV));
621  SDOperand Result = DAG.getLoad(ArgVT, Chain, Val,
622                                 DAG.getSrcValue(NULL));
623  unsigned Amt;
624  if (ArgVT == MVT::i32)
625    Amt = 4;
626  else {
627    assert((ArgVT == MVT::i64 || ArgVT == MVT::f64) &&
628           "Other types should have been promoted for varargs!");
629    Amt = 8;
630  }
631  Val = DAG.getNode(ISD::ADD, Val.getValueType(), Val,
632                    DAG.getConstant(Amt, Val.getValueType()));
633  Chain = DAG.getNode(ISD::STORE, MVT::Other, Chain,
634                      Val, VAListP, DAG.getSrcValue(VAListV));
635  return std::make_pair(Result, Chain);
636}
637
638//===----------------------------------------------------------------------===//
639//                    Fast Calling Convention implementation
640//===----------------------------------------------------------------------===//
641//
642// The X86 'fast' calling convention passes up to two integer arguments in
643// registers (an appropriate portion of EAX/EDX), passes arguments in C order,
644// and requires that the callee pop its arguments off the stack (allowing proper
645// tail calls), and has the same return value conventions as C calling convs.
646//
647// This calling convention always arranges for the callee pop value to be 8n+4
648// bytes, which is needed for tail recursion elimination and stack alignment
649// reasons.
650//
651// Note that this can be enhanced in the future to pass fp vals in registers
652// (when we have a global fp allocator) and do other tricks.
653//
654
655/// AddLiveIn - This helper function adds the specified physical register to the
656/// MachineFunction as a live in value.  It also creates a corresponding virtual
657/// register for it.
658static unsigned AddLiveIn(MachineFunction &MF, unsigned PReg,
659                          TargetRegisterClass *RC) {
660  assert(RC->contains(PReg) && "Not the correct regclass!");
661  unsigned VReg = MF.getSSARegMap()->createVirtualRegister(RC);
662  MF.addLiveIn(PReg, VReg);
663  return VReg;
664}
665
666
667std::vector<SDOperand>
668X86TargetLowering::LowerFastCCArguments(Function &F, SelectionDAG &DAG) {
669  std::vector<SDOperand> ArgValues;
670
671  MachineFunction &MF = DAG.getMachineFunction();
672  MachineFrameInfo *MFI = MF.getFrameInfo();
673
674  // Add DAG nodes to load the arguments...  On entry to a function the stack
675  // frame looks like this:
676  //
677  // [ESP] -- return address
678  // [ESP + 4] -- first nonreg argument (leftmost lexically)
679  // [ESP + 8] -- second nonreg argument, if first argument is 4 bytes in size
680  //    ...
681  unsigned ArgOffset = 0;   // Frame mechanisms handle retaddr slot
682
683  // Keep track of the number of integer regs passed so far.  This can be either
684  // 0 (neither EAX or EDX used), 1 (EAX is used) or 2 (EAX and EDX are both
685  // used).
686  unsigned NumIntRegs = 0;
687
688  for (Function::arg_iterator I = F.arg_begin(), E = F.arg_end(); I != E; ++I) {
689    MVT::ValueType ObjectVT = getValueType(I->getType());
690    unsigned ArgIncrement = 4;
691    unsigned ObjSize = 0;
692    SDOperand ArgValue;
693
694    switch (ObjectVT) {
695    default: assert(0 && "Unhandled argument type!");
696    case MVT::i1:
697    case MVT::i8:
698      if (NumIntRegs < 2) {
699        if (!I->use_empty()) {
700          unsigned VReg = AddLiveIn(MF, NumIntRegs ? X86::DL : X86::AL,
701                                    X86::R8RegisterClass);
702          ArgValue = DAG.getCopyFromReg(DAG.getRoot(), VReg, MVT::i8);
703          DAG.setRoot(ArgValue.getValue(1));
704          if (ObjectVT == MVT::i1)
705            // FIXME: Should insert a assertzext here.
706            ArgValue = DAG.getNode(ISD::TRUNCATE, MVT::i1, ArgValue);
707        }
708        ++NumIntRegs;
709        break;
710      }
711
712      ObjSize = 1;
713      break;
714    case MVT::i16:
715      if (NumIntRegs < 2) {
716        if (!I->use_empty()) {
717          unsigned VReg = AddLiveIn(MF, NumIntRegs ? X86::DX : X86::AX,
718                                    X86::R16RegisterClass);
719          ArgValue = DAG.getCopyFromReg(DAG.getRoot(), VReg, MVT::i16);
720          DAG.setRoot(ArgValue.getValue(1));
721        }
722        ++NumIntRegs;
723        break;
724      }
725      ObjSize = 2;
726      break;
727    case MVT::i32:
728      if (NumIntRegs < 2) {
729        if (!I->use_empty()) {
730          unsigned VReg = AddLiveIn(MF,NumIntRegs ? X86::EDX : X86::EAX,
731                                    X86::R32RegisterClass);
732          ArgValue = DAG.getCopyFromReg(DAG.getRoot(), VReg, MVT::i32);
733          DAG.setRoot(ArgValue.getValue(1));
734        }
735        ++NumIntRegs;
736        break;
737      }
738      ObjSize = 4;
739      break;
740    case MVT::i64:
741      if (NumIntRegs == 0) {
742        if (!I->use_empty()) {
743          unsigned BotReg = AddLiveIn(MF, X86::EAX, X86::R32RegisterClass);
744          unsigned TopReg = AddLiveIn(MF, X86::EDX, X86::R32RegisterClass);
745
746          SDOperand Low = DAG.getCopyFromReg(DAG.getRoot(), BotReg, MVT::i32);
747          SDOperand Hi  = DAG.getCopyFromReg(Low.getValue(1), TopReg, MVT::i32);
748          DAG.setRoot(Hi.getValue(1));
749
750          ArgValue = DAG.getNode(ISD::BUILD_PAIR, MVT::i64, Low, Hi);
751        }
752        NumIntRegs = 2;
753        break;
754      } else if (NumIntRegs == 1) {
755        if (!I->use_empty()) {
756          unsigned BotReg = AddLiveIn(MF, X86::EDX, X86::R32RegisterClass);
757          SDOperand Low = DAG.getCopyFromReg(DAG.getRoot(), BotReg, MVT::i32);
758          DAG.setRoot(Low.getValue(1));
759
760          // Load the high part from memory.
761          // Create the frame index object for this incoming parameter...
762          int FI = MFI->CreateFixedObject(4, ArgOffset);
763          SDOperand FIN = DAG.getFrameIndex(FI, MVT::i32);
764          SDOperand Hi = DAG.getLoad(MVT::i32, DAG.getEntryNode(), FIN,
765                                     DAG.getSrcValue(NULL));
766          ArgValue = DAG.getNode(ISD::BUILD_PAIR, MVT::i64, Low, Hi);
767        }
768        ArgOffset += 4;
769        NumIntRegs = 2;
770        break;
771      }
772      ObjSize = ArgIncrement = 8;
773      break;
774    case MVT::f32: ObjSize = 4;                break;
775    case MVT::f64: ObjSize = ArgIncrement = 8; break;
776    }
777
778    // Don't codegen dead arguments.  FIXME: remove this check when we can nuke
779    // dead loads.
780    if (ObjSize && !I->use_empty()) {
781      // Create the frame index object for this incoming parameter...
782      int FI = MFI->CreateFixedObject(ObjSize, ArgOffset);
783
784      // Create the SelectionDAG nodes corresponding to a load from this
785      // parameter.
786      SDOperand FIN = DAG.getFrameIndex(FI, MVT::i32);
787
788      ArgValue = DAG.getLoad(ObjectVT, DAG.getEntryNode(), FIN,
789                             DAG.getSrcValue(NULL));
790    } else if (ArgValue.Val == 0) {
791      if (MVT::isInteger(ObjectVT))
792        ArgValue = DAG.getConstant(0, ObjectVT);
793      else
794        ArgValue = DAG.getConstantFP(0, ObjectVT);
795    }
796    ArgValues.push_back(ArgValue);
797
798    if (ObjSize)
799      ArgOffset += ArgIncrement;   // Move on to the next argument.
800  }
801
802  // Make sure the instruction takes 8n+4 bytes to make sure the start of the
803  // arguments and the arguments after the retaddr has been pushed are aligned.
804  if ((ArgOffset & 7) == 0)
805    ArgOffset += 4;
806
807  VarArgsFrameIndex = 0xAAAAAAA;   // fastcc functions can't have varargs.
808  ReturnAddrIndex = 0;             // No return address slot generated yet.
809  BytesToPopOnReturn = ArgOffset;  // Callee pops all stack arguments.
810  BytesCallerReserves = 0;
811
812  // Finally, inform the code generator which regs we return values in.
813  switch (getValueType(F.getReturnType())) {
814  default: assert(0 && "Unknown type!");
815  case MVT::isVoid: break;
816  case MVT::i1:
817  case MVT::i8:
818  case MVT::i16:
819  case MVT::i32:
820    MF.addLiveOut(X86::EAX);
821    break;
822  case MVT::i64:
823    MF.addLiveOut(X86::EAX);
824    MF.addLiveOut(X86::EDX);
825    break;
826  case MVT::f32:
827  case MVT::f64:
828    MF.addLiveOut(X86::ST0);
829    break;
830  }
831  return ArgValues;
832}
833
834std::pair<SDOperand, SDOperand>
835X86TargetLowering::LowerFastCCCallTo(SDOperand Chain, const Type *RetTy,
836                                     bool isTailCall, SDOperand Callee,
837                                     ArgListTy &Args, SelectionDAG &DAG) {
838  // Count how many bytes are to be pushed on the stack.
839  unsigned NumBytes = 0;
840
841  // Keep track of the number of integer regs passed so far.  This can be either
842  // 0 (neither EAX or EDX used), 1 (EAX is used) or 2 (EAX and EDX are both
843  // used).
844  unsigned NumIntRegs = 0;
845
846  for (unsigned i = 0, e = Args.size(); i != e; ++i)
847    switch (getValueType(Args[i].second)) {
848    default: assert(0 && "Unknown value type!");
849    case MVT::i1:
850    case MVT::i8:
851    case MVT::i16:
852    case MVT::i32:
853      if (NumIntRegs < 2) {
854        ++NumIntRegs;
855        break;
856      }
857      // fall through
858    case MVT::f32:
859      NumBytes += 4;
860      break;
861    case MVT::i64:
862      if (NumIntRegs == 0) {
863        NumIntRegs = 2;
864        break;
865      } else if (NumIntRegs == 1) {
866        NumIntRegs = 2;
867        NumBytes += 4;
868        break;
869      }
870
871      // fall through
872    case MVT::f64:
873      NumBytes += 8;
874      break;
875    }
876
877  // Make sure the instruction takes 8n+4 bytes to make sure the start of the
878  // arguments and the arguments after the retaddr has been pushed are aligned.
879  if ((NumBytes & 7) == 0)
880    NumBytes += 4;
881
882  Chain = DAG.getNode(ISD::CALLSEQ_START, MVT::Other, Chain,
883                      DAG.getConstant(NumBytes, getPointerTy()));
884
885  // Arguments go on the stack in reverse order, as specified by the ABI.
886  unsigned ArgOffset = 0;
887  SDOperand StackPtr = DAG.getCopyFromReg(DAG.getEntryNode(),
888                                          X86::ESP, MVT::i32);
889  NumIntRegs = 0;
890  std::vector<SDOperand> Stores;
891  std::vector<SDOperand> RegValuesToPass;
892  for (unsigned i = 0, e = Args.size(); i != e; ++i) {
893    switch (getValueType(Args[i].second)) {
894    default: assert(0 && "Unexpected ValueType for argument!");
895    case MVT::i1:
896      Args[i].first = DAG.getNode(ISD::ANY_EXTEND, MVT::i8, Args[i].first);
897      // Fall through.
898    case MVT::i8:
899    case MVT::i16:
900    case MVT::i32:
901      if (NumIntRegs < 2) {
902        RegValuesToPass.push_back(Args[i].first);
903        ++NumIntRegs;
904        break;
905      }
906      // Fall through
907    case MVT::f32: {
908      SDOperand PtrOff = DAG.getConstant(ArgOffset, getPointerTy());
909      PtrOff = DAG.getNode(ISD::ADD, MVT::i32, StackPtr, PtrOff);
910      Stores.push_back(DAG.getNode(ISD::STORE, MVT::Other, Chain,
911                                   Args[i].first, PtrOff,
912                                   DAG.getSrcValue(NULL)));
913      ArgOffset += 4;
914      break;
915    }
916    case MVT::i64:
917      if (NumIntRegs < 2) {    // Can pass part of it in regs?
918        SDOperand Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, MVT::i32,
919                                   Args[i].first, DAG.getConstant(1, MVT::i32));
920        SDOperand Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, MVT::i32,
921                                   Args[i].first, DAG.getConstant(0, MVT::i32));
922        RegValuesToPass.push_back(Lo);
923        ++NumIntRegs;
924        if (NumIntRegs < 2) {   // Pass both parts in regs?
925          RegValuesToPass.push_back(Hi);
926          ++NumIntRegs;
927        } else {
928          // Pass the high part in memory.
929          SDOperand PtrOff = DAG.getConstant(ArgOffset, getPointerTy());
930          PtrOff = DAG.getNode(ISD::ADD, MVT::i32, StackPtr, PtrOff);
931          Stores.push_back(DAG.getNode(ISD::STORE, MVT::Other, Chain,
932                                       Hi, PtrOff, DAG.getSrcValue(NULL)));
933          ArgOffset += 4;
934        }
935        break;
936      }
937      // Fall through
938    case MVT::f64:
939      SDOperand PtrOff = DAG.getConstant(ArgOffset, getPointerTy());
940      PtrOff = DAG.getNode(ISD::ADD, MVT::i32, StackPtr, PtrOff);
941      Stores.push_back(DAG.getNode(ISD::STORE, MVT::Other, Chain,
942                                   Args[i].first, PtrOff,
943                                   DAG.getSrcValue(NULL)));
944      ArgOffset += 8;
945      break;
946    }
947  }
948  if (!Stores.empty())
949    Chain = DAG.getNode(ISD::TokenFactor, MVT::Other, Stores);
950
951  // Make sure the instruction takes 8n+4 bytes to make sure the start of the
952  // arguments and the arguments after the retaddr has been pushed are aligned.
953  if ((ArgOffset & 7) == 0)
954    ArgOffset += 4;
955
956  std::vector<MVT::ValueType> RetVals;
957  MVT::ValueType RetTyVT = getValueType(RetTy);
958
959  RetVals.push_back(MVT::Other);
960
961  // The result values produced have to be legal.  Promote the result.
962  switch (RetTyVT) {
963  case MVT::isVoid: break;
964  default:
965    RetVals.push_back(RetTyVT);
966    break;
967  case MVT::i1:
968  case MVT::i8:
969  case MVT::i16:
970    RetVals.push_back(MVT::i32);
971    break;
972  case MVT::f32:
973    if (X86ScalarSSE)
974      RetVals.push_back(MVT::f32);
975    else
976      RetVals.push_back(MVT::f64);
977    break;
978  case MVT::i64:
979    RetVals.push_back(MVT::i32);
980    RetVals.push_back(MVT::i32);
981    break;
982  }
983
984  if (X86DAGIsel) {
985    // Build a sequence of copy-to-reg nodes chained together with token chain
986    // and flag operands which copy the outgoing args into registers.
987    SDOperand InFlag;
988    for (unsigned i = 0, e = RegValuesToPass.size(); i != e; ++i) {
989      unsigned CCReg;
990      SDOperand RegToPass = RegValuesToPass[i];
991      switch (RegToPass.getValueType()) {
992      default: assert(0 && "Bad thing to pass in regs");
993      case MVT::i8:
994        CCReg = (i == 0) ? X86::AL  : X86::DL;
995        break;
996      case MVT::i16:
997        CCReg = (i == 0) ? X86::AX  : X86::DX;
998        break;
999      case MVT::i32:
1000        CCReg = (i == 0) ? X86::EAX : X86::EDX;
1001        break;
1002      }
1003
1004      Chain = DAG.getCopyToReg(Chain, CCReg, RegToPass, InFlag);
1005      InFlag = Chain.getValue(1);
1006    }
1007
1008    std::vector<MVT::ValueType> NodeTys;
1009    NodeTys.push_back(MVT::Other);   // Returns a chain
1010    NodeTys.push_back(MVT::Flag);    // Returns a flag for retval copy to use.
1011    std::vector<SDOperand> Ops;
1012    Ops.push_back(Chain);
1013    Ops.push_back(Callee);
1014    if (InFlag.Val)
1015      Ops.push_back(InFlag);
1016
1017    // FIXME: Do not generate X86ISD::TAILCALL for now.
1018    Chain = DAG.getNode(X86ISD::CALL, NodeTys, Ops);
1019    InFlag = Chain.getValue(1);
1020
1021    SDOperand RetVal;
1022    if (RetTyVT != MVT::isVoid) {
1023      switch (RetTyVT) {
1024      default: assert(0 && "Unknown value type to return!");
1025      case MVT::i1:
1026      case MVT::i8:
1027        RetVal = DAG.getCopyFromReg(Chain, X86::AL, MVT::i8, InFlag);
1028        Chain = RetVal.getValue(1);
1029        break;
1030      case MVT::i16:
1031        RetVal = DAG.getCopyFromReg(Chain, X86::AX, MVT::i16, InFlag);
1032        Chain = RetVal.getValue(1);
1033        break;
1034      case MVT::i32:
1035        RetVal = DAG.getCopyFromReg(Chain, X86::EAX, MVT::i32, InFlag);
1036        Chain = RetVal.getValue(1);
1037        break;
1038      case MVT::i64: {
1039        SDOperand Lo = DAG.getCopyFromReg(Chain, X86::EAX, MVT::i32, InFlag);
1040        SDOperand Hi = DAG.getCopyFromReg(Lo.getValue(1), X86::EDX, MVT::i32,
1041                                          Lo.getValue(2));
1042        RetVal = DAG.getNode(ISD::BUILD_PAIR, MVT::i64, Lo, Hi);
1043        Chain = Hi.getValue(1);
1044        break;
1045      }
1046      case MVT::f32:
1047      case MVT::f64: {
1048        std::vector<MVT::ValueType> Tys;
1049        Tys.push_back(MVT::f64);
1050        Tys.push_back(MVT::Other);
1051        std::vector<SDOperand> Ops;
1052        Ops.push_back(Chain);
1053        Ops.push_back(InFlag);
1054        RetVal = DAG.getNode(X86ISD::FP_GET_RESULT, Tys, Ops);
1055        Chain = RetVal.getValue(1);
1056        if (X86ScalarSSE) {
1057          unsigned Size = MVT::getSizeInBits(MVT::f64)/8;
1058          MachineFunction &MF = DAG.getMachineFunction();
1059          int SSFI = MF.getFrameInfo()->CreateStackObject(Size, Size);
1060          SDOperand StackSlot = DAG.getFrameIndex(SSFI, getPointerTy());
1061          Tys.clear();
1062          Tys.push_back(MVT::Other);
1063          Ops.clear();
1064          Ops.push_back(Chain);
1065          Ops.push_back(RetVal);
1066          Ops.push_back(StackSlot);
1067          Ops.push_back(DAG.getValueType(RetTyVT));
1068          Chain = DAG.getNode(X86ISD::FST, Tys, Ops);
1069          RetVal = DAG.getLoad(RetTyVT, Chain, StackSlot,
1070                               DAG.getSrcValue(NULL));
1071          Chain = RetVal.getValue(1);
1072        } else if (RetTyVT == MVT::f32)
1073          RetVal = DAG.getNode(ISD::FP_ROUND, MVT::f32, RetVal);
1074        break;
1075      }
1076      }
1077    }
1078
1079    Chain = DAG.getNode(ISD::CALLSEQ_END, MVT::Other, Chain,
1080                        DAG.getConstant(ArgOffset, getPointerTy()),
1081                        DAG.getConstant(ArgOffset, getPointerTy()));
1082    return std::make_pair(RetVal, Chain);
1083  } else {
1084    std::vector<SDOperand> Ops;
1085    Ops.push_back(Chain);
1086    Ops.push_back(Callee);
1087    Ops.push_back(DAG.getConstant(ArgOffset, getPointerTy()));
1088    // Callee pops all arg values on the stack.
1089    Ops.push_back(DAG.getConstant(ArgOffset, getPointerTy()));
1090
1091    // Pass register arguments as needed.
1092    Ops.insert(Ops.end(), RegValuesToPass.begin(), RegValuesToPass.end());
1093
1094    SDOperand TheCall = DAG.getNode(isTailCall ? X86ISD::TAILCALL : X86ISD::CALL,
1095                                    RetVals, Ops);
1096    Chain = DAG.getNode(ISD::CALLSEQ_END, MVT::Other, TheCall);
1097
1098    SDOperand ResultVal;
1099    switch (RetTyVT) {
1100    case MVT::isVoid: break;
1101    default:
1102      ResultVal = TheCall.getValue(1);
1103      break;
1104    case MVT::i1:
1105    case MVT::i8:
1106    case MVT::i16:
1107      ResultVal = DAG.getNode(ISD::TRUNCATE, RetTyVT, TheCall.getValue(1));
1108      break;
1109    case MVT::f32:
1110      // FIXME: we would really like to remember that this FP_ROUND operation is
1111      // okay to eliminate if we allow excess FP precision.
1112      ResultVal = DAG.getNode(ISD::FP_ROUND, MVT::f32, TheCall.getValue(1));
1113      break;
1114    case MVT::i64:
1115      ResultVal = DAG.getNode(ISD::BUILD_PAIR, MVT::i64, TheCall.getValue(1),
1116                              TheCall.getValue(2));
1117      break;
1118    }
1119
1120    return std::make_pair(ResultVal, Chain);
1121  }
1122}
1123
1124SDOperand X86TargetLowering::getReturnAddressFrameIndex(SelectionDAG &DAG) {
1125  if (ReturnAddrIndex == 0) {
1126    // Set up a frame object for the return address.
1127    MachineFunction &MF = DAG.getMachineFunction();
1128    ReturnAddrIndex = MF.getFrameInfo()->CreateFixedObject(4, -4);
1129  }
1130
1131  return DAG.getFrameIndex(ReturnAddrIndex, MVT::i32);
1132}
1133
1134
1135
1136std::pair<SDOperand, SDOperand> X86TargetLowering::
1137LowerFrameReturnAddress(bool isFrameAddress, SDOperand Chain, unsigned Depth,
1138                        SelectionDAG &DAG) {
1139  SDOperand Result;
1140  if (Depth)        // Depths > 0 not supported yet!
1141    Result = DAG.getConstant(0, getPointerTy());
1142  else {
1143    SDOperand RetAddrFI = getReturnAddressFrameIndex(DAG);
1144    if (!isFrameAddress)
1145      // Just load the return address
1146      Result = DAG.getLoad(MVT::i32, DAG.getEntryNode(), RetAddrFI,
1147                           DAG.getSrcValue(NULL));
1148    else
1149      Result = DAG.getNode(ISD::SUB, MVT::i32, RetAddrFI,
1150                           DAG.getConstant(4, MVT::i32));
1151  }
1152  return std::make_pair(Result, Chain);
1153}
1154
1155/// getCondBrOpcodeForX86CC - Returns the X86 conditional branch opcode
1156/// which corresponds to the condition code.
1157static unsigned getCondBrOpcodeForX86CC(unsigned X86CC) {
1158  switch (X86CC) {
1159  default: assert(0 && "Unknown X86 conditional code!");
1160  case X86ISD::COND_A:  return X86::JA;
1161  case X86ISD::COND_AE: return X86::JAE;
1162  case X86ISD::COND_B:  return X86::JB;
1163  case X86ISD::COND_BE: return X86::JBE;
1164  case X86ISD::COND_E:  return X86::JE;
1165  case X86ISD::COND_G:  return X86::JG;
1166  case X86ISD::COND_GE: return X86::JGE;
1167  case X86ISD::COND_L:  return X86::JL;
1168  case X86ISD::COND_LE: return X86::JLE;
1169  case X86ISD::COND_NE: return X86::JNE;
1170  case X86ISD::COND_NO: return X86::JNO;
1171  case X86ISD::COND_NP: return X86::JNP;
1172  case X86ISD::COND_NS: return X86::JNS;
1173  case X86ISD::COND_O:  return X86::JO;
1174  case X86ISD::COND_P:  return X86::JP;
1175  case X86ISD::COND_S:  return X86::JS;
1176  }
1177}
1178
1179/// getX86CC - do a one to one translation of a ISD::CondCode to the X86
1180/// specific condition code. It returns a X86ISD::COND_INVALID if it cannot
1181/// do a direct translation.
1182static unsigned getX86CC(SDOperand CC, bool isFP) {
1183  ISD::CondCode SetCCOpcode = cast<CondCodeSDNode>(CC)->get();
1184  unsigned X86CC = X86ISD::COND_INVALID;
1185  if (!isFP) {
1186    switch (SetCCOpcode) {
1187    default: break;
1188    case ISD::SETEQ:  X86CC = X86ISD::COND_E;  break;
1189    case ISD::SETGT:  X86CC = X86ISD::COND_G;  break;
1190    case ISD::SETGE:  X86CC = X86ISD::COND_GE; break;
1191    case ISD::SETLT:  X86CC = X86ISD::COND_L;  break;
1192    case ISD::SETLE:  X86CC = X86ISD::COND_LE; break;
1193    case ISD::SETNE:  X86CC = X86ISD::COND_NE; break;
1194    case ISD::SETULT: X86CC = X86ISD::COND_B;  break;
1195    case ISD::SETUGT: X86CC = X86ISD::COND_A;  break;
1196    case ISD::SETULE: X86CC = X86ISD::COND_BE; break;
1197    case ISD::SETUGE: X86CC = X86ISD::COND_AE; break;
1198    }
1199  } else {
1200    // On a floating point condition, the flags are set as follows:
1201    // ZF  PF  CF   op
1202    //  0 | 0 | 0 | X > Y
1203    //  0 | 0 | 1 | X < Y
1204    //  1 | 0 | 0 | X == Y
1205    //  1 | 1 | 1 | unordered
1206    switch (SetCCOpcode) {
1207    default: break;
1208    case ISD::SETUEQ:
1209    case ISD::SETEQ: X86CC = X86ISD::COND_E;  break;
1210    case ISD::SETOGT:
1211    case ISD::SETGT: X86CC = X86ISD::COND_A;  break;
1212    case ISD::SETOGE:
1213    case ISD::SETGE: X86CC = X86ISD::COND_AE; break;
1214    case ISD::SETULT:
1215    case ISD::SETLT: X86CC = X86ISD::COND_B;  break;
1216    case ISD::SETULE:
1217    case ISD::SETLE: X86CC = X86ISD::COND_BE; break;
1218    case ISD::SETONE:
1219    case ISD::SETNE: X86CC = X86ISD::COND_NE; break;
1220    case ISD::SETUO: X86CC = X86ISD::COND_P;  break;
1221    case ISD::SETO:  X86CC = X86ISD::COND_NP; break;
1222    }
1223  }
1224  return X86CC;
1225}
1226
1227/// hasFPCMov - is there a floating point cmov for the specific X86 condition
1228/// code. Current x86 isa includes the following FP cmov instructions:
1229/// fcmovb, fcomvbe, fcomve, fcmovu, fcmovae, fcmova, fcmovne, fcmovnu.
1230static bool hasFPCMov(unsigned X86CC) {
1231  switch (X86CC) {
1232  default:
1233    return false;
1234  case X86ISD::COND_B:
1235  case X86ISD::COND_BE:
1236  case X86ISD::COND_E:
1237  case X86ISD::COND_P:
1238  case X86ISD::COND_A:
1239  case X86ISD::COND_AE:
1240  case X86ISD::COND_NE:
1241  case X86ISD::COND_NP:
1242    return true;
1243  }
1244}
1245
1246MachineBasicBlock *
1247X86TargetLowering::InsertAtEndOfBasicBlock(MachineInstr *MI,
1248                                           MachineBasicBlock *BB) {
1249  assert((MI->getOpcode() == X86::CMOV_FR32 ||
1250          MI->getOpcode() == X86::CMOV_FR64) &&
1251         "Unexpected instr type to insert");
1252
1253  // To "insert" a SELECT_CC instruction, we actually have to insert the diamond
1254  // control-flow pattern.  The incoming instruction knows the destination vreg
1255  // to set, the condition code register to branch on, the true/false values to
1256  // select between, and a branch opcode to use.
1257  const BasicBlock *LLVM_BB = BB->getBasicBlock();
1258  ilist<MachineBasicBlock>::iterator It = BB;
1259  ++It;
1260
1261  //  thisMBB:
1262  //  ...
1263  //   TrueVal = ...
1264  //   cmpTY ccX, r1, r2
1265  //   bCC copy1MBB
1266  //   fallthrough --> copy0MBB
1267  MachineBasicBlock *thisMBB = BB;
1268  MachineBasicBlock *copy0MBB = new MachineBasicBlock(LLVM_BB);
1269  MachineBasicBlock *sinkMBB = new MachineBasicBlock(LLVM_BB);
1270  unsigned Opc = getCondBrOpcodeForX86CC(MI->getOperand(3).getImmedValue());
1271  BuildMI(BB, Opc, 1).addMBB(sinkMBB);
1272  MachineFunction *F = BB->getParent();
1273  F->getBasicBlockList().insert(It, copy0MBB);
1274  F->getBasicBlockList().insert(It, sinkMBB);
1275  // Update machine-CFG edges
1276  BB->addSuccessor(copy0MBB);
1277  BB->addSuccessor(sinkMBB);
1278
1279  //  copy0MBB:
1280  //   %FalseValue = ...
1281  //   # fallthrough to sinkMBB
1282  BB = copy0MBB;
1283
1284  // Update machine-CFG edges
1285  BB->addSuccessor(sinkMBB);
1286
1287  //  sinkMBB:
1288  //   %Result = phi [ %FalseValue, copy0MBB ], [ %TrueValue, thisMBB ]
1289  //  ...
1290  BB = sinkMBB;
1291  BuildMI(BB, X86::PHI, 4, MI->getOperand(0).getReg())
1292    .addReg(MI->getOperand(1).getReg()).addMBB(copy0MBB)
1293    .addReg(MI->getOperand(2).getReg()).addMBB(thisMBB);
1294
1295  delete MI;   // The pseudo instruction is gone now.
1296  return BB;
1297}
1298
1299
1300//===----------------------------------------------------------------------===//
1301//                           X86 Custom Lowering Hooks
1302//===----------------------------------------------------------------------===//
1303
1304/// LowerOperation - Provide custom lowering hooks for some operations.
1305///
1306SDOperand X86TargetLowering::LowerOperation(SDOperand Op, SelectionDAG &DAG) {
1307  switch (Op.getOpcode()) {
1308  default: assert(0 && "Should not custom lower this!");
1309  case ISD::ADD_PARTS:
1310  case ISD::SUB_PARTS: {
1311    assert(Op.getNumOperands() == 4 && Op.getValueType() == MVT::i32 &&
1312           "Not an i64 add/sub!");
1313    bool isAdd = Op.getOpcode() == ISD::ADD_PARTS;
1314    std::vector<MVT::ValueType> Tys;
1315    Tys.push_back(MVT::i32);
1316    Tys.push_back(MVT::Flag);
1317    std::vector<SDOperand> Ops;
1318    Ops.push_back(Op.getOperand(0));
1319    Ops.push_back(Op.getOperand(2));
1320    SDOperand Lo = DAG.getNode(isAdd ? X86ISD::ADD_FLAG : X86ISD::SUB_FLAG,
1321                               Tys, Ops);
1322    SDOperand Hi = DAG.getNode(isAdd ? X86ISD::ADC : X86ISD::SBB, MVT::i32,
1323                               Op.getOperand(1), Op.getOperand(3),
1324                               Lo.getValue(1));
1325    Tys.clear();
1326    Tys.push_back(MVT::i32);
1327    Tys.push_back(MVT::i32);
1328    Ops.clear();
1329    Ops.push_back(Lo);
1330    Ops.push_back(Hi);
1331    return DAG.getNode(ISD::MERGE_VALUES, Tys, Ops);
1332  }
1333  case ISD::SHL_PARTS:
1334  case ISD::SRA_PARTS:
1335  case ISD::SRL_PARTS: {
1336    assert(Op.getNumOperands() == 3 && Op.getValueType() == MVT::i32 &&
1337           "Not an i64 shift!");
1338    bool isSRA = Op.getOpcode() == ISD::SRA_PARTS;
1339    SDOperand ShOpLo = Op.getOperand(0);
1340    SDOperand ShOpHi = Op.getOperand(1);
1341    SDOperand ShAmt  = Op.getOperand(2);
1342    SDOperand Tmp1 = isSRA ? DAG.getNode(ISD::SRA, MVT::i32, ShOpHi,
1343                                         DAG.getConstant(31, MVT::i32))
1344                           : DAG.getConstant(0, MVT::i32);
1345
1346    SDOperand Tmp2, Tmp3;
1347    if (Op.getOpcode() == ISD::SHL_PARTS) {
1348      Tmp2 = DAG.getNode(X86ISD::SHLD, MVT::i32, ShOpHi, ShOpLo, ShAmt);
1349      Tmp3 = DAG.getNode(ISD::SHL, MVT::i32, ShOpLo, ShAmt);
1350    } else {
1351      Tmp2 = DAG.getNode(X86ISD::SHRD, MVT::i32, ShOpLo, ShOpHi, ShAmt);
1352      Tmp3 = DAG.getNode(isSRA ? ISD::SRA : ISD::SHL, MVT::i32, ShOpHi, ShAmt);
1353    }
1354
1355    SDOperand InFlag = DAG.getNode(X86ISD::TEST, MVT::Flag,
1356                                   ShAmt, DAG.getConstant(32, MVT::i8));
1357
1358    SDOperand Hi, Lo;
1359    SDOperand CC = DAG.getConstant(X86ISD::COND_NE, MVT::i8);
1360
1361    std::vector<MVT::ValueType> Tys;
1362    Tys.push_back(MVT::i32);
1363    Tys.push_back(MVT::Flag);
1364    std::vector<SDOperand> Ops;
1365    if (Op.getOpcode() == ISD::SHL_PARTS) {
1366      Ops.push_back(Tmp2);
1367      Ops.push_back(Tmp3);
1368      Ops.push_back(CC);
1369      Ops.push_back(InFlag);
1370      Hi = DAG.getNode(X86ISD::CMOV, Tys, Ops);
1371      InFlag = Hi.getValue(1);
1372
1373      Ops.clear();
1374      Ops.push_back(Tmp3);
1375      Ops.push_back(Tmp1);
1376      Ops.push_back(CC);
1377      Ops.push_back(InFlag);
1378      Lo = DAG.getNode(X86ISD::CMOV, Tys, Ops);
1379    } else {
1380      Ops.push_back(Tmp2);
1381      Ops.push_back(Tmp3);
1382      Ops.push_back(CC);
1383      Ops.push_back(InFlag);
1384      Lo = DAG.getNode(X86ISD::CMOV, Tys, Ops);
1385      InFlag = Lo.getValue(1);
1386
1387      Ops.clear();
1388      Ops.push_back(Tmp3);
1389      Ops.push_back(Tmp1);
1390      Ops.push_back(CC);
1391      Ops.push_back(InFlag);
1392      Hi = DAG.getNode(X86ISD::CMOV, Tys, Ops);
1393    }
1394
1395    Tys.clear();
1396    Tys.push_back(MVT::i32);
1397    Tys.push_back(MVT::i32);
1398    Ops.clear();
1399    Ops.push_back(Lo);
1400    Ops.push_back(Hi);
1401    return DAG.getNode(ISD::MERGE_VALUES, Tys, Ops);
1402  }
1403  case ISD::SINT_TO_FP: {
1404    assert(Op.getValueType() == MVT::f64 &&
1405           Op.getOperand(0).getValueType() == MVT::i64 &&
1406           "Unknown SINT_TO_FP to lower!");
1407    // We lower sint64->FP into a store to a temporary stack slot, followed by a
1408    // FILD64m node.
1409    MachineFunction &MF = DAG.getMachineFunction();
1410    int SSFI = MF.getFrameInfo()->CreateStackObject(8, 8);
1411    SDOperand StackSlot = DAG.getFrameIndex(SSFI, getPointerTy());
1412    SDOperand Store = DAG.getNode(ISD::STORE, MVT::Other, DAG.getEntryNode(),
1413                           Op.getOperand(0), StackSlot, DAG.getSrcValue(NULL));
1414    std::vector<MVT::ValueType> RTs;
1415    RTs.push_back(MVT::f64);
1416    RTs.push_back(MVT::Other);
1417    std::vector<SDOperand> Ops;
1418    Ops.push_back(Store);
1419    Ops.push_back(StackSlot);
1420    return DAG.getNode(X86ISD::FILD64m, RTs, Ops);
1421  }
1422  case ISD::FP_TO_SINT: {
1423    assert(Op.getValueType() <= MVT::i64 && Op.getValueType() >= MVT::i16 &&
1424           Op.getOperand(0).getValueType() == MVT::f64 &&
1425           "Unknown FP_TO_SINT to lower!");
1426    // We lower FP->sint64 into FISTP64, followed by a load, all to a temporary
1427    // stack slot.
1428    MachineFunction &MF = DAG.getMachineFunction();
1429    unsigned MemSize = MVT::getSizeInBits(Op.getValueType())/8;
1430    int SSFI = MF.getFrameInfo()->CreateStackObject(MemSize, MemSize);
1431    SDOperand StackSlot = DAG.getFrameIndex(SSFI, getPointerTy());
1432
1433    unsigned Opc;
1434    switch (Op.getValueType()) {
1435    default: assert(0 && "Invalid FP_TO_SINT to lower!");
1436    case MVT::i16: Opc = X86ISD::FP_TO_INT16_IN_MEM; break;
1437    case MVT::i32: Opc = X86ISD::FP_TO_INT32_IN_MEM; break;
1438    case MVT::i64: Opc = X86ISD::FP_TO_INT64_IN_MEM; break;
1439    }
1440
1441    // Build the FP_TO_INT*_IN_MEM
1442    std::vector<SDOperand> Ops;
1443    Ops.push_back(DAG.getEntryNode());
1444    Ops.push_back(Op.getOperand(0));
1445    Ops.push_back(StackSlot);
1446    SDOperand FIST = DAG.getNode(Opc, MVT::Other, Ops);
1447
1448    // Load the result.
1449    return DAG.getLoad(Op.getValueType(), FIST, StackSlot,
1450                       DAG.getSrcValue(NULL));
1451  }
1452  case ISD::READCYCLECOUNTER: {
1453    std::vector<MVT::ValueType> Tys;
1454    Tys.push_back(MVT::Other);
1455    Tys.push_back(MVT::Flag);
1456    std::vector<SDOperand> Ops;
1457    Ops.push_back(Op.getOperand(0));
1458    SDOperand rd = DAG.getNode(X86ISD::RDTSC_DAG, Tys, Ops);
1459    Ops.clear();
1460    Ops.push_back(DAG.getCopyFromReg(rd, X86::EAX, MVT::i32, rd.getValue(1)));
1461    Ops.push_back(DAG.getCopyFromReg(Ops[0].getValue(1), X86::EDX,
1462                                     MVT::i32, Ops[0].getValue(2)));
1463    Ops.push_back(Ops[1].getValue(1));
1464    Tys[0] = Tys[1] = MVT::i32;
1465    Tys.push_back(MVT::Other);
1466    return DAG.getNode(ISD::MERGE_VALUES, Tys, Ops);
1467  }
1468  case ISD::SETCC: {
1469    assert(Op.getValueType() == MVT::i8 && "SetCC type must be 8-bit integer");
1470    SDOperand CC   = Op.getOperand(2);
1471    SDOperand Cond = DAG.getNode(X86ISD::CMP, MVT::Flag,
1472                                 Op.getOperand(0), Op.getOperand(1));
1473    ISD::CondCode SetCCOpcode = cast<CondCodeSDNode>(CC)->get();
1474    bool isFP = MVT::isFloatingPoint(Op.getOperand(1).getValueType());
1475    unsigned X86CC = getX86CC(CC, isFP);
1476    if (X86CC != X86ISD::COND_INVALID) {
1477      return DAG.getNode(X86ISD::SETCC, MVT::i8,
1478                         DAG.getConstant(X86CC, MVT::i8), Cond);
1479    } else {
1480      assert(isFP && "Illegal integer SetCC!");
1481
1482      std::vector<MVT::ValueType> Tys;
1483      std::vector<SDOperand> Ops;
1484      switch (SetCCOpcode) {
1485      default: assert(false && "Illegal floating point SetCC!");
1486      case ISD::SETOEQ: {  // !PF & ZF
1487        Tys.push_back(MVT::i8);
1488        Tys.push_back(MVT::Flag);
1489        Ops.push_back(DAG.getConstant(X86ISD::COND_NP, MVT::i8));
1490        Ops.push_back(Cond);
1491        SDOperand Tmp1 = DAG.getNode(X86ISD::SETCC, Tys, Ops);
1492        SDOperand Tmp2 = DAG.getNode(X86ISD::SETCC, MVT::i8,
1493                                     DAG.getConstant(X86ISD::COND_E, MVT::i8),
1494                                     Tmp1.getValue(1));
1495        return DAG.getNode(ISD::AND, MVT::i8, Tmp1, Tmp2);
1496      }
1497      case ISD::SETOLT: {  // !PF & CF
1498        Tys.push_back(MVT::i8);
1499        Tys.push_back(MVT::Flag);
1500        Ops.push_back(DAG.getConstant(X86ISD::COND_NP, MVT::i8));
1501        Ops.push_back(Cond);
1502        SDOperand Tmp1 = DAG.getNode(X86ISD::SETCC, Tys, Ops);
1503        SDOperand Tmp2 = DAG.getNode(X86ISD::SETCC, MVT::i8,
1504                                     DAG.getConstant(X86ISD::COND_B, MVT::i8),
1505                                     Tmp1.getValue(1));
1506        return DAG.getNode(ISD::AND, MVT::i8, Tmp1, Tmp2);
1507      }
1508      case ISD::SETOLE: {  // !PF & (CF || ZF)
1509        Tys.push_back(MVT::i8);
1510        Tys.push_back(MVT::Flag);
1511        Ops.push_back(DAG.getConstant(X86ISD::COND_NP, MVT::i8));
1512        Ops.push_back(Cond);
1513        SDOperand Tmp1 = DAG.getNode(X86ISD::SETCC, Tys, Ops);
1514        SDOperand Tmp2 = DAG.getNode(X86ISD::SETCC, MVT::i8,
1515                                     DAG.getConstant(X86ISD::COND_BE, MVT::i8),
1516                                     Tmp1.getValue(1));
1517        return DAG.getNode(ISD::AND, MVT::i8, Tmp1, Tmp2);
1518      }
1519      case ISD::SETUGT: {  // PF | (!ZF & !CF)
1520        Tys.push_back(MVT::i8);
1521        Tys.push_back(MVT::Flag);
1522        Ops.push_back(DAG.getConstant(X86ISD::COND_P, MVT::i8));
1523        Ops.push_back(Cond);
1524        SDOperand Tmp1 = DAG.getNode(X86ISD::SETCC, Tys, Ops);
1525        SDOperand Tmp2 = DAG.getNode(X86ISD::SETCC, MVT::i8,
1526                                     DAG.getConstant(X86ISD::COND_A, MVT::i8),
1527                                     Tmp1.getValue(1));
1528        return DAG.getNode(ISD::OR, MVT::i8, Tmp1, Tmp2);
1529      }
1530      case ISD::SETUGE: {  // PF | !CF
1531        Tys.push_back(MVT::i8);
1532        Tys.push_back(MVT::Flag);
1533        Ops.push_back(DAG.getConstant(X86ISD::COND_P, MVT::i8));
1534        Ops.push_back(Cond);
1535        SDOperand Tmp1 = DAG.getNode(X86ISD::SETCC, Tys, Ops);
1536        SDOperand Tmp2 = DAG.getNode(X86ISD::SETCC, MVT::i8,
1537                                     DAG.getConstant(X86ISD::COND_AE, MVT::i8),
1538                                     Tmp1.getValue(1));
1539        return DAG.getNode(ISD::OR, MVT::i8, Tmp1, Tmp2);
1540      }
1541      case ISD::SETUNE: {  // PF | !ZF
1542        Tys.push_back(MVT::i8);
1543        Tys.push_back(MVT::Flag);
1544        Ops.push_back(DAG.getConstant(X86ISD::COND_P, MVT::i8));
1545        Ops.push_back(Cond);
1546        SDOperand Tmp1 = DAG.getNode(X86ISD::SETCC, Tys, Ops);
1547        SDOperand Tmp2 = DAG.getNode(X86ISD::SETCC, MVT::i8,
1548                                     DAG.getConstant(X86ISD::COND_NE, MVT::i8),
1549                                     Tmp1.getValue(1));
1550        return DAG.getNode(ISD::OR, MVT::i8, Tmp1, Tmp2);
1551      }
1552      }
1553    }
1554  }
1555  case ISD::SELECT: {
1556    MVT::ValueType VT = Op.getValueType();
1557    bool isFP      = MVT::isFloatingPoint(VT);
1558    bool isFPStack = isFP && (X86Vector < SSE2);
1559    bool isFPSSE   = isFP && (X86Vector >= SSE2);
1560    bool isValid   = false;
1561    SDOperand Op0 = Op.getOperand(0);
1562    SDOperand Cond, CC;
1563    if (Op0.getOpcode() == X86ISD::SETCC) {
1564      CC   = Op0.getOperand(0);
1565      Cond = Op0.getOperand(1);
1566      isValid =
1567        !(isFPStack && !hasFPCMov(cast<ConstantSDNode>(CC)->getSignExtended()));
1568    } else if (Op0.getOpcode() == ISD::SETCC) {
1569      CC = Op0.getOperand(2);
1570      bool isFP = MVT::isFloatingPoint(Op0.getOperand(1).getValueType());
1571      unsigned X86CC = getX86CC(CC, isFP);
1572      CC = DAG.getConstant(X86CC, MVT::i8);
1573      Cond = DAG.getNode(X86ISD::CMP, MVT::Flag,
1574                         Op0.getOperand(0), Op0.getOperand(1));
1575      isValid = true;
1576    }
1577
1578    if (!isValid) {
1579      CC = DAG.getConstant(X86ISD::COND_E, MVT::i8);
1580      Cond = DAG.getNode(X86ISD::TEST, MVT::Flag, Op0, Op0);
1581    }
1582
1583    std::vector<MVT::ValueType> Tys;
1584    Tys.push_back(Op.getValueType());
1585    Tys.push_back(MVT::Flag);
1586    std::vector<SDOperand> Ops;
1587    Ops.push_back(Op.getOperand(1));
1588    Ops.push_back(Op.getOperand(2));
1589    Ops.push_back(CC);
1590    Ops.push_back(Cond);
1591    return DAG.getNode(X86ISD::CMOV, Tys, Ops);
1592  }
1593  case ISD::BRCOND: {
1594    SDOperand Cond  = Op.getOperand(1);
1595    SDOperand Dest  = Op.getOperand(2);
1596    SDOperand CC;
1597    // TODO: handle Cond == OR / AND / XOR
1598    if (Cond.getOpcode() == X86ISD::SETCC) {
1599      CC = Cond.getOperand(0);
1600      Cond = Cond.getOperand(1);
1601    } else if (Cond.getOpcode() == ISD::SETCC) {
1602      CC = Cond.getOperand(2);
1603      bool isFP = MVT::isFloatingPoint(Cond.getOperand(1).getValueType());
1604      unsigned X86CC = getX86CC(CC, isFP);
1605      CC = DAG.getConstant(X86CC, MVT::i8);
1606      Cond = DAG.getNode(X86ISD::CMP, MVT::Flag,
1607                         Cond.getOperand(0), Cond.getOperand(1));
1608    } else {
1609      CC = DAG.getConstant(X86ISD::COND_NE, MVT::i8);
1610      Cond = DAG.getNode(X86ISD::TEST, MVT::Flag, Cond, Cond);
1611    }
1612    return DAG.getNode(X86ISD::BRCOND, Op.getValueType(),
1613                       Op.getOperand(0), Op.getOperand(2), CC, Cond);
1614  }
1615  case ISD::RET: {
1616    // Can only be return void.
1617    return DAG.getNode(X86ISD::RET_FLAG, MVT::Other, Op.getOperand(0),
1618                       DAG.getConstant(getBytesToPopOnReturn(), MVT::i16));
1619  }
1620  case ISD::MEMSET: {
1621    SDOperand InFlag;
1622    SDOperand Chain = Op.getOperand(0);
1623    unsigned Align =
1624      (unsigned)cast<ConstantSDNode>(Op.getOperand(4))->getValue();
1625    if (Align == 0) Align = 1;
1626
1627    MVT::ValueType AVT;
1628    SDOperand Count;
1629    if (ConstantSDNode *ValC = dyn_cast<ConstantSDNode>(Op.getOperand(2))) {
1630      unsigned ValReg;
1631      unsigned Val = ValC->getValue() & 255;
1632
1633      // If the value is a constant, then we can potentially use larger sets.
1634      switch (Align & 3) {
1635      case 2:   // WORD aligned
1636        AVT = MVT::i16;
1637        if (ConstantSDNode *I = dyn_cast<ConstantSDNode>(Op.getOperand(3)))
1638          Count = DAG.getConstant(I->getValue() / 2, MVT::i32);
1639        else
1640          Count = DAG.getNode(ISD::SRL, MVT::i32, Op.getOperand(3),
1641                              DAG.getConstant(1, MVT::i8));
1642        Val    = (Val << 8) | Val;
1643        ValReg = X86::AX;
1644        break;
1645      case 0:   // DWORD aligned
1646        AVT = MVT::i32;
1647        if (ConstantSDNode *I = dyn_cast<ConstantSDNode>(Op.getOperand(3)))
1648          Count = DAG.getConstant(I->getValue() / 4, MVT::i32);
1649        else
1650          Count = DAG.getNode(ISD::SRL, MVT::i32, Op.getOperand(3),
1651                              DAG.getConstant(2, MVT::i8));
1652        Val = (Val << 8)  | Val;
1653        Val = (Val << 16) | Val;
1654        ValReg = X86::EAX;
1655        break;
1656      default:  // Byte aligned
1657        AVT = MVT::i8;
1658        Count = Op.getOperand(3);
1659        ValReg = X86::AL;
1660        break;
1661      }
1662
1663      Chain  = DAG.getCopyToReg(Chain, ValReg, DAG.getConstant(Val, AVT),
1664                                InFlag);
1665      InFlag = Chain.getValue(1);
1666    } else {
1667      AVT    = MVT::i8;
1668      Count  = Op.getOperand(3);
1669      Chain  = DAG.getCopyToReg(Chain, X86::AL, Op.getOperand(2), InFlag);
1670      InFlag = Chain.getValue(1);
1671    }
1672
1673    Chain  = DAG.getCopyToReg(Chain, X86::ECX, Count, InFlag);
1674    InFlag = Chain.getValue(1);
1675    Chain  = DAG.getCopyToReg(Chain, X86::EDI, Op.getOperand(1), InFlag);
1676    InFlag = Chain.getValue(1);
1677
1678    return DAG.getNode(X86ISD::REP_STOS, MVT::Other, Chain,
1679                       DAG.getValueType(AVT), InFlag);
1680  }
1681  case ISD::MEMCPY: {
1682    SDOperand Chain = Op.getOperand(0);
1683    unsigned Align =
1684      (unsigned)cast<ConstantSDNode>(Op.getOperand(4))->getValue();
1685    if (Align == 0) Align = 1;
1686
1687    MVT::ValueType AVT;
1688    SDOperand Count;
1689    switch (Align & 3) {
1690    case 2:   // WORD aligned
1691      AVT = MVT::i16;
1692      if (ConstantSDNode *I = dyn_cast<ConstantSDNode>(Op.getOperand(3)))
1693        Count = DAG.getConstant(I->getValue() / 2, MVT::i32);
1694      else
1695        Count = DAG.getNode(ISD::SRL, MVT::i32, Op.getOperand(3),
1696                            DAG.getConstant(1, MVT::i8));
1697      break;
1698    case 0:   // DWORD aligned
1699      AVT = MVT::i32;
1700      if (ConstantSDNode *I = dyn_cast<ConstantSDNode>(Op.getOperand(3)))
1701        Count = DAG.getConstant(I->getValue() / 4, MVT::i32);
1702      else
1703        Count = DAG.getNode(ISD::SRL, MVT::i32, Op.getOperand(3),
1704                            DAG.getConstant(2, MVT::i8));
1705      break;
1706    default:  // Byte aligned
1707      AVT = MVT::i8;
1708      Count = Op.getOperand(3);
1709      break;
1710    }
1711
1712    SDOperand InFlag;
1713    Chain  = DAG.getCopyToReg(Chain, X86::ECX, Count, InFlag);
1714    InFlag = Chain.getValue(1);
1715    Chain  = DAG.getCopyToReg(Chain, X86::EDI, Op.getOperand(1), InFlag);
1716    InFlag = Chain.getValue(1);
1717    Chain  = DAG.getCopyToReg(Chain, X86::ESI, Op.getOperand(2), InFlag);
1718    InFlag = Chain.getValue(1);
1719
1720    return DAG.getNode(X86ISD::REP_MOVS, MVT::Other, Chain,
1721                       DAG.getValueType(AVT), InFlag);
1722  }
1723  case ISD::GlobalAddress: {
1724    GlobalValue *GV = cast<GlobalAddressSDNode>(Op)->getGlobal();
1725    SDOperand GVOp = DAG.getTargetGlobalAddress(GV, getPointerTy());
1726    // For Darwin, external and weak symbols are indirect, so we want to load
1727    // the value at address GV, not the value of GV itself.  This means that
1728    // the GlobalAddress must be in the base or index register of the address,
1729    // not the GV offset field.
1730    if (getTargetMachine().
1731        getSubtarget<X86Subtarget>().getIndirectExternAndWeakGlobals() &&
1732        (GV->hasWeakLinkage() || GV->isExternal()))
1733      return DAG.getLoad(MVT::i32, DAG.getEntryNode(),
1734                         GVOp, DAG.getSrcValue(NULL));
1735    else
1736      return GVOp;
1737    break;
1738  }
1739  }
1740}
1741
1742const char *X86TargetLowering::getTargetNodeName(unsigned Opcode) const {
1743  switch (Opcode) {
1744  default: return NULL;
1745  case X86ISD::ADD_FLAG:           return "X86ISD::ADD_FLAG";
1746  case X86ISD::SUB_FLAG:           return "X86ISD::SUB_FLAG";
1747  case X86ISD::ADC:                return "X86ISD::ADC";
1748  case X86ISD::SBB:                return "X86ISD::SBB";
1749  case X86ISD::SHLD:               return "X86ISD::SHLD";
1750  case X86ISD::SHRD:               return "X86ISD::SHRD";
1751  case X86ISD::FILD64m:            return "X86ISD::FILD64m";
1752  case X86ISD::FP_TO_INT16_IN_MEM: return "X86ISD::FP_TO_INT16_IN_MEM";
1753  case X86ISD::FP_TO_INT32_IN_MEM: return "X86ISD::FP_TO_INT32_IN_MEM";
1754  case X86ISD::FP_TO_INT64_IN_MEM: return "X86ISD::FP_TO_INT64_IN_MEM";
1755  case X86ISD::FLD:                return "X86ISD::FLD";
1756  case X86ISD::FST:                return "X86ISD::FST";
1757  case X86ISD::FP_GET_RESULT:      return "X86ISD::FP_GET_RESULT";
1758  case X86ISD::FP_SET_RESULT:      return "X86ISD::FP_SET_RESULT";
1759  case X86ISD::CALL:               return "X86ISD::CALL";
1760  case X86ISD::TAILCALL:           return "X86ISD::TAILCALL";
1761  case X86ISD::RDTSC_DAG:          return "X86ISD::RDTSC_DAG";
1762  case X86ISD::CMP:                return "X86ISD::CMP";
1763  case X86ISD::TEST:               return "X86ISD::TEST";
1764  case X86ISD::SETCC:              return "X86ISD::SETCC";
1765  case X86ISD::CMOV:               return "X86ISD::CMOV";
1766  case X86ISD::BRCOND:             return "X86ISD::BRCOND";
1767  case X86ISD::RET_FLAG:           return "X86ISD::RET_FLAG";
1768  case X86ISD::REP_STOS:           return "X86ISD::RET_STOS";
1769  case X86ISD::REP_MOVS:           return "X86ISD::RET_MOVS";
1770  }
1771}
1772
1773bool X86TargetLowering::isMaskedValueZeroForTargetNode(const SDOperand &Op,
1774                                                       uint64_t Mask) const {
1775
1776  unsigned Opc = Op.getOpcode();
1777
1778  switch (Opc) {
1779  default:
1780    assert(Opc >= ISD::BUILTIN_OP_END && "Expected a target specific node");
1781    break;
1782  case X86ISD::SETCC: return (Mask & 1) == 0;
1783  }
1784
1785  return false;
1786}
1787