X86ISelLowering.cpp revision e90da97f3ef0dab5c97976ce227a4407918bf5a7 
1//===-- X86ISelLowering.h - X86 DAG Lowering Interface ----------*- C++ -*-===//
2//
3//                     The LLVM Compiler Infrastructure
4//
5// This file was developed by Chris Lattner and is distributed under
6// the University of Illinois Open Source License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10// This file defines the interfaces that X86 uses to lower LLVM code into a
11// selection DAG.
12//
13//===----------------------------------------------------------------------===//
14
15#include "X86.h"
16#include "X86ISelLowering.h"
17#include "X86TargetMachine.h"
18#include "llvm/CallingConv.h"
19#include "llvm/Function.h"
20#include "llvm/CodeGen/MachineFrameInfo.h"
21#include "llvm/CodeGen/MachineFunction.h"
22#include "llvm/CodeGen/MachineInstrBuilder.h"
23#include "llvm/CodeGen/SelectionDAG.h"
24#include "llvm/CodeGen/SSARegMap.h"
25#include "llvm/Target/TargetOptions.h"
26using namespace llvm;
27
28// FIXME: temporary.
29#include "llvm/Support/CommandLine.h"
30static cl::opt<bool> EnableFastCC("enable-x86-fastcc", cl::Hidden,
31                                  cl::desc("Enable fastcc on X86"));
32
33X86TargetLowering::X86TargetLowering(TargetMachine &TM)
34  : TargetLowering(TM) {
35  // Set up the TargetLowering object.
36
37  // X86 is weird, it always uses i8 for shift amounts and setcc results.
38  setShiftAmountType(MVT::i8);
39  setSetCCResultType(MVT::i8);
40  setSetCCResultContents(ZeroOrOneSetCCResult);
41  setShiftAmountFlavor(Mask);   // shl X, 32 == shl X, 0
42  setStackPointerRegisterToSaveRestore(X86::ESP);
43
44  // Set up the register classes.
45  addRegisterClass(MVT::i8, X86::R8RegisterClass);
46  addRegisterClass(MVT::i16, X86::R16RegisterClass);
47  addRegisterClass(MVT::i32, X86::R32RegisterClass);
48
49  // Promote all UINT_TO_FP to larger SINT_TO_FP's, as X86 doesn't have this
50  // operation.
51  setOperationAction(ISD::UINT_TO_FP       , MVT::i1   , Promote);
52  setOperationAction(ISD::UINT_TO_FP       , MVT::i8   , Promote);
53  setOperationAction(ISD::UINT_TO_FP       , MVT::i16  , Promote);
54  setOperationAction(ISD::UINT_TO_FP       , MVT::i32  , Promote);
55
56  // Promote i1/i8 SINT_TO_FP to larger SINT_TO_FP's, as X86 doesn't have
57  // this operation.
58  setOperationAction(ISD::SINT_TO_FP       , MVT::i1   , Promote);
59  setOperationAction(ISD::SINT_TO_FP       , MVT::i8   , Promote);
60
61  if (!X86ScalarSSE) {
62    // We can handle SINT_TO_FP and FP_TO_SINT from/TO i64 even though i64
63    // isn't legal.
64    setOperationAction(ISD::SINT_TO_FP     , MVT::i64  , Custom);
65    setOperationAction(ISD::FP_TO_SINT     , MVT::i64  , Custom);
66    setOperationAction(ISD::FP_TO_SINT     , MVT::i32  , Custom);
67    setOperationAction(ISD::FP_TO_SINT     , MVT::i16  , Custom);
68  }
69
70  // Handle FP_TO_UINT by promoting the destination to a larger signed
71  // conversion.
72  setOperationAction(ISD::FP_TO_UINT       , MVT::i1   , Promote);
73  setOperationAction(ISD::FP_TO_UINT       , MVT::i8   , Promote);
74  setOperationAction(ISD::FP_TO_UINT       , MVT::i16  , Promote);
75
76  if (!X86ScalarSSE)
77    setOperationAction(ISD::FP_TO_UINT     , MVT::i32  , Promote);
78
79  // Promote i1/i8 FP_TO_SINT to larger FP_TO_SINTS's, as X86 doesn't have
80  // this operation.
81  setOperationAction(ISD::FP_TO_SINT       , MVT::i1   , Promote);
82  setOperationAction(ISD::FP_TO_SINT       , MVT::i8   , Promote);
83  setOperationAction(ISD::FP_TO_SINT       , MVT::i16  , Promote);
84
85  setOperationAction(ISD::BIT_CONVERT, MVT::f32, Expand);
86  setOperationAction(ISD::BIT_CONVERT, MVT::i32, Expand);
87
88  if (X86DAGIsel) {
89    setOperationAction(ISD::BRCOND         , MVT::Other, Custom);
90  }
91  setOperationAction(ISD::BRCONDTWOWAY     , MVT::Other, Expand);
92  setOperationAction(ISD::BRTWOWAY_CC      , MVT::Other, Expand);
93  setOperationAction(ISD::MEMMOVE          , MVT::Other, Expand);
94  setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i16  , Expand);
95  setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i8   , Expand);
96  setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i1   , Expand);
97  setOperationAction(ISD::FP_ROUND_INREG   , MVT::f32  , Expand);
98  setOperationAction(ISD::SEXTLOAD         , MVT::i1   , Expand);
99  setOperationAction(ISD::FREM             , MVT::f64  , Expand);
100  setOperationAction(ISD::CTPOP            , MVT::i8   , Expand);
101  setOperationAction(ISD::CTTZ             , MVT::i8   , Expand);
102  setOperationAction(ISD::CTLZ             , MVT::i8   , Expand);
103  setOperationAction(ISD::CTPOP            , MVT::i16  , Expand);
104  setOperationAction(ISD::CTTZ             , MVT::i16  , Expand);
105  setOperationAction(ISD::CTLZ             , MVT::i16  , Expand);
106  setOperationAction(ISD::CTPOP            , MVT::i32  , Expand);
107  setOperationAction(ISD::CTTZ             , MVT::i32  , Expand);
108  setOperationAction(ISD::CTLZ             , MVT::i32  , Expand);
109  setOperationAction(ISD::READCYCLECOUNTER , MVT::i64  , Custom);
110
111  if (!X86DAGIsel) {
112    setOperationAction(ISD::ROTL           , MVT::i8   , Expand);
113    setOperationAction(ISD::ROTR           , MVT::i8   , Expand);
114    setOperationAction(ISD::ROTL           , MVT::i16  , Expand);
115    setOperationAction(ISD::ROTR           , MVT::i16  , Expand);
116    setOperationAction(ISD::ROTL           , MVT::i32  , Expand);
117    setOperationAction(ISD::ROTR           , MVT::i32  , Expand);
118  }
119
120  setOperationAction(ISD::READIO           , MVT::i1   , Expand);
121  setOperationAction(ISD::READIO           , MVT::i8   , Expand);
122  setOperationAction(ISD::READIO           , MVT::i16  , Expand);
123  setOperationAction(ISD::READIO           , MVT::i32  , Expand);
124  setOperationAction(ISD::WRITEIO          , MVT::i1   , Expand);
125  setOperationAction(ISD::WRITEIO          , MVT::i8   , Expand);
126  setOperationAction(ISD::WRITEIO          , MVT::i16  , Expand);
127  setOperationAction(ISD::WRITEIO          , MVT::i32  , Expand);
128
129  // These should be promoted to a larger select which is supported.
130  setOperationAction(ISD::SELECT           , MVT::i1   , Promote);
131  setOperationAction(ISD::SELECT           , MVT::i8   , Promote);
132  if (X86DAGIsel) {
133    // X86 wants to expand cmov itself.
134    setOperationAction(ISD::SELECT         , MVT::i16  , Custom);
135    setOperationAction(ISD::SELECT         , MVT::i32  , Custom);
136    setOperationAction(ISD::SELECT         , MVT::f32  , Custom);
137    setOperationAction(ISD::SELECT         , MVT::f64  , Custom);
138    setOperationAction(ISD::SETCC          , MVT::i8   , Custom);
139    setOperationAction(ISD::SETCC          , MVT::i16  , Custom);
140    setOperationAction(ISD::SETCC          , MVT::i32  , Custom);
141    setOperationAction(ISD::SETCC          , MVT::f32  , Custom);
142    setOperationAction(ISD::SETCC          , MVT::f64  , Custom);
143    // X86 ret instruction may pop stack.
144    setOperationAction(ISD::RET            , MVT::Other, Custom);
145    // Darwin ABI issue.
146    setOperationAction(ISD::GlobalAddress  , MVT::i32  , Custom);
147    // 64-bit addm sub, shl, sra, srl (iff 32-bit x86)
148    setOperationAction(ISD::ADD_PARTS      , MVT::i32  , Custom);
149    setOperationAction(ISD::SUB_PARTS      , MVT::i32  , Custom);
150    setOperationAction(ISD::SHL_PARTS      , MVT::i32  , Custom);
151    setOperationAction(ISD::SRA_PARTS      , MVT::i32  , Custom);
152    setOperationAction(ISD::SRL_PARTS      , MVT::i32  , Custom);
153    // X86 wants to expand memset / memcpy itself.
154    setOperationAction(ISD::MEMSET         , MVT::Other, Custom);
155    setOperationAction(ISD::MEMCPY         , MVT::Other, Custom);
156  }
157
158  // We don't have line number support yet.
159  setOperationAction(ISD::LOCATION, MVT::Other, Expand);
160  setOperationAction(ISD::DEBUG_LOC, MVT::Other, Expand);
161  setOperationAction(ISD::DEBUG_LABEL, MVT::Other, Expand);
162
163  // Not implemented yet.
164  setOperationAction(ISD::STACKSAVE, MVT::Other, Expand);
165  setOperationAction(ISD::STACKRESTORE, MVT::Other, Expand);
166
167  if (X86ScalarSSE) {
168    // Set up the FP register classes.
169    addRegisterClass(MVT::f32, X86::FR32RegisterClass);
170    addRegisterClass(MVT::f64, X86::FR64RegisterClass);
171
172    // SSE has no load+extend ops
173    setOperationAction(ISD::EXTLOAD,  MVT::f32, Expand);
174    setOperationAction(ISD::ZEXTLOAD, MVT::f32, Expand);
175
176    // SSE has no i16 to fp conversion, only i32
177    setOperationAction(ISD::SINT_TO_FP, MVT::i16, Promote);
178    setOperationAction(ISD::FP_TO_SINT, MVT::i16, Promote);
179
180    // Expand FP_TO_UINT into a select.
181    // FIXME: We would like to use a Custom expander here eventually to do
182    // the optimal thing for SSE vs. the default expansion in the legalizer.
183    setOperationAction(ISD::FP_TO_UINT       , MVT::i32  , Expand);
184
185    // We don't support sin/cos/sqrt/fmod
186    setOperationAction(ISD::FSIN , MVT::f64, Expand);
187    setOperationAction(ISD::FCOS , MVT::f64, Expand);
188    setOperationAction(ISD::FABS , MVT::f64, Expand);
189    setOperationAction(ISD::FNEG , MVT::f64, Expand);
190    setOperationAction(ISD::FREM , MVT::f64, Expand);
191    setOperationAction(ISD::FSIN , MVT::f32, Expand);
192    setOperationAction(ISD::FCOS , MVT::f32, Expand);
193    setOperationAction(ISD::FABS , MVT::f32, Expand);
194    setOperationAction(ISD::FNEG , MVT::f32, Expand);
195    setOperationAction(ISD::FREM , MVT::f32, Expand);
196
197    addLegalFPImmediate(+0.0); // xorps / xorpd
198  } else {
199    // Set up the FP register classes.
200    addRegisterClass(MVT::f64, X86::RFPRegisterClass);
201
202    if (X86DAGIsel) {
203      setOperationAction(ISD::SINT_TO_FP, MVT::i16, Custom);
204      setOperationAction(ISD::SINT_TO_FP, MVT::i32, Custom);
205    }
206
207    if (!UnsafeFPMath) {
208      setOperationAction(ISD::FSIN           , MVT::f64  , Expand);
209      setOperationAction(ISD::FCOS           , MVT::f64  , Expand);
210    }
211
212    addLegalFPImmediate(+0.0); // FLD0
213    addLegalFPImmediate(+1.0); // FLD1
214    addLegalFPImmediate(-0.0); // FLD0/FCHS
215    addLegalFPImmediate(-1.0); // FLD1/FCHS
216  }
217  computeRegisterProperties();
218
219  maxStoresPerMemSet = 8; // For %llvm.memset -> sequence of stores
220  maxStoresPerMemCpy = 8; // For %llvm.memcpy -> sequence of stores
221  maxStoresPerMemMove = 8; // For %llvm.memmove -> sequence of stores
222  allowUnalignedMemoryAccesses = true; // x86 supports it!
223}
224
225std::vector<SDOperand>
226X86TargetLowering::LowerArguments(Function &F, SelectionDAG &DAG) {
227  if (F.getCallingConv() == CallingConv::Fast && EnableFastCC)
228    return LowerFastCCArguments(F, DAG);
229  return LowerCCCArguments(F, DAG);
230}
231
232std::pair<SDOperand, SDOperand>
233X86TargetLowering::LowerCallTo(SDOperand Chain, const Type *RetTy,
234                               bool isVarArg, unsigned CallingConv,
235                               bool isTailCall,
236                               SDOperand Callee, ArgListTy &Args,
237                               SelectionDAG &DAG) {
238  assert((!isVarArg || CallingConv == CallingConv::C) &&
239         "Only C takes varargs!");
240
241  // If the callee is a GlobalAddress node (quite common, every direct call is)
242  // turn it into a TargetGlobalAddress node so that legalize doesn't hack it.
243  if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee))
244    Callee = DAG.getTargetGlobalAddress(G->getGlobal(), getPointerTy());
245  else if (ExternalSymbolSDNode *S = dyn_cast<ExternalSymbolSDNode>(Callee))
246    Callee = DAG.getTargetExternalSymbol(S->getSymbol(), getPointerTy());
247
248  if (CallingConv == CallingConv::Fast && EnableFastCC)
249    return LowerFastCCCallTo(Chain, RetTy, isTailCall, Callee, Args, DAG);
250  return  LowerCCCCallTo(Chain, RetTy, isVarArg, isTailCall, Callee, Args, DAG);
251}
252
253SDOperand X86TargetLowering::LowerReturnTo(SDOperand Chain, SDOperand Op,
254                                           SelectionDAG &DAG) {
255  if (!X86DAGIsel)
256    return DAG.getNode(ISD::RET, MVT::Other, Chain, Op);
257
258  SDOperand Copy;
259  MVT::ValueType OpVT = Op.getValueType();
260  switch (OpVT) {
261    default: assert(0 && "Unknown type to return!");
262    case MVT::i32:
263      Copy = DAG.getCopyToReg(Chain, X86::EAX, Op, SDOperand());
264      break;
265    case MVT::i64: {
266      SDOperand Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, MVT::i32, Op,
267                                 DAG.getConstant(1, MVT::i32));
268      SDOperand Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, MVT::i32, Op,
269                                 DAG.getConstant(0, MVT::i32));
270      Copy = DAG.getCopyToReg(Chain, X86::EDX, Hi, SDOperand());
271      Copy = DAG.getCopyToReg(Copy,  X86::EAX, Lo, Copy.getValue(1));
272      break;
273    }
274    case MVT::f32:
275    case MVT::f64:
276      if (!X86ScalarSSE) {
277        if (OpVT == MVT::f32)
278          Op = DAG.getNode(ISD::FP_EXTEND, MVT::f64, Op);
279        std::vector<MVT::ValueType> Tys;
280        Tys.push_back(MVT::Other);
281        Tys.push_back(MVT::Flag);
282        std::vector<SDOperand> Ops;
283        Ops.push_back(Chain);
284        Ops.push_back(Op);
285        Copy = DAG.getNode(X86ISD::FP_SET_RESULT, Tys, Ops);
286      } else {
287        // Spill the value to memory and reload it into top of stack.
288        unsigned Size = MVT::getSizeInBits(OpVT)/8;
289        MachineFunction &MF = DAG.getMachineFunction();
290        int SSFI = MF.getFrameInfo()->CreateStackObject(Size, Size);
291        SDOperand StackSlot = DAG.getFrameIndex(SSFI, getPointerTy());
292        Chain = DAG.getNode(ISD::STORE, MVT::Other, Chain, Op,
293                            StackSlot, DAG.getSrcValue(NULL));
294        std::vector<MVT::ValueType> Tys;
295        Tys.push_back(MVT::f64);
296        Tys.push_back(MVT::Other);
297        std::vector<SDOperand> Ops;
298        Ops.push_back(Chain);
299        Ops.push_back(StackSlot);
300        Ops.push_back(DAG.getValueType(OpVT));
301        Copy = DAG.getNode(X86ISD::FLD, Tys, Ops);
302        Tys.clear();
303        Tys.push_back(MVT::Other);
304        Tys.push_back(MVT::Flag);
305        Ops.clear();
306        Ops.push_back(Copy.getValue(1));
307        Ops.push_back(Copy);
308        Copy = DAG.getNode(X86ISD::FP_SET_RESULT, Tys, Ops);
309      }
310      break;
311  }
312
313  return DAG.getNode(X86ISD::RET_FLAG, MVT::Other,
314                     Copy, DAG.getConstant(getBytesToPopOnReturn(), MVT::i16),
315                     Copy.getValue(1));
316}
317
318//===----------------------------------------------------------------------===//
319//                    C Calling Convention implementation
320//===----------------------------------------------------------------------===//
321
322std::vector<SDOperand>
323X86TargetLowering::LowerCCCArguments(Function &F, SelectionDAG &DAG) {
324  std::vector<SDOperand> ArgValues;
325
326  MachineFunction &MF = DAG.getMachineFunction();
327  MachineFrameInfo *MFI = MF.getFrameInfo();
328
329  // Add DAG nodes to load the arguments...  On entry to a function on the X86,
330  // the stack frame looks like this:
331  //
332  // [ESP] -- return address
333  // [ESP + 4] -- first argument (leftmost lexically)
334  // [ESP + 8] -- second argument, if first argument is four bytes in size
335  //    ...
336  //
337  unsigned ArgOffset = 0;   // Frame mechanisms handle retaddr slot
338  for (Function::arg_iterator I = F.arg_begin(), E = F.arg_end(); I != E; ++I) {
339    MVT::ValueType ObjectVT = getValueType(I->getType());
340    unsigned ArgIncrement = 4;
341    unsigned ObjSize;
342    switch (ObjectVT) {
343    default: assert(0 && "Unhandled argument type!");
344    case MVT::i1:
345    case MVT::i8:  ObjSize = 1;                break;
346    case MVT::i16: ObjSize = 2;                break;
347    case MVT::i32: ObjSize = 4;                break;
348    case MVT::i64: ObjSize = ArgIncrement = 8; break;
349    case MVT::f32: ObjSize = 4;                break;
350    case MVT::f64: ObjSize = ArgIncrement = 8; break;
351    }
352    // Create the frame index object for this incoming parameter...
353    int FI = MFI->CreateFixedObject(ObjSize, ArgOffset);
354
355    // Create the SelectionDAG nodes corresponding to a load from this parameter
356    SDOperand FIN = DAG.getFrameIndex(FI, MVT::i32);
357
358    // Don't codegen dead arguments.  FIXME: remove this check when we can nuke
359    // dead loads.
360    SDOperand ArgValue;
361    if (!I->use_empty())
362      ArgValue = DAG.getLoad(ObjectVT, DAG.getEntryNode(), FIN,
363                             DAG.getSrcValue(NULL));
364    else {
365      if (MVT::isInteger(ObjectVT))
366        ArgValue = DAG.getConstant(0, ObjectVT);
367      else
368        ArgValue = DAG.getConstantFP(0, ObjectVT);
369    }
370    ArgValues.push_back(ArgValue);
371
372    ArgOffset += ArgIncrement;   // Move on to the next argument...
373  }
374
375  // If the function takes variable number of arguments, make a frame index for
376  // the start of the first vararg value... for expansion of llvm.va_start.
377  if (F.isVarArg())
378    VarArgsFrameIndex = MFI->CreateFixedObject(1, ArgOffset);
379  ReturnAddrIndex = 0;     // No return address slot generated yet.
380  BytesToPopOnReturn = 0;  // Callee pops nothing.
381  BytesCallerReserves = ArgOffset;
382
383  // Finally, inform the code generator which regs we return values in.
384  switch (getValueType(F.getReturnType())) {
385  default: assert(0 && "Unknown type!");
386  case MVT::isVoid: break;
387  case MVT::i1:
388  case MVT::i8:
389  case MVT::i16:
390  case MVT::i32:
391    MF.addLiveOut(X86::EAX);
392    break;
393  case MVT::i64:
394    MF.addLiveOut(X86::EAX);
395    MF.addLiveOut(X86::EDX);
396    break;
397  case MVT::f32:
398  case MVT::f64:
399    MF.addLiveOut(X86::ST0);
400    break;
401  }
402  return ArgValues;
403}
404
405std::pair<SDOperand, SDOperand>
406X86TargetLowering::LowerCCCCallTo(SDOperand Chain, const Type *RetTy,
407                                  bool isVarArg, bool isTailCall,
408                                  SDOperand Callee, ArgListTy &Args,
409                                  SelectionDAG &DAG) {
410  // Count how many bytes are to be pushed on the stack.
411  unsigned NumBytes = 0;
412
413  if (Args.empty()) {
414    // Save zero bytes.
415    Chain = DAG.getNode(ISD::CALLSEQ_START, MVT::Other, Chain,
416                        DAG.getConstant(0, getPointerTy()));
417  } else {
418    for (unsigned i = 0, e = Args.size(); i != e; ++i)
419      switch (getValueType(Args[i].second)) {
420      default: assert(0 && "Unknown value type!");
421      case MVT::i1:
422      case MVT::i8:
423      case MVT::i16:
424      case MVT::i32:
425      case MVT::f32:
426        NumBytes += 4;
427        break;
428      case MVT::i64:
429      case MVT::f64:
430        NumBytes += 8;
431        break;
432      }
433
434    Chain = DAG.getNode(ISD::CALLSEQ_START, MVT::Other, Chain,
435                        DAG.getConstant(NumBytes, getPointerTy()));
436
437    // Arguments go on the stack in reverse order, as specified by the ABI.
438    unsigned ArgOffset = 0;
439    SDOperand StackPtr = DAG.getRegister(X86::ESP, MVT::i32);
440    std::vector<SDOperand> Stores;
441
442    for (unsigned i = 0, e = Args.size(); i != e; ++i) {
443      SDOperand PtrOff = DAG.getConstant(ArgOffset, getPointerTy());
444      PtrOff = DAG.getNode(ISD::ADD, MVT::i32, StackPtr, PtrOff);
445
446      switch (getValueType(Args[i].second)) {
447      default: assert(0 && "Unexpected ValueType for argument!");
448      case MVT::i1:
449      case MVT::i8:
450      case MVT::i16:
451        // Promote the integer to 32 bits.  If the input type is signed use a
452        // sign extend, otherwise use a zero extend.
453        if (Args[i].second->isSigned())
454          Args[i].first =DAG.getNode(ISD::SIGN_EXTEND, MVT::i32, Args[i].first);
455        else
456          Args[i].first =DAG.getNode(ISD::ZERO_EXTEND, MVT::i32, Args[i].first);
457
458        // FALL THROUGH
459      case MVT::i32:
460      case MVT::f32:
461        Stores.push_back(DAG.getNode(ISD::STORE, MVT::Other, Chain,
462                                     Args[i].first, PtrOff,
463                                     DAG.getSrcValue(NULL)));
464        ArgOffset += 4;
465        break;
466      case MVT::i64:
467      case MVT::f64:
468        Stores.push_back(DAG.getNode(ISD::STORE, MVT::Other, Chain,
469                                     Args[i].first, PtrOff,
470                                     DAG.getSrcValue(NULL)));
471        ArgOffset += 8;
472        break;
473      }
474    }
475    Chain = DAG.getNode(ISD::TokenFactor, MVT::Other, Stores);
476  }
477
478  std::vector<MVT::ValueType> RetVals;
479  MVT::ValueType RetTyVT = getValueType(RetTy);
480  RetVals.push_back(MVT::Other);
481
482  // The result values produced have to be legal.  Promote the result.
483  switch (RetTyVT) {
484  case MVT::isVoid: break;
485  default:
486    RetVals.push_back(RetTyVT);
487    break;
488  case MVT::i1:
489  case MVT::i8:
490  case MVT::i16:
491    RetVals.push_back(MVT::i32);
492    break;
493  case MVT::f32:
494    if (X86ScalarSSE)
495      RetVals.push_back(MVT::f32);
496    else
497      RetVals.push_back(MVT::f64);
498    break;
499  case MVT::i64:
500    RetVals.push_back(MVT::i32);
501    RetVals.push_back(MVT::i32);
502    break;
503  }
504
505  if (X86DAGIsel) {
506    std::vector<MVT::ValueType> NodeTys;
507    NodeTys.push_back(MVT::Other);   // Returns a chain
508    NodeTys.push_back(MVT::Flag);    // Returns a flag for retval copy to use.
509    std::vector<SDOperand> Ops;
510    Ops.push_back(Chain);
511    Ops.push_back(Callee);
512
513    // FIXME: Do not generate X86ISD::TAILCALL for now.
514    Chain = DAG.getNode(X86ISD::CALL, NodeTys, Ops);
515    SDOperand InFlag = Chain.getValue(1);
516
517    SDOperand RetVal;
518    if (RetTyVT != MVT::isVoid) {
519      switch (RetTyVT) {
520      default: assert(0 && "Unknown value type to return!");
521      case MVT::i1:
522      case MVT::i8:
523        RetVal = DAG.getCopyFromReg(Chain, X86::AL, MVT::i8, InFlag);
524        Chain = RetVal.getValue(1);
525        break;
526      case MVT::i16:
527        RetVal = DAG.getCopyFromReg(Chain, X86::AX, MVT::i16, InFlag);
528        Chain = RetVal.getValue(1);
529        break;
530      case MVT::i32:
531        RetVal = DAG.getCopyFromReg(Chain, X86::EAX, MVT::i32, InFlag);
532        Chain = RetVal.getValue(1);
533        break;
534      case MVT::i64: {
535        SDOperand Lo = DAG.getCopyFromReg(Chain, X86::EAX, MVT::i32, InFlag);
536        SDOperand Hi = DAG.getCopyFromReg(Lo.getValue(1), X86::EDX, MVT::i32,
537                                          Lo.getValue(2));
538        RetVal = DAG.getNode(ISD::BUILD_PAIR, MVT::i64, Lo, Hi);
539        Chain = Hi.getValue(1);
540        break;
541      }
542      case MVT::f32:
543      case MVT::f64: {
544        std::vector<MVT::ValueType> Tys;
545        Tys.push_back(MVT::f64);
546        Tys.push_back(MVT::Other);
547        std::vector<SDOperand> Ops;
548        Ops.push_back(Chain);
549        Ops.push_back(InFlag);
550        RetVal = DAG.getNode(X86ISD::FP_GET_RESULT, Tys, Ops);
551        Chain = RetVal.getValue(1);
552        if (X86ScalarSSE) {
553          unsigned Size = MVT::getSizeInBits(MVT::f64)/8;
554          MachineFunction &MF = DAG.getMachineFunction();
555          int SSFI = MF.getFrameInfo()->CreateStackObject(Size, Size);
556          SDOperand StackSlot = DAG.getFrameIndex(SSFI, getPointerTy());
557          Tys.clear();
558          Tys.push_back(MVT::Other);
559          Ops.clear();
560          Ops.push_back(Chain);
561          Ops.push_back(RetVal);
562          Ops.push_back(StackSlot);
563          Ops.push_back(DAG.getValueType(RetTyVT));
564          Chain = DAG.getNode(X86ISD::FST, Tys, Ops);
565          RetVal = DAG.getLoad(RetTyVT, Chain, StackSlot,
566                               DAG.getSrcValue(NULL));
567          Chain = RetVal.getValue(1);
568        } else if (RetTyVT == MVT::f32)
569          RetVal = DAG.getNode(ISD::FP_ROUND, MVT::f32, RetVal);
570        break;
571      }
572      }
573    }
574
575    Chain = DAG.getNode(ISD::CALLSEQ_END, MVT::Other, Chain,
576                        DAG.getConstant(NumBytes, getPointerTy()),
577                        DAG.getConstant(0, getPointerTy()));
578    return std::make_pair(RetVal, Chain);
579  } else {
580    std::vector<SDOperand> Ops;
581    Ops.push_back(Chain);
582    Ops.push_back(Callee);
583    Ops.push_back(DAG.getConstant(NumBytes, getPointerTy()));
584    Ops.push_back(DAG.getConstant(0, getPointerTy()));
585
586    SDOperand TheCall = DAG.getNode(isTailCall ? X86ISD::TAILCALL : X86ISD::CALL,
587                                    RetVals, Ops);
588
589    SDOperand ResultVal;
590    switch (RetTyVT) {
591    case MVT::isVoid: break;
592    default:
593      ResultVal = TheCall.getValue(1);
594      break;
595    case MVT::i1:
596    case MVT::i8:
597    case MVT::i16:
598      ResultVal = DAG.getNode(ISD::TRUNCATE, RetTyVT, TheCall.getValue(1));
599      break;
600    case MVT::f32:
601      // FIXME: we would really like to remember that this FP_ROUND operation is
602      // okay to eliminate if we allow excess FP precision.
603      ResultVal = DAG.getNode(ISD::FP_ROUND, MVT::f32, TheCall.getValue(1));
604      break;
605    case MVT::i64:
606      ResultVal = DAG.getNode(ISD::BUILD_PAIR, MVT::i64, TheCall.getValue(1),
607                              TheCall.getValue(2));
608      break;
609    }
610
611    Chain = DAG.getNode(ISD::CALLSEQ_END, MVT::Other, TheCall);
612    return std::make_pair(ResultVal, Chain);
613  }
614}
615
616SDOperand
617X86TargetLowering::LowerVAStart(SDOperand Chain, SDOperand VAListP,
618                                Value *VAListV, SelectionDAG &DAG) {
619  // vastart just stores the address of the VarArgsFrameIndex slot.
620  SDOperand FR = DAG.getFrameIndex(VarArgsFrameIndex, MVT::i32);
621  return DAG.getNode(ISD::STORE, MVT::Other, Chain, FR, VAListP,
622                     DAG.getSrcValue(VAListV));
623}
624
625
626std::pair<SDOperand,SDOperand>
627X86TargetLowering::LowerVAArg(SDOperand Chain, SDOperand VAListP,
628                              Value *VAListV, const Type *ArgTy,
629                              SelectionDAG &DAG) {
630  MVT::ValueType ArgVT = getValueType(ArgTy);
631  SDOperand Val = DAG.getLoad(MVT::i32, Chain,
632                              VAListP, DAG.getSrcValue(VAListV));
633  SDOperand Result = DAG.getLoad(ArgVT, Chain, Val,
634                                 DAG.getSrcValue(NULL));
635  unsigned Amt;
636  if (ArgVT == MVT::i32)
637    Amt = 4;
638  else {
639    assert((ArgVT == MVT::i64 || ArgVT == MVT::f64) &&
640           "Other types should have been promoted for varargs!");
641    Amt = 8;
642  }
643  Val = DAG.getNode(ISD::ADD, Val.getValueType(), Val,
644                    DAG.getConstant(Amt, Val.getValueType()));
645  Chain = DAG.getNode(ISD::STORE, MVT::Other, Chain,
646                      Val, VAListP, DAG.getSrcValue(VAListV));
647  return std::make_pair(Result, Chain);
648}
649
650//===----------------------------------------------------------------------===//
651//                    Fast Calling Convention implementation
652//===----------------------------------------------------------------------===//
653//
654// The X86 'fast' calling convention passes up to two integer arguments in
655// registers (an appropriate portion of EAX/EDX), passes arguments in C order,
656// and requires that the callee pop its arguments off the stack (allowing proper
657// tail calls), and has the same return value conventions as C calling convs.
658//
659// This calling convention always arranges for the callee pop value to be 8n+4
660// bytes, which is needed for tail recursion elimination and stack alignment
661// reasons.
662//
663// Note that this can be enhanced in the future to pass fp vals in registers
664// (when we have a global fp allocator) and do other tricks.
665//
666
667/// AddLiveIn - This helper function adds the specified physical register to the
668/// MachineFunction as a live in value.  It also creates a corresponding virtual
669/// register for it.
670static unsigned AddLiveIn(MachineFunction &MF, unsigned PReg,
671                          TargetRegisterClass *RC) {
672  assert(RC->contains(PReg) && "Not the correct regclass!");
673  unsigned VReg = MF.getSSARegMap()->createVirtualRegister(RC);
674  MF.addLiveIn(PReg, VReg);
675  return VReg;
676}
677
678
679std::vector<SDOperand>
680X86TargetLowering::LowerFastCCArguments(Function &F, SelectionDAG &DAG) {
681  std::vector<SDOperand> ArgValues;
682
683  MachineFunction &MF = DAG.getMachineFunction();
684  MachineFrameInfo *MFI = MF.getFrameInfo();
685
686  // Add DAG nodes to load the arguments...  On entry to a function the stack
687  // frame looks like this:
688  //
689  // [ESP] -- return address
690  // [ESP + 4] -- first nonreg argument (leftmost lexically)
691  // [ESP + 8] -- second nonreg argument, if first argument is 4 bytes in size
692  //    ...
693  unsigned ArgOffset = 0;   // Frame mechanisms handle retaddr slot
694
695  // Keep track of the number of integer regs passed so far.  This can be either
696  // 0 (neither EAX or EDX used), 1 (EAX is used) or 2 (EAX and EDX are both
697  // used).
698  unsigned NumIntRegs = 0;
699
700  for (Function::arg_iterator I = F.arg_begin(), E = F.arg_end(); I != E; ++I) {
701    MVT::ValueType ObjectVT = getValueType(I->getType());
702    unsigned ArgIncrement = 4;
703    unsigned ObjSize = 0;
704    SDOperand ArgValue;
705
706    switch (ObjectVT) {
707    default: assert(0 && "Unhandled argument type!");
708    case MVT::i1:
709    case MVT::i8:
710      if (NumIntRegs < 2) {
711        if (!I->use_empty()) {
712          unsigned VReg = AddLiveIn(MF, NumIntRegs ? X86::DL : X86::AL,
713                                    X86::R8RegisterClass);
714          ArgValue = DAG.getCopyFromReg(DAG.getRoot(), VReg, MVT::i8);
715          DAG.setRoot(ArgValue.getValue(1));
716          if (ObjectVT == MVT::i1)
717            // FIXME: Should insert a assertzext here.
718            ArgValue = DAG.getNode(ISD::TRUNCATE, MVT::i1, ArgValue);
719        }
720        ++NumIntRegs;
721        break;
722      }
723
724      ObjSize = 1;
725      break;
726    case MVT::i16:
727      if (NumIntRegs < 2) {
728        if (!I->use_empty()) {
729          unsigned VReg = AddLiveIn(MF, NumIntRegs ? X86::DX : X86::AX,
730                                    X86::R16RegisterClass);
731          ArgValue = DAG.getCopyFromReg(DAG.getRoot(), VReg, MVT::i16);
732          DAG.setRoot(ArgValue.getValue(1));
733        }
734        ++NumIntRegs;
735        break;
736      }
737      ObjSize = 2;
738      break;
739    case MVT::i32:
740      if (NumIntRegs < 2) {
741        if (!I->use_empty()) {
742          unsigned VReg = AddLiveIn(MF,NumIntRegs ? X86::EDX : X86::EAX,
743                                    X86::R32RegisterClass);
744          ArgValue = DAG.getCopyFromReg(DAG.getRoot(), VReg, MVT::i32);
745          DAG.setRoot(ArgValue.getValue(1));
746        }
747        ++NumIntRegs;
748        break;
749      }
750      ObjSize = 4;
751      break;
752    case MVT::i64:
753      if (NumIntRegs == 0) {
754        if (!I->use_empty()) {
755          unsigned BotReg = AddLiveIn(MF, X86::EAX, X86::R32RegisterClass);
756          unsigned TopReg = AddLiveIn(MF, X86::EDX, X86::R32RegisterClass);
757
758          SDOperand Low = DAG.getCopyFromReg(DAG.getRoot(), BotReg, MVT::i32);
759          SDOperand Hi  = DAG.getCopyFromReg(Low.getValue(1), TopReg, MVT::i32);
760          DAG.setRoot(Hi.getValue(1));
761
762          ArgValue = DAG.getNode(ISD::BUILD_PAIR, MVT::i64, Low, Hi);
763        }
764        NumIntRegs = 2;
765        break;
766      } else if (NumIntRegs == 1) {
767        if (!I->use_empty()) {
768          unsigned BotReg = AddLiveIn(MF, X86::EDX, X86::R32RegisterClass);
769          SDOperand Low = DAG.getCopyFromReg(DAG.getRoot(), BotReg, MVT::i32);
770          DAG.setRoot(Low.getValue(1));
771
772          // Load the high part from memory.
773          // Create the frame index object for this incoming parameter...
774          int FI = MFI->CreateFixedObject(4, ArgOffset);
775          SDOperand FIN = DAG.getFrameIndex(FI, MVT::i32);
776          SDOperand Hi = DAG.getLoad(MVT::i32, DAG.getEntryNode(), FIN,
777                                     DAG.getSrcValue(NULL));
778          ArgValue = DAG.getNode(ISD::BUILD_PAIR, MVT::i64, Low, Hi);
779        }
780        ArgOffset += 4;
781        NumIntRegs = 2;
782        break;
783      }
784      ObjSize = ArgIncrement = 8;
785      break;
786    case MVT::f32: ObjSize = 4;                break;
787    case MVT::f64: ObjSize = ArgIncrement = 8; break;
788    }
789
790    // Don't codegen dead arguments.  FIXME: remove this check when we can nuke
791    // dead loads.
792    if (ObjSize && !I->use_empty()) {
793      // Create the frame index object for this incoming parameter...
794      int FI = MFI->CreateFixedObject(ObjSize, ArgOffset);
795
796      // Create the SelectionDAG nodes corresponding to a load from this
797      // parameter.
798      SDOperand FIN = DAG.getFrameIndex(FI, MVT::i32);
799
800      ArgValue = DAG.getLoad(ObjectVT, DAG.getEntryNode(), FIN,
801                             DAG.getSrcValue(NULL));
802    } else if (ArgValue.Val == 0) {
803      if (MVT::isInteger(ObjectVT))
804        ArgValue = DAG.getConstant(0, ObjectVT);
805      else
806        ArgValue = DAG.getConstantFP(0, ObjectVT);
807    }
808    ArgValues.push_back(ArgValue);
809
810    if (ObjSize)
811      ArgOffset += ArgIncrement;   // Move on to the next argument.
812  }
813
814  // Make sure the instruction takes 8n+4 bytes to make sure the start of the
815  // arguments and the arguments after the retaddr has been pushed are aligned.
816  if ((ArgOffset & 7) == 0)
817    ArgOffset += 4;
818
819  VarArgsFrameIndex = 0xAAAAAAA;   // fastcc functions can't have varargs.
820  ReturnAddrIndex = 0;             // No return address slot generated yet.
821  BytesToPopOnReturn = ArgOffset;  // Callee pops all stack arguments.
822  BytesCallerReserves = 0;
823
824  // Finally, inform the code generator which regs we return values in.
825  switch (getValueType(F.getReturnType())) {
826  default: assert(0 && "Unknown type!");
827  case MVT::isVoid: break;
828  case MVT::i1:
829  case MVT::i8:
830  case MVT::i16:
831  case MVT::i32:
832    MF.addLiveOut(X86::EAX);
833    break;
834  case MVT::i64:
835    MF.addLiveOut(X86::EAX);
836    MF.addLiveOut(X86::EDX);
837    break;
838  case MVT::f32:
839  case MVT::f64:
840    MF.addLiveOut(X86::ST0);
841    break;
842  }
843  return ArgValues;
844}
845
846std::pair<SDOperand, SDOperand>
847X86TargetLowering::LowerFastCCCallTo(SDOperand Chain, const Type *RetTy,
848                                     bool isTailCall, SDOperand Callee,
849                                     ArgListTy &Args, SelectionDAG &DAG) {
850  // Count how many bytes are to be pushed on the stack.
851  unsigned NumBytes = 0;
852
853  // Keep track of the number of integer regs passed so far.  This can be either
854  // 0 (neither EAX or EDX used), 1 (EAX is used) or 2 (EAX and EDX are both
855  // used).
856  unsigned NumIntRegs = 0;
857
858  for (unsigned i = 0, e = Args.size(); i != e; ++i)
859    switch (getValueType(Args[i].second)) {
860    default: assert(0 && "Unknown value type!");
861    case MVT::i1:
862    case MVT::i8:
863    case MVT::i16:
864    case MVT::i32:
865      if (NumIntRegs < 2) {
866        ++NumIntRegs;
867        break;
868      }
869      // fall through
870    case MVT::f32:
871      NumBytes += 4;
872      break;
873    case MVT::i64:
874      if (NumIntRegs == 0) {
875        NumIntRegs = 2;
876        break;
877      } else if (NumIntRegs == 1) {
878        NumIntRegs = 2;
879        NumBytes += 4;
880        break;
881      }
882
883      // fall through
884    case MVT::f64:
885      NumBytes += 8;
886      break;
887    }
888
889  // Make sure the instruction takes 8n+4 bytes to make sure the start of the
890  // arguments and the arguments after the retaddr has been pushed are aligned.
891  if ((NumBytes & 7) == 0)
892    NumBytes += 4;
893
894  Chain = DAG.getNode(ISD::CALLSEQ_START, MVT::Other, Chain,
895                      DAG.getConstant(NumBytes, getPointerTy()));
896
897  // Arguments go on the stack in reverse order, as specified by the ABI.
898  unsigned ArgOffset = 0;
899  SDOperand StackPtr = DAG.getCopyFromReg(DAG.getEntryNode(),
900                                          X86::ESP, MVT::i32);
901  NumIntRegs = 0;
902  std::vector<SDOperand> Stores;
903  std::vector<SDOperand> RegValuesToPass;
904  for (unsigned i = 0, e = Args.size(); i != e; ++i) {
905    switch (getValueType(Args[i].second)) {
906    default: assert(0 && "Unexpected ValueType for argument!");
907    case MVT::i1:
908      Args[i].first = DAG.getNode(ISD::ANY_EXTEND, MVT::i8, Args[i].first);
909      // Fall through.
910    case MVT::i8:
911    case MVT::i16:
912    case MVT::i32:
913      if (NumIntRegs < 2) {
914        RegValuesToPass.push_back(Args[i].first);
915        ++NumIntRegs;
916        break;
917      }
918      // Fall through
919    case MVT::f32: {
920      SDOperand PtrOff = DAG.getConstant(ArgOffset, getPointerTy());
921      PtrOff = DAG.getNode(ISD::ADD, MVT::i32, StackPtr, PtrOff);
922      Stores.push_back(DAG.getNode(ISD::STORE, MVT::Other, Chain,
923                                   Args[i].first, PtrOff,
924                                   DAG.getSrcValue(NULL)));
925      ArgOffset += 4;
926      break;
927    }
928    case MVT::i64:
929      if (NumIntRegs < 2) {    // Can pass part of it in regs?
930        SDOperand Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, MVT::i32,
931                                   Args[i].first, DAG.getConstant(1, MVT::i32));
932        SDOperand Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, MVT::i32,
933                                   Args[i].first, DAG.getConstant(0, MVT::i32));
934        RegValuesToPass.push_back(Lo);
935        ++NumIntRegs;
936        if (NumIntRegs < 2) {   // Pass both parts in regs?
937          RegValuesToPass.push_back(Hi);
938          ++NumIntRegs;
939        } else {
940          // Pass the high part in memory.
941          SDOperand PtrOff = DAG.getConstant(ArgOffset, getPointerTy());
942          PtrOff = DAG.getNode(ISD::ADD, MVT::i32, StackPtr, PtrOff);
943          Stores.push_back(DAG.getNode(ISD::STORE, MVT::Other, Chain,
944                                       Hi, PtrOff, DAG.getSrcValue(NULL)));
945          ArgOffset += 4;
946        }
947        break;
948      }
949      // Fall through
950    case MVT::f64:
951      SDOperand PtrOff = DAG.getConstant(ArgOffset, getPointerTy());
952      PtrOff = DAG.getNode(ISD::ADD, MVT::i32, StackPtr, PtrOff);
953      Stores.push_back(DAG.getNode(ISD::STORE, MVT::Other, Chain,
954                                   Args[i].first, PtrOff,
955                                   DAG.getSrcValue(NULL)));
956      ArgOffset += 8;
957      break;
958    }
959  }
960  if (!Stores.empty())
961    Chain = DAG.getNode(ISD::TokenFactor, MVT::Other, Stores);
962
963  // Make sure the instruction takes 8n+4 bytes to make sure the start of the
964  // arguments and the arguments after the retaddr has been pushed are aligned.
965  if ((ArgOffset & 7) == 0)
966    ArgOffset += 4;
967
968  std::vector<MVT::ValueType> RetVals;
969  MVT::ValueType RetTyVT = getValueType(RetTy);
970
971  RetVals.push_back(MVT::Other);
972
973  // The result values produced have to be legal.  Promote the result.
974  switch (RetTyVT) {
975  case MVT::isVoid: break;
976  default:
977    RetVals.push_back(RetTyVT);
978    break;
979  case MVT::i1:
980  case MVT::i8:
981  case MVT::i16:
982    RetVals.push_back(MVT::i32);
983    break;
984  case MVT::f32:
985    if (X86ScalarSSE)
986      RetVals.push_back(MVT::f32);
987    else
988      RetVals.push_back(MVT::f64);
989    break;
990  case MVT::i64:
991    RetVals.push_back(MVT::i32);
992    RetVals.push_back(MVT::i32);
993    break;
994  }
995
996  if (X86DAGIsel) {
997    // Build a sequence of copy-to-reg nodes chained together with token chain
998    // and flag operands which copy the outgoing args into registers.
999    SDOperand InFlag;
1000    for (unsigned i = 0, e = RegValuesToPass.size(); i != e; ++i) {
1001      unsigned CCReg;
1002      SDOperand RegToPass = RegValuesToPass[i];
1003      switch (RegToPass.getValueType()) {
1004      default: assert(0 && "Bad thing to pass in regs");
1005      case MVT::i8:
1006        CCReg = (i == 0) ? X86::AL  : X86::DL;
1007        break;
1008      case MVT::i16:
1009        CCReg = (i == 0) ? X86::AX  : X86::DX;
1010        break;
1011      case MVT::i32:
1012        CCReg = (i == 0) ? X86::EAX : X86::EDX;
1013        break;
1014      }
1015
1016      Chain = DAG.getCopyToReg(Chain, CCReg, RegToPass, InFlag);
1017      InFlag = Chain.getValue(1);
1018    }
1019
1020    std::vector<MVT::ValueType> NodeTys;
1021    NodeTys.push_back(MVT::Other);   // Returns a chain
1022    NodeTys.push_back(MVT::Flag);    // Returns a flag for retval copy to use.
1023    std::vector<SDOperand> Ops;
1024    Ops.push_back(Chain);
1025    Ops.push_back(Callee);
1026    if (InFlag.Val)
1027      Ops.push_back(InFlag);
1028
1029    // FIXME: Do not generate X86ISD::TAILCALL for now.
1030    Chain = DAG.getNode(X86ISD::CALL, NodeTys, Ops);
1031    InFlag = Chain.getValue(1);
1032
1033    SDOperand RetVal;
1034    if (RetTyVT != MVT::isVoid) {
1035      switch (RetTyVT) {
1036      default: assert(0 && "Unknown value type to return!");
1037      case MVT::i1:
1038      case MVT::i8:
1039        RetVal = DAG.getCopyFromReg(Chain, X86::AL, MVT::i8, InFlag);
1040        Chain = RetVal.getValue(1);
1041        break;
1042      case MVT::i16:
1043        RetVal = DAG.getCopyFromReg(Chain, X86::AX, MVT::i16, InFlag);
1044        Chain = RetVal.getValue(1);
1045        break;
1046      case MVT::i32:
1047        RetVal = DAG.getCopyFromReg(Chain, X86::EAX, MVT::i32, InFlag);
1048        Chain = RetVal.getValue(1);
1049        break;
1050      case MVT::i64: {
1051        SDOperand Lo = DAG.getCopyFromReg(Chain, X86::EAX, MVT::i32, InFlag);
1052        SDOperand Hi = DAG.getCopyFromReg(Lo.getValue(1), X86::EDX, MVT::i32,
1053                                          Lo.getValue(2));
1054        RetVal = DAG.getNode(ISD::BUILD_PAIR, MVT::i64, Lo, Hi);
1055        Chain = Hi.getValue(1);
1056        break;
1057      }
1058      case MVT::f32:
1059      case MVT::f64: {
1060        std::vector<MVT::ValueType> Tys;
1061        Tys.push_back(MVT::f64);
1062        Tys.push_back(MVT::Other);
1063        std::vector<SDOperand> Ops;
1064        Ops.push_back(Chain);
1065        Ops.push_back(InFlag);
1066        RetVal = DAG.getNode(X86ISD::FP_GET_RESULT, Tys, Ops);
1067        Chain = RetVal.getValue(1);
1068        if (X86ScalarSSE) {
1069          unsigned Size = MVT::getSizeInBits(MVT::f64)/8;
1070          MachineFunction &MF = DAG.getMachineFunction();
1071          int SSFI = MF.getFrameInfo()->CreateStackObject(Size, Size);
1072          SDOperand StackSlot = DAG.getFrameIndex(SSFI, getPointerTy());
1073          Tys.clear();
1074          Tys.push_back(MVT::Other);
1075          Ops.clear();
1076          Ops.push_back(Chain);
1077          Ops.push_back(RetVal);
1078          Ops.push_back(StackSlot);
1079          Ops.push_back(DAG.getValueType(RetTyVT));
1080          Chain = DAG.getNode(X86ISD::FST, Tys, Ops);
1081          RetVal = DAG.getLoad(RetTyVT, Chain, StackSlot,
1082                               DAG.getSrcValue(NULL));
1083          Chain = RetVal.getValue(1);
1084        } else if (RetTyVT == MVT::f32)
1085          RetVal = DAG.getNode(ISD::FP_ROUND, MVT::f32, RetVal);
1086        break;
1087      }
1088      }
1089    }
1090
1091    Chain = DAG.getNode(ISD::CALLSEQ_END, MVT::Other, Chain,
1092                        DAG.getConstant(ArgOffset, getPointerTy()),
1093                        DAG.getConstant(ArgOffset, getPointerTy()));
1094    return std::make_pair(RetVal, Chain);
1095  } else {
1096    std::vector<SDOperand> Ops;
1097    Ops.push_back(Chain);
1098    Ops.push_back(Callee);
1099    Ops.push_back(DAG.getConstant(ArgOffset, getPointerTy()));
1100    // Callee pops all arg values on the stack.
1101    Ops.push_back(DAG.getConstant(ArgOffset, getPointerTy()));
1102
1103    // Pass register arguments as needed.
1104    Ops.insert(Ops.end(), RegValuesToPass.begin(), RegValuesToPass.end());
1105
1106    SDOperand TheCall = DAG.getNode(isTailCall ? X86ISD::TAILCALL : X86ISD::CALL,
1107                                    RetVals, Ops);
1108    Chain = DAG.getNode(ISD::CALLSEQ_END, MVT::Other, TheCall);
1109
1110    SDOperand ResultVal;
1111    switch (RetTyVT) {
1112    case MVT::isVoid: break;
1113    default:
1114      ResultVal = TheCall.getValue(1);
1115      break;
1116    case MVT::i1:
1117    case MVT::i8:
1118    case MVT::i16:
1119      ResultVal = DAG.getNode(ISD::TRUNCATE, RetTyVT, TheCall.getValue(1));
1120      break;
1121    case MVT::f32:
1122      // FIXME: we would really like to remember that this FP_ROUND operation is
1123      // okay to eliminate if we allow excess FP precision.
1124      ResultVal = DAG.getNode(ISD::FP_ROUND, MVT::f32, TheCall.getValue(1));
1125      break;
1126    case MVT::i64:
1127      ResultVal = DAG.getNode(ISD::BUILD_PAIR, MVT::i64, TheCall.getValue(1),
1128                              TheCall.getValue(2));
1129      break;
1130    }
1131
1132    return std::make_pair(ResultVal, Chain);
1133  }
1134}
1135
1136SDOperand X86TargetLowering::getReturnAddressFrameIndex(SelectionDAG &DAG) {
1137  if (ReturnAddrIndex == 0) {
1138    // Set up a frame object for the return address.
1139    MachineFunction &MF = DAG.getMachineFunction();
1140    ReturnAddrIndex = MF.getFrameInfo()->CreateFixedObject(4, -4);
1141  }
1142
1143  return DAG.getFrameIndex(ReturnAddrIndex, MVT::i32);
1144}
1145
1146
1147
1148std::pair<SDOperand, SDOperand> X86TargetLowering::
1149LowerFrameReturnAddress(bool isFrameAddress, SDOperand Chain, unsigned Depth,
1150                        SelectionDAG &DAG) {
1151  SDOperand Result;
1152  if (Depth)        // Depths > 0 not supported yet!
1153    Result = DAG.getConstant(0, getPointerTy());
1154  else {
1155    SDOperand RetAddrFI = getReturnAddressFrameIndex(DAG);
1156    if (!isFrameAddress)
1157      // Just load the return address
1158      Result = DAG.getLoad(MVT::i32, DAG.getEntryNode(), RetAddrFI,
1159                           DAG.getSrcValue(NULL));
1160    else
1161      Result = DAG.getNode(ISD::SUB, MVT::i32, RetAddrFI,
1162                           DAG.getConstant(4, MVT::i32));
1163  }
1164  return std::make_pair(Result, Chain);
1165}
1166
1167/// getCondBrOpcodeForX86CC - Returns the X86 conditional branch opcode
1168/// which corresponds to the condition code.
1169static unsigned getCondBrOpcodeForX86CC(unsigned X86CC) {
1170  switch (X86CC) {
1171  default: assert(0 && "Unknown X86 conditional code!");
1172  case X86ISD::COND_A:  return X86::JA;
1173  case X86ISD::COND_AE: return X86::JAE;
1174  case X86ISD::COND_B:  return X86::JB;
1175  case X86ISD::COND_BE: return X86::JBE;
1176  case X86ISD::COND_E:  return X86::JE;
1177  case X86ISD::COND_G:  return X86::JG;
1178  case X86ISD::COND_GE: return X86::JGE;
1179  case X86ISD::COND_L:  return X86::JL;
1180  case X86ISD::COND_LE: return X86::JLE;
1181  case X86ISD::COND_NE: return X86::JNE;
1182  case X86ISD::COND_NO: return X86::JNO;
1183  case X86ISD::COND_NP: return X86::JNP;
1184  case X86ISD::COND_NS: return X86::JNS;
1185  case X86ISD::COND_O:  return X86::JO;
1186  case X86ISD::COND_P:  return X86::JP;
1187  case X86ISD::COND_S:  return X86::JS;
1188  }
1189}
1190
1191/// getX86CC - do a one to one translation of a ISD::CondCode to the X86
1192/// specific condition code. It returns a X86ISD::COND_INVALID if it cannot
1193/// do a direct translation.
1194static unsigned getX86CC(SDOperand CC, bool isFP) {
1195  ISD::CondCode SetCCOpcode = cast<CondCodeSDNode>(CC)->get();
1196  unsigned X86CC = X86ISD::COND_INVALID;
1197  if (!isFP) {
1198    switch (SetCCOpcode) {
1199    default: break;
1200    case ISD::SETEQ:  X86CC = X86ISD::COND_E;  break;
1201    case ISD::SETGT:  X86CC = X86ISD::COND_G;  break;
1202    case ISD::SETGE:  X86CC = X86ISD::COND_GE; break;
1203    case ISD::SETLT:  X86CC = X86ISD::COND_L;  break;
1204    case ISD::SETLE:  X86CC = X86ISD::COND_LE; break;
1205    case ISD::SETNE:  X86CC = X86ISD::COND_NE; break;
1206    case ISD::SETULT: X86CC = X86ISD::COND_B;  break;
1207    case ISD::SETUGT: X86CC = X86ISD::COND_A;  break;
1208    case ISD::SETULE: X86CC = X86ISD::COND_BE; break;
1209    case ISD::SETUGE: X86CC = X86ISD::COND_AE; break;
1210    }
1211  } else {
1212    // On a floating point condition, the flags are set as follows:
1213    // ZF  PF  CF   op
1214    //  0 | 0 | 0 | X > Y
1215    //  0 | 0 | 1 | X < Y
1216    //  1 | 0 | 0 | X == Y
1217    //  1 | 1 | 1 | unordered
1218    switch (SetCCOpcode) {
1219    default: break;
1220    case ISD::SETUEQ:
1221    case ISD::SETEQ: X86CC = X86ISD::COND_E;  break;
1222    case ISD::SETOGT:
1223    case ISD::SETGT: X86CC = X86ISD::COND_A;  break;
1224    case ISD::SETOGE:
1225    case ISD::SETGE: X86CC = X86ISD::COND_AE; break;
1226    case ISD::SETULT:
1227    case ISD::SETLT: X86CC = X86ISD::COND_B;  break;
1228    case ISD::SETULE:
1229    case ISD::SETLE: X86CC = X86ISD::COND_BE; break;
1230    case ISD::SETONE:
1231    case ISD::SETNE: X86CC = X86ISD::COND_NE; break;
1232    case ISD::SETUO: X86CC = X86ISD::COND_P;  break;
1233    case ISD::SETO:  X86CC = X86ISD::COND_NP; break;
1234    }
1235  }
1236  return X86CC;
1237}
1238
1239/// hasFPCMov - is there a floating point cmov for the specific X86 condition
1240/// code. Current x86 isa includes the following FP cmov instructions:
1241/// fcmovb, fcomvbe, fcomve, fcmovu, fcmovae, fcmova, fcmovne, fcmovnu.
1242static bool hasFPCMov(unsigned X86CC) {
1243  switch (X86CC) {
1244  default:
1245    return false;
1246  case X86ISD::COND_B:
1247  case X86ISD::COND_BE:
1248  case X86ISD::COND_E:
1249  case X86ISD::COND_P:
1250  case X86ISD::COND_A:
1251  case X86ISD::COND_AE:
1252  case X86ISD::COND_NE:
1253  case X86ISD::COND_NP:
1254    return true;
1255  }
1256}
1257
1258MachineBasicBlock *
1259X86TargetLowering::InsertAtEndOfBasicBlock(MachineInstr *MI,
1260                                           MachineBasicBlock *BB) {
1261  assert((MI->getOpcode() == X86::CMOV_FR32 ||
1262          MI->getOpcode() == X86::CMOV_FR64) &&
1263         "Unexpected instr type to insert");
1264
1265  // To "insert" a SELECT_CC instruction, we actually have to insert the diamond
1266  // control-flow pattern.  The incoming instruction knows the destination vreg
1267  // to set, the condition code register to branch on, the true/false values to
1268  // select between, and a branch opcode to use.
1269  const BasicBlock *LLVM_BB = BB->getBasicBlock();
1270  ilist<MachineBasicBlock>::iterator It = BB;
1271  ++It;
1272
1273  //  thisMBB:
1274  //  ...
1275  //   TrueVal = ...
1276  //   cmpTY ccX, r1, r2
1277  //   bCC copy1MBB
1278  //   fallthrough --> copy0MBB
1279  MachineBasicBlock *thisMBB = BB;
1280  MachineBasicBlock *copy0MBB = new MachineBasicBlock(LLVM_BB);
1281  MachineBasicBlock *sinkMBB = new MachineBasicBlock(LLVM_BB);
1282  unsigned Opc = getCondBrOpcodeForX86CC(MI->getOperand(3).getImmedValue());
1283  BuildMI(BB, Opc, 1).addMBB(sinkMBB);
1284  MachineFunction *F = BB->getParent();
1285  F->getBasicBlockList().insert(It, copy0MBB);
1286  F->getBasicBlockList().insert(It, sinkMBB);
1287  // Update machine-CFG edges
1288  BB->addSuccessor(copy0MBB);
1289  BB->addSuccessor(sinkMBB);
1290
1291  //  copy0MBB:
1292  //   %FalseValue = ...
1293  //   # fallthrough to sinkMBB
1294  BB = copy0MBB;
1295
1296  // Update machine-CFG edges
1297  BB->addSuccessor(sinkMBB);
1298
1299  //  sinkMBB:
1300  //   %Result = phi [ %FalseValue, copy0MBB ], [ %TrueValue, thisMBB ]
1301  //  ...
1302  BB = sinkMBB;
1303  BuildMI(BB, X86::PHI, 4, MI->getOperand(0).getReg())
1304    .addReg(MI->getOperand(1).getReg()).addMBB(copy0MBB)
1305    .addReg(MI->getOperand(2).getReg()).addMBB(thisMBB);
1306
1307  delete MI;   // The pseudo instruction is gone now.
1308  return BB;
1309}
1310
1311
1312//===----------------------------------------------------------------------===//
1313//                           X86 Custom Lowering Hooks
1314//===----------------------------------------------------------------------===//
1315
1316/// LowerOperation - Provide custom lowering hooks for some operations.
1317///
1318SDOperand X86TargetLowering::LowerOperation(SDOperand Op, SelectionDAG &DAG) {
1319  switch (Op.getOpcode()) {
1320  default: assert(0 && "Should not custom lower this!");
1321  case ISD::ADD_PARTS:
1322  case ISD::SUB_PARTS: {
1323    assert(Op.getNumOperands() == 4 && Op.getValueType() == MVT::i32 &&
1324           "Not an i64 add/sub!");
1325    bool isAdd = Op.getOpcode() == ISD::ADD_PARTS;
1326    std::vector<MVT::ValueType> Tys;
1327    Tys.push_back(MVT::i32);
1328    Tys.push_back(MVT::Flag);
1329    std::vector<SDOperand> Ops;
1330    Ops.push_back(Op.getOperand(0));
1331    Ops.push_back(Op.getOperand(2));
1332    SDOperand Lo = DAG.getNode(isAdd ? X86ISD::ADD_FLAG : X86ISD::SUB_FLAG,
1333                               Tys, Ops);
1334    SDOperand Hi = DAG.getNode(isAdd ? X86ISD::ADC : X86ISD::SBB, MVT::i32,
1335                               Op.getOperand(1), Op.getOperand(3),
1336                               Lo.getValue(1));
1337    Tys.clear();
1338    Tys.push_back(MVT::i32);
1339    Tys.push_back(MVT::i32);
1340    Ops.clear();
1341    Ops.push_back(Lo);
1342    Ops.push_back(Hi);
1343    return DAG.getNode(ISD::MERGE_VALUES, Tys, Ops);
1344  }
1345  case ISD::SHL_PARTS:
1346  case ISD::SRA_PARTS:
1347  case ISD::SRL_PARTS: {
1348    assert(Op.getNumOperands() == 3 && Op.getValueType() == MVT::i32 &&
1349           "Not an i64 shift!");
1350    bool isSRA = Op.getOpcode() == ISD::SRA_PARTS;
1351    SDOperand ShOpLo = Op.getOperand(0);
1352    SDOperand ShOpHi = Op.getOperand(1);
1353    SDOperand ShAmt  = Op.getOperand(2);
1354    SDOperand Tmp1 = isSRA ? DAG.getNode(ISD::SRA, MVT::i32, ShOpHi,
1355                                         DAG.getConstant(31, MVT::i32))
1356                           : DAG.getConstant(0, MVT::i32);
1357
1358    SDOperand Tmp2, Tmp3;
1359    if (Op.getOpcode() == ISD::SHL_PARTS) {
1360      Tmp2 = DAG.getNode(X86ISD::SHLD, MVT::i32, ShOpHi, ShOpLo, ShAmt);
1361      Tmp3 = DAG.getNode(ISD::SHL, MVT::i32, ShOpLo, ShAmt);
1362    } else {
1363      Tmp2 = DAG.getNode(X86ISD::SHRD, MVT::i32, ShOpLo, ShOpHi, ShAmt);
1364      Tmp3 = DAG.getNode(isSRA ? ISD::SRA : ISD::SHL, MVT::i32, ShOpHi, ShAmt);
1365    }
1366
1367    SDOperand InFlag = DAG.getNode(X86ISD::TEST, MVT::Flag,
1368                                   ShAmt, DAG.getConstant(32, MVT::i8));
1369
1370    SDOperand Hi, Lo;
1371    SDOperand CC = DAG.getConstant(X86ISD::COND_NE, MVT::i8);
1372
1373    std::vector<MVT::ValueType> Tys;
1374    Tys.push_back(MVT::i32);
1375    Tys.push_back(MVT::Flag);
1376    std::vector<SDOperand> Ops;
1377    if (Op.getOpcode() == ISD::SHL_PARTS) {
1378      Ops.push_back(Tmp2);
1379      Ops.push_back(Tmp3);
1380      Ops.push_back(CC);
1381      Ops.push_back(InFlag);
1382      Hi = DAG.getNode(X86ISD::CMOV, Tys, Ops);
1383      InFlag = Hi.getValue(1);
1384
1385      Ops.clear();
1386      Ops.push_back(Tmp3);
1387      Ops.push_back(Tmp1);
1388      Ops.push_back(CC);
1389      Ops.push_back(InFlag);
1390      Lo = DAG.getNode(X86ISD::CMOV, Tys, Ops);
1391    } else {
1392      Ops.push_back(Tmp2);
1393      Ops.push_back(Tmp3);
1394      Ops.push_back(CC);
1395      Ops.push_back(InFlag);
1396      Lo = DAG.getNode(X86ISD::CMOV, Tys, Ops);
1397      InFlag = Lo.getValue(1);
1398
1399      Ops.clear();
1400      Ops.push_back(Tmp3);
1401      Ops.push_back(Tmp1);
1402      Ops.push_back(CC);
1403      Ops.push_back(InFlag);
1404      Hi = DAG.getNode(X86ISD::CMOV, Tys, Ops);
1405    }
1406
1407    Tys.clear();
1408    Tys.push_back(MVT::i32);
1409    Tys.push_back(MVT::i32);
1410    Ops.clear();
1411    Ops.push_back(Lo);
1412    Ops.push_back(Hi);
1413    return DAG.getNode(ISD::MERGE_VALUES, Tys, Ops);
1414  }
1415  case ISD::SINT_TO_FP: {
1416    assert(Op.getValueType() == MVT::f64 &&
1417           Op.getOperand(0).getValueType() <= MVT::i64 &&
1418           Op.getOperand(0).getValueType() >= MVT::i16 &&
1419           "Unknown SINT_TO_FP to lower!");
1420
1421    SDOperand Result;
1422    MVT::ValueType SrcVT = Op.getOperand(0).getValueType();
1423    unsigned Size = MVT::getSizeInBits(SrcVT)/8;
1424    MachineFunction &MF = DAG.getMachineFunction();
1425    int SSFI = MF.getFrameInfo()->CreateStackObject(Size, Size);
1426    SDOperand StackSlot = DAG.getFrameIndex(SSFI, getPointerTy());
1427    SDOperand Chain = DAG.getNode(ISD::STORE, MVT::Other,
1428                                  DAG.getEntryNode(), Op.getOperand(0),
1429                                  StackSlot, DAG.getSrcValue(NULL));
1430
1431    // Build the FILD
1432    std::vector<MVT::ValueType> Tys;
1433    Tys.push_back(MVT::f64);
1434    Tys.push_back(MVT::Flag);
1435    std::vector<SDOperand> Ops;
1436    Ops.push_back(Chain);
1437    Ops.push_back(StackSlot);
1438    Ops.push_back(DAG.getValueType(SrcVT));
1439    Result = DAG.getNode(X86ISD::FILD, Tys, Ops);
1440    return Result;
1441  }
1442  case ISD::FP_TO_SINT: {
1443    assert(Op.getValueType() <= MVT::i64 && Op.getValueType() >= MVT::i16 &&
1444           Op.getOperand(0).getValueType() == MVT::f64 &&
1445           "Unknown FP_TO_SINT to lower!");
1446    // We lower FP->sint64 into FISTP64, followed by a load, all to a temporary
1447    // stack slot.
1448    MachineFunction &MF = DAG.getMachineFunction();
1449    unsigned MemSize = MVT::getSizeInBits(Op.getValueType())/8;
1450    int SSFI = MF.getFrameInfo()->CreateStackObject(MemSize, MemSize);
1451    SDOperand StackSlot = DAG.getFrameIndex(SSFI, getPointerTy());
1452
1453    unsigned Opc;
1454    switch (Op.getValueType()) {
1455    default: assert(0 && "Invalid FP_TO_SINT to lower!");
1456    case MVT::i16: Opc = X86ISD::FP_TO_INT16_IN_MEM; break;
1457    case MVT::i32: Opc = X86ISD::FP_TO_INT32_IN_MEM; break;
1458    case MVT::i64: Opc = X86ISD::FP_TO_INT64_IN_MEM; break;
1459    }
1460
1461    // Build the FP_TO_INT*_IN_MEM
1462    std::vector<SDOperand> Ops;
1463    Ops.push_back(DAG.getEntryNode());
1464    Ops.push_back(Op.getOperand(0));
1465    Ops.push_back(StackSlot);
1466    SDOperand FIST = DAG.getNode(Opc, MVT::Other, Ops);
1467
1468    // Load the result.
1469    return DAG.getLoad(Op.getValueType(), FIST, StackSlot,
1470                       DAG.getSrcValue(NULL));
1471  }
1472  case ISD::READCYCLECOUNTER: {
1473    std::vector<MVT::ValueType> Tys;
1474    Tys.push_back(MVT::Other);
1475    Tys.push_back(MVT::Flag);
1476    std::vector<SDOperand> Ops;
1477    Ops.push_back(Op.getOperand(0));
1478    SDOperand rd = DAG.getNode(X86ISD::RDTSC_DAG, Tys, Ops);
1479    Ops.clear();
1480    Ops.push_back(DAG.getCopyFromReg(rd, X86::EAX, MVT::i32, rd.getValue(1)));
1481    Ops.push_back(DAG.getCopyFromReg(Ops[0].getValue(1), X86::EDX,
1482                                     MVT::i32, Ops[0].getValue(2)));
1483    Ops.push_back(Ops[1].getValue(1));
1484    Tys[0] = Tys[1] = MVT::i32;
1485    Tys.push_back(MVT::Other);
1486    return DAG.getNode(ISD::MERGE_VALUES, Tys, Ops);
1487  }
1488  case ISD::SETCC: {
1489    assert(Op.getValueType() == MVT::i8 && "SetCC type must be 8-bit integer");
1490    SDOperand CC   = Op.getOperand(2);
1491    SDOperand Cond = DAG.getNode(X86ISD::CMP, MVT::Flag,
1492                                 Op.getOperand(0), Op.getOperand(1));
1493    ISD::CondCode SetCCOpcode = cast<CondCodeSDNode>(CC)->get();
1494    bool isFP = MVT::isFloatingPoint(Op.getOperand(1).getValueType());
1495    unsigned X86CC = getX86CC(CC, isFP);
1496    if (X86CC != X86ISD::COND_INVALID) {
1497      return DAG.getNode(X86ISD::SETCC, MVT::i8,
1498                         DAG.getConstant(X86CC, MVT::i8), Cond);
1499    } else {
1500      assert(isFP && "Illegal integer SetCC!");
1501
1502      std::vector<MVT::ValueType> Tys;
1503      std::vector<SDOperand> Ops;
1504      switch (SetCCOpcode) {
1505      default: assert(false && "Illegal floating point SetCC!");
1506      case ISD::SETOEQ: {  // !PF & ZF
1507        Tys.push_back(MVT::i8);
1508        Tys.push_back(MVT::Flag);
1509        Ops.push_back(DAG.getConstant(X86ISD::COND_NP, MVT::i8));
1510        Ops.push_back(Cond);
1511        SDOperand Tmp1 = DAG.getNode(X86ISD::SETCC, Tys, Ops);
1512        SDOperand Tmp2 = DAG.getNode(X86ISD::SETCC, MVT::i8,
1513                                     DAG.getConstant(X86ISD::COND_E, MVT::i8),
1514                                     Tmp1.getValue(1));
1515        return DAG.getNode(ISD::AND, MVT::i8, Tmp1, Tmp2);
1516      }
1517      case ISD::SETOLT: {  // !PF & CF
1518        Tys.push_back(MVT::i8);
1519        Tys.push_back(MVT::Flag);
1520        Ops.push_back(DAG.getConstant(X86ISD::COND_NP, MVT::i8));
1521        Ops.push_back(Cond);
1522        SDOperand Tmp1 = DAG.getNode(X86ISD::SETCC, Tys, Ops);
1523        SDOperand Tmp2 = DAG.getNode(X86ISD::SETCC, MVT::i8,
1524                                     DAG.getConstant(X86ISD::COND_B, MVT::i8),
1525                                     Tmp1.getValue(1));
1526        return DAG.getNode(ISD::AND, MVT::i8, Tmp1, Tmp2);
1527      }
1528      case ISD::SETOLE: {  // !PF & (CF || ZF)
1529        Tys.push_back(MVT::i8);
1530        Tys.push_back(MVT::Flag);
1531        Ops.push_back(DAG.getConstant(X86ISD::COND_NP, MVT::i8));
1532        Ops.push_back(Cond);
1533        SDOperand Tmp1 = DAG.getNode(X86ISD::SETCC, Tys, Ops);
1534        SDOperand Tmp2 = DAG.getNode(X86ISD::SETCC, MVT::i8,
1535                                     DAG.getConstant(X86ISD::COND_BE, MVT::i8),
1536                                     Tmp1.getValue(1));
1537        return DAG.getNode(ISD::AND, MVT::i8, Tmp1, Tmp2);
1538      }
1539      case ISD::SETUGT: {  // PF | (!ZF & !CF)
1540        Tys.push_back(MVT::i8);
1541        Tys.push_back(MVT::Flag);
1542        Ops.push_back(DAG.getConstant(X86ISD::COND_P, MVT::i8));
1543        Ops.push_back(Cond);
1544        SDOperand Tmp1 = DAG.getNode(X86ISD::SETCC, Tys, Ops);
1545        SDOperand Tmp2 = DAG.getNode(X86ISD::SETCC, MVT::i8,
1546                                     DAG.getConstant(X86ISD::COND_A, MVT::i8),
1547                                     Tmp1.getValue(1));
1548        return DAG.getNode(ISD::OR, MVT::i8, Tmp1, Tmp2);
1549      }
1550      case ISD::SETUGE: {  // PF | !CF
1551        Tys.push_back(MVT::i8);
1552        Tys.push_back(MVT::Flag);
1553        Ops.push_back(DAG.getConstant(X86ISD::COND_P, MVT::i8));
1554        Ops.push_back(Cond);
1555        SDOperand Tmp1 = DAG.getNode(X86ISD::SETCC, Tys, Ops);
1556        SDOperand Tmp2 = DAG.getNode(X86ISD::SETCC, MVT::i8,
1557                                     DAG.getConstant(X86ISD::COND_AE, MVT::i8),
1558                                     Tmp1.getValue(1));
1559        return DAG.getNode(ISD::OR, MVT::i8, Tmp1, Tmp2);
1560      }
1561      case ISD::SETUNE: {  // PF | !ZF
1562        Tys.push_back(MVT::i8);
1563        Tys.push_back(MVT::Flag);
1564        Ops.push_back(DAG.getConstant(X86ISD::COND_P, MVT::i8));
1565        Ops.push_back(Cond);
1566        SDOperand Tmp1 = DAG.getNode(X86ISD::SETCC, Tys, Ops);
1567        SDOperand Tmp2 = DAG.getNode(X86ISD::SETCC, MVT::i8,
1568                                     DAG.getConstant(X86ISD::COND_NE, MVT::i8),
1569                                     Tmp1.getValue(1));
1570        return DAG.getNode(ISD::OR, MVT::i8, Tmp1, Tmp2);
1571      }
1572      }
1573    }
1574  }
1575  case ISD::SELECT: {
1576    MVT::ValueType VT = Op.getValueType();
1577    bool isFP      = MVT::isFloatingPoint(VT);
1578    bool isFPStack = isFP && (X86Vector < SSE2);
1579    bool isFPSSE   = isFP && (X86Vector >= SSE2);
1580    bool addTest   = false;
1581    SDOperand Op0 = Op.getOperand(0);
1582    SDOperand Cond, CC;
1583    if (Op0.getOpcode() == X86ISD::SETCC) {
1584      // If condition flag is set by a X86ISD::CMP, then make a copy of it
1585      // (since flag operand cannot be shared). If the X86ISD::SETCC does not
1586      // have another use it will be eliminated.
1587      // If the X86ISD::SETCC has more than one use, then it's probably better
1588      // to use a test instead of duplicating the X86ISD::CMP (for register
1589      // pressure reason).
1590      if (Op0.hasOneUse() && Op0.getOperand(1).getOpcode() == X86ISD::CMP) {
1591        CC   = Op0.getOperand(0);
1592        Cond = Op0.getOperand(1);
1593        addTest =
1594          isFPStack && !hasFPCMov(cast<ConstantSDNode>(CC)->getSignExtended());
1595      } else
1596        addTest = true;
1597    } else if (Op0.getOpcode() == ISD::SETCC) {
1598      CC = Op0.getOperand(2);
1599      bool isFP = MVT::isFloatingPoint(Op0.getOperand(1).getValueType());
1600      unsigned X86CC = getX86CC(CC, isFP);
1601      CC = DAG.getConstant(X86CC, MVT::i8);
1602      Cond = DAG.getNode(X86ISD::CMP, MVT::Flag,
1603                         Op0.getOperand(0), Op0.getOperand(1));
1604      addTest = true;
1605    } else
1606      addTest = true;
1607
1608    if (addTest) {
1609      CC = DAG.getConstant(X86ISD::COND_NE, MVT::i8);
1610      Cond = DAG.getNode(X86ISD::TEST, MVT::Flag, Op0, Op0);
1611    }
1612
1613    std::vector<MVT::ValueType> Tys;
1614    Tys.push_back(Op.getValueType());
1615    Tys.push_back(MVT::Flag);
1616    std::vector<SDOperand> Ops;
1617    // X86ISD::CMOV means set the result (which is operand 1) to the RHS if
1618    // condition is true.
1619    Ops.push_back(Op.getOperand(2));
1620    Ops.push_back(Op.getOperand(1));
1621    Ops.push_back(CC);
1622    Ops.push_back(Cond);
1623    return DAG.getNode(X86ISD::CMOV, Tys, Ops);
1624  }
1625  case ISD::BRCOND: {
1626    bool addTest = false;
1627    SDOperand Cond  = Op.getOperand(1);
1628    SDOperand Dest  = Op.getOperand(2);
1629    SDOperand CC;
1630    if (Cond.getOpcode() == X86ISD::SETCC) {
1631      // If condition flag is set by a X86ISD::CMP, then make a copy of it
1632      // (since flag operand cannot be shared). If the X86ISD::SETCC does not
1633      // have another use it will be eliminated.
1634      // If the X86ISD::SETCC has more than one use, then it's probably better
1635      // to use a test instead of duplicating the X86ISD::CMP (for register
1636      // pressure reason).
1637      if (Cond.hasOneUse() && Cond.getOperand(1).getOpcode() == X86ISD::CMP) {
1638        CC   = Cond.getOperand(0);
1639        Cond = DAG.getNode(X86ISD::CMP, MVT::Flag,
1640                           Cond.getOperand(1).getOperand(0),
1641                           Cond.getOperand(1).getOperand(1));
1642      } else
1643        addTest = true;
1644    } else if (Cond.getOpcode() == ISD::SETCC) {
1645      CC = Cond.getOperand(2);
1646      bool isFP = MVT::isFloatingPoint(Cond.getOperand(1).getValueType());
1647      unsigned X86CC = getX86CC(CC, isFP);
1648      CC = DAG.getConstant(X86CC, MVT::i8);
1649      Cond = DAG.getNode(X86ISD::CMP, MVT::Flag,
1650                         Cond.getOperand(0), Cond.getOperand(1));
1651    } else
1652      addTest = true;
1653
1654    if (addTest) {
1655      CC = DAG.getConstant(X86ISD::COND_NE, MVT::i8);
1656      Cond = DAG.getNode(X86ISD::TEST, MVT::Flag, Cond, Cond);
1657    }
1658    return DAG.getNode(X86ISD::BRCOND, Op.getValueType(),
1659                       Op.getOperand(0), Op.getOperand(2), CC, Cond);
1660  }
1661  case ISD::RET: {
1662    // Can only be return void.
1663    return DAG.getNode(X86ISD::RET_FLAG, MVT::Other, Op.getOperand(0),
1664                       DAG.getConstant(getBytesToPopOnReturn(), MVT::i16));
1665  }
1666  case ISD::MEMSET: {
1667    SDOperand InFlag;
1668    SDOperand Chain = Op.getOperand(0);
1669    unsigned Align =
1670      (unsigned)cast<ConstantSDNode>(Op.getOperand(4))->getValue();
1671    if (Align == 0) Align = 1;
1672
1673    MVT::ValueType AVT;
1674    SDOperand Count;
1675    if (ConstantSDNode *ValC = dyn_cast<ConstantSDNode>(Op.getOperand(2))) {
1676      unsigned ValReg;
1677      unsigned Val = ValC->getValue() & 255;
1678
1679      // If the value is a constant, then we can potentially use larger sets.
1680      switch (Align & 3) {
1681      case 2:   // WORD aligned
1682        AVT = MVT::i16;
1683        if (ConstantSDNode *I = dyn_cast<ConstantSDNode>(Op.getOperand(3)))
1684          Count = DAG.getConstant(I->getValue() / 2, MVT::i32);
1685        else
1686          Count = DAG.getNode(ISD::SRL, MVT::i32, Op.getOperand(3),
1687                              DAG.getConstant(1, MVT::i8));
1688        Val    = (Val << 8) | Val;
1689        ValReg = X86::AX;
1690        break;
1691      case 0:   // DWORD aligned
1692        AVT = MVT::i32;
1693        if (ConstantSDNode *I = dyn_cast<ConstantSDNode>(Op.getOperand(3)))
1694          Count = DAG.getConstant(I->getValue() / 4, MVT::i32);
1695        else
1696          Count = DAG.getNode(ISD::SRL, MVT::i32, Op.getOperand(3),
1697                              DAG.getConstant(2, MVT::i8));
1698        Val = (Val << 8)  | Val;
1699        Val = (Val << 16) | Val;
1700        ValReg = X86::EAX;
1701        break;
1702      default:  // Byte aligned
1703        AVT = MVT::i8;
1704        Count = Op.getOperand(3);
1705        ValReg = X86::AL;
1706        break;
1707      }
1708
1709      Chain  = DAG.getCopyToReg(Chain, ValReg, DAG.getConstant(Val, AVT),
1710                                InFlag);
1711      InFlag = Chain.getValue(1);
1712    } else {
1713      AVT    = MVT::i8;
1714      Count  = Op.getOperand(3);
1715      Chain  = DAG.getCopyToReg(Chain, X86::AL, Op.getOperand(2), InFlag);
1716      InFlag = Chain.getValue(1);
1717    }
1718
1719    Chain  = DAG.getCopyToReg(Chain, X86::ECX, Count, InFlag);
1720    InFlag = Chain.getValue(1);
1721    Chain  = DAG.getCopyToReg(Chain, X86::EDI, Op.getOperand(1), InFlag);
1722    InFlag = Chain.getValue(1);
1723
1724    return DAG.getNode(X86ISD::REP_STOS, MVT::Other, Chain,
1725                       DAG.getValueType(AVT), InFlag);
1726  }
1727  case ISD::MEMCPY: {
1728    SDOperand Chain = Op.getOperand(0);
1729    unsigned Align =
1730      (unsigned)cast<ConstantSDNode>(Op.getOperand(4))->getValue();
1731    if (Align == 0) Align = 1;
1732
1733    MVT::ValueType AVT;
1734    SDOperand Count;
1735    switch (Align & 3) {
1736    case 2:   // WORD aligned
1737      AVT = MVT::i16;
1738      if (ConstantSDNode *I = dyn_cast<ConstantSDNode>(Op.getOperand(3)))
1739        Count = DAG.getConstant(I->getValue() / 2, MVT::i32);
1740      else
1741        Count = DAG.getNode(ISD::SRL, MVT::i32, Op.getOperand(3),
1742                            DAG.getConstant(1, MVT::i8));
1743      break;
1744    case 0:   // DWORD aligned
1745      AVT = MVT::i32;
1746      if (ConstantSDNode *I = dyn_cast<ConstantSDNode>(Op.getOperand(3)))
1747        Count = DAG.getConstant(I->getValue() / 4, MVT::i32);
1748      else
1749        Count = DAG.getNode(ISD::SRL, MVT::i32, Op.getOperand(3),
1750                            DAG.getConstant(2, MVT::i8));
1751      break;
1752    default:  // Byte aligned
1753      AVT = MVT::i8;
1754      Count = Op.getOperand(3);
1755      break;
1756    }
1757
1758    SDOperand InFlag;
1759    Chain  = DAG.getCopyToReg(Chain, X86::ECX, Count, InFlag);
1760    InFlag = Chain.getValue(1);
1761    Chain  = DAG.getCopyToReg(Chain, X86::EDI, Op.getOperand(1), InFlag);
1762    InFlag = Chain.getValue(1);
1763    Chain  = DAG.getCopyToReg(Chain, X86::ESI, Op.getOperand(2), InFlag);
1764    InFlag = Chain.getValue(1);
1765
1766    return DAG.getNode(X86ISD::REP_MOVS, MVT::Other, Chain,
1767                       DAG.getValueType(AVT), InFlag);
1768  }
1769  case ISD::GlobalAddress: {
1770    SDOperand Result;
1771    GlobalValue *GV = cast<GlobalAddressSDNode>(Op)->getGlobal();
1772    // For Darwin, external and weak symbols are indirect, so we want to load
1773    // the value at address GV, not the value of GV itself.  This means that
1774    // the GlobalAddress must be in the base or index register of the address,
1775    // not the GV offset field.
1776    if (getTargetMachine().
1777        getSubtarget<X86Subtarget>().getIndirectExternAndWeakGlobals() &&
1778        (GV->hasWeakLinkage() || GV->isExternal()))
1779      Result = DAG.getLoad(MVT::i32, DAG.getEntryNode(),
1780                           DAG.getTargetGlobalAddress(GV, getPointerTy()),
1781                           DAG.getSrcValue(NULL));
1782    return Result;
1783  }
1784  }
1785}
1786
1787const char *X86TargetLowering::getTargetNodeName(unsigned Opcode) const {
1788  switch (Opcode) {
1789  default: return NULL;
1790  case X86ISD::ADD_FLAG:           return "X86ISD::ADD_FLAG";
1791  case X86ISD::SUB_FLAG:           return "X86ISD::SUB_FLAG";
1792  case X86ISD::ADC:                return "X86ISD::ADC";
1793  case X86ISD::SBB:                return "X86ISD::SBB";
1794  case X86ISD::SHLD:               return "X86ISD::SHLD";
1795  case X86ISD::SHRD:               return "X86ISD::SHRD";
1796  case X86ISD::FILD:               return "X86ISD::FILD";
1797  case X86ISD::FP_TO_INT16_IN_MEM: return "X86ISD::FP_TO_INT16_IN_MEM";
1798  case X86ISD::FP_TO_INT32_IN_MEM: return "X86ISD::FP_TO_INT32_IN_MEM";
1799  case X86ISD::FP_TO_INT64_IN_MEM: return "X86ISD::FP_TO_INT64_IN_MEM";
1800  case X86ISD::FLD:                return "X86ISD::FLD";
1801  case X86ISD::FST:                return "X86ISD::FST";
1802  case X86ISD::FP_GET_RESULT:      return "X86ISD::FP_GET_RESULT";
1803  case X86ISD::FP_SET_RESULT:      return "X86ISD::FP_SET_RESULT";
1804  case X86ISD::CALL:               return "X86ISD::CALL";
1805  case X86ISD::TAILCALL:           return "X86ISD::TAILCALL";
1806  case X86ISD::RDTSC_DAG:          return "X86ISD::RDTSC_DAG";
1807  case X86ISD::CMP:                return "X86ISD::CMP";
1808  case X86ISD::TEST:               return "X86ISD::TEST";
1809  case X86ISD::SETCC:              return "X86ISD::SETCC";
1810  case X86ISD::CMOV:               return "X86ISD::CMOV";
1811  case X86ISD::BRCOND:             return "X86ISD::BRCOND";
1812  case X86ISD::RET_FLAG:           return "X86ISD::RET_FLAG";
1813  case X86ISD::REP_STOS:           return "X86ISD::RET_STOS";
1814  case X86ISD::REP_MOVS:           return "X86ISD::RET_MOVS";
1815  }
1816}
1817
1818bool X86TargetLowering::isMaskedValueZeroForTargetNode(const SDOperand &Op,
1819                                                       uint64_t Mask) const {
1820
1821  unsigned Opc = Op.getOpcode();
1822
1823  switch (Opc) {
1824  default:
1825    assert(Opc >= ISD::BUILTIN_OP_END && "Expected a target specific node");
1826    break;
1827  case X86ISD::SETCC: return (Mask & 1) == 0;
1828  }
1829
1830  return false;
1831}
1832