X86ISelLowering.cpp revision 002fe9baf2bdc50532c6238b92ac46191fd01c38 
1//===-- X86ISelLowering.h - X86 DAG Lowering Interface ----------*- C++ -*-===//
2//
3//                     The LLVM Compiler Infrastructure
4//
5// This file was developed by Chris Lattner and is distributed under
6// the University of Illinois Open Source License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10// This file defines the interfaces that X86 uses to lower LLVM code into a
11// selection DAG.
12//
13//===----------------------------------------------------------------------===//
14
15#include "X86.h"
16#include "X86ISelLowering.h"
17#include "X86TargetMachine.h"
18#include "llvm/CallingConv.h"
19#include "llvm/Function.h"
20#include "llvm/CodeGen/MachineFrameInfo.h"
21#include "llvm/CodeGen/MachineFunction.h"
22#include "llvm/CodeGen/MachineInstrBuilder.h"
23#include "llvm/CodeGen/SelectionDAG.h"
24#include "llvm/CodeGen/SSARegMap.h"
25#include "llvm/Target/TargetOptions.h"
26using namespace llvm;
27
28// FIXME: temporary.
29#include "llvm/Support/CommandLine.h"
30static cl::opt<bool> EnableFastCC("enable-x86-fastcc", cl::Hidden,
31                                  cl::desc("Enable fastcc on X86"));
32
33X86TargetLowering::X86TargetLowering(TargetMachine &TM)
34  : TargetLowering(TM) {
35  // Set up the TargetLowering object.
36
37  // X86 is weird, it always uses i8 for shift amounts and setcc results.
38  setShiftAmountType(MVT::i8);
39  setSetCCResultType(MVT::i8);
40  setSetCCResultContents(ZeroOrOneSetCCResult);
41  setShiftAmountFlavor(Mask);   // shl X, 32 == shl X, 0
42
43  // Set up the register classes.
44  addRegisterClass(MVT::i8, X86::R8RegisterClass);
45  addRegisterClass(MVT::i16, X86::R16RegisterClass);
46  addRegisterClass(MVT::i32, X86::R32RegisterClass);
47
48  // Promote all UINT_TO_FP to larger SINT_TO_FP's, as X86 doesn't have this
49  // operation.
50  setOperationAction(ISD::UINT_TO_FP       , MVT::i1   , Promote);
51  setOperationAction(ISD::UINT_TO_FP       , MVT::i8   , Promote);
52  setOperationAction(ISD::UINT_TO_FP       , MVT::i16  , Promote);
53  setOperationAction(ISD::UINT_TO_FP       , MVT::i32  , Promote);
54
55  // Promote i1/i8 SINT_TO_FP to larger SINT_TO_FP's, as X86 doesn't have
56  // this operation.
57  setOperationAction(ISD::SINT_TO_FP       , MVT::i1   , Promote);
58  setOperationAction(ISD::SINT_TO_FP       , MVT::i8   , Promote);
59
60  if (!X86ScalarSSE) {
61    // We can handle SINT_TO_FP and FP_TO_SINT from/TO i64 even though i64
62    // isn't legal.
63    setOperationAction(ISD::SINT_TO_FP     , MVT::i64  , Custom);
64    setOperationAction(ISD::FP_TO_SINT     , MVT::i64  , Custom);
65    setOperationAction(ISD::FP_TO_SINT     , MVT::i32  , Custom);
66    setOperationAction(ISD::FP_TO_SINT     , MVT::i16  , Custom);
67  }
68
69  // Handle FP_TO_UINT by promoting the destination to a larger signed
70  // conversion.
71  setOperationAction(ISD::FP_TO_UINT       , MVT::i1   , Promote);
72  setOperationAction(ISD::FP_TO_UINT       , MVT::i8   , Promote);
73  setOperationAction(ISD::FP_TO_UINT       , MVT::i16  , Promote);
74
75  if (!X86ScalarSSE)
76    setOperationAction(ISD::FP_TO_UINT     , MVT::i32  , Promote);
77
78  // Promote i1/i8 FP_TO_SINT to larger FP_TO_SINTS's, as X86 doesn't have
79  // this operation.
80  setOperationAction(ISD::FP_TO_SINT       , MVT::i1   , Promote);
81  setOperationAction(ISD::FP_TO_SINT       , MVT::i8   , Promote);
82  setOperationAction(ISD::FP_TO_SINT       , MVT::i16  , Promote);
83
84  setOperationAction(ISD::BIT_CONVERT, MVT::f32, Expand);
85  setOperationAction(ISD::BIT_CONVERT, MVT::i32, Expand);
86
87  if (X86DAGIsel) {
88    setOperationAction(ISD::BRCOND         , MVT::Other, Custom);
89  }
90  setOperationAction(ISD::BRCONDTWOWAY     , MVT::Other, Expand);
91  setOperationAction(ISD::BRTWOWAY_CC      , MVT::Other, Expand);
92  setOperationAction(ISD::MEMMOVE          , MVT::Other, Expand);
93  setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i16  , Expand);
94  setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i8   , Expand);
95  setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i1   , Expand);
96  setOperationAction(ISD::FP_ROUND_INREG   , MVT::f32  , Expand);
97  setOperationAction(ISD::SEXTLOAD         , MVT::i1   , Expand);
98  setOperationAction(ISD::FREM             , MVT::f64  , Expand);
99  setOperationAction(ISD::CTPOP            , MVT::i8   , Expand);
100  setOperationAction(ISD::CTTZ             , MVT::i8   , Expand);
101  setOperationAction(ISD::CTLZ             , MVT::i8   , Expand);
102  setOperationAction(ISD::CTPOP            , MVT::i16  , Expand);
103  setOperationAction(ISD::CTTZ             , MVT::i16  , Expand);
104  setOperationAction(ISD::CTLZ             , MVT::i16  , Expand);
105  setOperationAction(ISD::CTPOP            , MVT::i32  , Expand);
106  setOperationAction(ISD::CTTZ             , MVT::i32  , Expand);
107  setOperationAction(ISD::CTLZ             , MVT::i32  , Expand);
108  setOperationAction(ISD::READCYCLECOUNTER , MVT::i64  , Custom);
109
110  if (!X86DAGIsel) {
111    setOperationAction(ISD::ROTL           , MVT::i8   , Expand);
112    setOperationAction(ISD::ROTR           , MVT::i8   , Expand);
113    setOperationAction(ISD::ROTL           , MVT::i16  , Expand);
114    setOperationAction(ISD::ROTR           , MVT::i16  , Expand);
115    setOperationAction(ISD::ROTL           , MVT::i32  , Expand);
116    setOperationAction(ISD::ROTR           , MVT::i32  , Expand);
117  }
118
119  setOperationAction(ISD::READIO           , MVT::i1   , Expand);
120  setOperationAction(ISD::READIO           , MVT::i8   , Expand);
121  setOperationAction(ISD::READIO           , MVT::i16  , Expand);
122  setOperationAction(ISD::READIO           , MVT::i32  , Expand);
123  setOperationAction(ISD::WRITEIO          , MVT::i1   , Expand);
124  setOperationAction(ISD::WRITEIO          , MVT::i8   , Expand);
125  setOperationAction(ISD::WRITEIO          , MVT::i16  , Expand);
126  setOperationAction(ISD::WRITEIO          , MVT::i32  , Expand);
127
128  // These should be promoted to a larger select which is supported.
129  setOperationAction(ISD::SELECT           , MVT::i1   , Promote);
130  setOperationAction(ISD::SELECT           , MVT::i8   , Promote);
131  if (X86DAGIsel) {
132    // X86 wants to expand cmov itself.
133    setOperationAction(ISD::SELECT         , MVT::i16  , Custom);
134    setOperationAction(ISD::SELECT         , MVT::i32  , Custom);
135    setOperationAction(ISD::SELECT         , MVT::f32  , Custom);
136    setOperationAction(ISD::SELECT         , MVT::f64  , Custom);
137    setOperationAction(ISD::SETCC          , MVT::i8   , Custom);
138    setOperationAction(ISD::SETCC          , MVT::i16  , Custom);
139    setOperationAction(ISD::SETCC          , MVT::i32  , Custom);
140    setOperationAction(ISD::SETCC          , MVT::f32  , Custom);
141    setOperationAction(ISD::SETCC          , MVT::f64  , Custom);
142    // X86 ret instruction may pop stack.
143    setOperationAction(ISD::RET            , MVT::Other, Custom);
144    // Darwin ABI issue.
145    setOperationAction(ISD::GlobalAddress  , MVT::i32  , Custom);
146    // 64-bit addm sub, shl, sra, srl (iff 32-bit x86)
147    setOperationAction(ISD::ADD_PARTS      , MVT::i32  , Custom);
148    setOperationAction(ISD::SUB_PARTS      , MVT::i32  , Custom);
149    setOperationAction(ISD::SHL_PARTS      , MVT::i32  , Custom);
150    setOperationAction(ISD::SRA_PARTS      , MVT::i32  , Custom);
151    setOperationAction(ISD::SRL_PARTS      , MVT::i32  , Custom);
152    // X86 wants to expand memset / memcpy itself.
153    setOperationAction(ISD::MEMSET         , MVT::Other, Custom);
154    setOperationAction(ISD::MEMCPY         , MVT::Other, Custom);
155  }
156
157  // We don't have line number support yet.
158  setOperationAction(ISD::LOCATION, MVT::Other, Expand);
159  setOperationAction(ISD::DEBUG_LOC, MVT::Other, Expand);
160  setOperationAction(ISD::DEBUG_LABEL, MVT::Other, Expand);
161
162  if (X86ScalarSSE) {
163    // Set up the FP register classes.
164    addRegisterClass(MVT::f32, X86::V4F4RegisterClass);
165    addRegisterClass(MVT::f64, X86::V2F8RegisterClass);
166
167    // SSE has no load+extend ops
168    setOperationAction(ISD::EXTLOAD,  MVT::f32, Expand);
169    setOperationAction(ISD::ZEXTLOAD, MVT::f32, Expand);
170
171    // SSE has no i16 to fp conversion, only i32
172    setOperationAction(ISD::SINT_TO_FP, MVT::i16, Promote);
173    setOperationAction(ISD::FP_TO_SINT, MVT::i16, Promote);
174
175    // Expand FP_TO_UINT into a select.
176    // FIXME: We would like to use a Custom expander here eventually to do
177    // the optimal thing for SSE vs. the default expansion in the legalizer.
178    setOperationAction(ISD::FP_TO_UINT       , MVT::i32  , Expand);
179
180    // We don't support sin/cos/sqrt/fmod
181    setOperationAction(ISD::FSIN , MVT::f64, Expand);
182    setOperationAction(ISD::FCOS , MVT::f64, Expand);
183    setOperationAction(ISD::FABS , MVT::f64, Expand);
184    setOperationAction(ISD::FNEG , MVT::f64, Expand);
185    setOperationAction(ISD::FREM , MVT::f64, Expand);
186    setOperationAction(ISD::FSIN , MVT::f32, Expand);
187    setOperationAction(ISD::FCOS , MVT::f32, Expand);
188    setOperationAction(ISD::FABS , MVT::f32, Expand);
189    setOperationAction(ISD::FNEG , MVT::f32, Expand);
190    setOperationAction(ISD::FREM , MVT::f32, Expand);
191
192    addLegalFPImmediate(+0.0); // xorps / xorpd
193  } else {
194    // Set up the FP register classes.
195    addRegisterClass(MVT::f64, X86::RFPRegisterClass);
196
197    if (!UnsafeFPMath) {
198      setOperationAction(ISD::FSIN           , MVT::f64  , Expand);
199      setOperationAction(ISD::FCOS           , MVT::f64  , Expand);
200    }
201
202    addLegalFPImmediate(+0.0); // FLD0
203    addLegalFPImmediate(+1.0); // FLD1
204    addLegalFPImmediate(-0.0); // FLD0/FCHS
205    addLegalFPImmediate(-1.0); // FLD1/FCHS
206  }
207  computeRegisterProperties();
208
209  maxStoresPerMemSet = 8; // For %llvm.memset -> sequence of stores
210  maxStoresPerMemCpy = 8; // For %llvm.memcpy -> sequence of stores
211  maxStoresPerMemMove = 8; // For %llvm.memmove -> sequence of stores
212  allowUnalignedMemoryAccesses = true; // x86 supports it!
213}
214
215std::vector<SDOperand>
216X86TargetLowering::LowerArguments(Function &F, SelectionDAG &DAG) {
217  if (F.getCallingConv() == CallingConv::Fast && EnableFastCC)
218    return LowerFastCCArguments(F, DAG);
219  return LowerCCCArguments(F, DAG);
220}
221
222std::pair<SDOperand, SDOperand>
223X86TargetLowering::LowerCallTo(SDOperand Chain, const Type *RetTy,
224                               bool isVarArg, unsigned CallingConv,
225                               bool isTailCall,
226                               SDOperand Callee, ArgListTy &Args,
227                               SelectionDAG &DAG) {
228  assert((!isVarArg || CallingConv == CallingConv::C) &&
229         "Only C takes varargs!");
230
231  // If the callee is a GlobalAddress node (quite common, every direct call is)
232  // turn it into a TargetGlobalAddress node so that legalize doesn't hack it.
233  if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee))
234    Callee = DAG.getTargetGlobalAddress(G->getGlobal(), getPointerTy());
235  else if (ExternalSymbolSDNode *S = dyn_cast<ExternalSymbolSDNode>(Callee))
236    Callee = DAG.getTargetExternalSymbol(S->getSymbol(), getPointerTy());
237
238  if (CallingConv == CallingConv::Fast && EnableFastCC)
239    return LowerFastCCCallTo(Chain, RetTy, isTailCall, Callee, Args, DAG);
240  return  LowerCCCCallTo(Chain, RetTy, isVarArg, isTailCall, Callee, Args, DAG);
241}
242
243SDOperand X86TargetLowering::LowerReturnTo(SDOperand Chain, SDOperand Op,
244                                           SelectionDAG &DAG) {
245  if (!X86DAGIsel)
246    return DAG.getNode(ISD::RET, MVT::Other, Chain, Op);
247
248  SDOperand Copy;
249  MVT::ValueType OpVT = Op.getValueType();
250  switch (OpVT) {
251    default: assert(0 && "Unknown type to return!");
252    case MVT::i32:
253      Copy = DAG.getCopyToReg(Chain, X86::EAX, Op, SDOperand());
254      break;
255    case MVT::i64: {
256      SDOperand Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, MVT::i32, Op,
257                                 DAG.getConstant(1, MVT::i32));
258      SDOperand Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, MVT::i32, Op,
259                                 DAG.getConstant(0, MVT::i32));
260      Copy = DAG.getCopyToReg(Chain, X86::EDX, Hi, SDOperand());
261      Copy = DAG.getCopyToReg(Copy,  X86::EAX, Lo, Copy.getValue(1));
262      break;
263    }
264    case MVT::f32:
265    case MVT::f64:
266      if (!X86ScalarSSE) {
267        if (OpVT == MVT::f32)
268          Op = DAG.getNode(ISD::FP_EXTEND, MVT::f64, Op);
269        std::vector<MVT::ValueType> Tys;
270        Tys.push_back(MVT::Other);
271        Tys.push_back(MVT::Flag);
272        std::vector<SDOperand> Ops;
273        Ops.push_back(Chain);
274        Ops.push_back(Op);
275        Copy = DAG.getNode(X86ISD::FP_SET_RESULT, Tys, Ops);
276      } else {
277        // Spill the value to memory and reload it into top of stack.
278        unsigned Size = MVT::getSizeInBits(OpVT)/8;
279        MachineFunction &MF = DAG.getMachineFunction();
280        int SSFI = MF.getFrameInfo()->CreateStackObject(Size, Size);
281        SDOperand StackSlot = DAG.getFrameIndex(SSFI, getPointerTy());
282        Chain = DAG.getNode(ISD::STORE, MVT::Other, Chain, Op,
283                            StackSlot, DAG.getSrcValue(NULL));
284        std::vector<MVT::ValueType> Tys;
285        Tys.push_back(MVT::f64);
286        Tys.push_back(MVT::Other);
287        std::vector<SDOperand> Ops;
288        Ops.push_back(Chain);
289        Ops.push_back(StackSlot);
290        Ops.push_back(DAG.getValueType(OpVT));
291        Copy = DAG.getNode(X86ISD::FLD, Tys, Ops);
292        Tys.clear();
293        Tys.push_back(MVT::Other);
294        Tys.push_back(MVT::Flag);
295        Ops.clear();
296        Ops.push_back(Copy.getValue(1));
297        Ops.push_back(Copy);
298        Copy = DAG.getNode(X86ISD::FP_SET_RESULT, Tys, Ops);
299      }
300      break;
301  }
302
303  return DAG.getNode(X86ISD::RET_FLAG, MVT::Other,
304                     Copy, DAG.getConstant(getBytesToPopOnReturn(), MVT::i16),
305                     Copy.getValue(1));
306}
307
308//===----------------------------------------------------------------------===//
309//                    C Calling Convention implementation
310//===----------------------------------------------------------------------===//
311
312std::vector<SDOperand>
313X86TargetLowering::LowerCCCArguments(Function &F, SelectionDAG &DAG) {
314  std::vector<SDOperand> ArgValues;
315
316  MachineFunction &MF = DAG.getMachineFunction();
317  MachineFrameInfo *MFI = MF.getFrameInfo();
318
319  // Add DAG nodes to load the arguments...  On entry to a function on the X86,
320  // the stack frame looks like this:
321  //
322  // [ESP] -- return address
323  // [ESP + 4] -- first argument (leftmost lexically)
324  // [ESP + 8] -- second argument, if first argument is four bytes in size
325  //    ...
326  //
327  unsigned ArgOffset = 0;   // Frame mechanisms handle retaddr slot
328  for (Function::arg_iterator I = F.arg_begin(), E = F.arg_end(); I != E; ++I) {
329    MVT::ValueType ObjectVT = getValueType(I->getType());
330    unsigned ArgIncrement = 4;
331    unsigned ObjSize;
332    switch (ObjectVT) {
333    default: assert(0 && "Unhandled argument type!");
334    case MVT::i1:
335    case MVT::i8:  ObjSize = 1;                break;
336    case MVT::i16: ObjSize = 2;                break;
337    case MVT::i32: ObjSize = 4;                break;
338    case MVT::i64: ObjSize = ArgIncrement = 8; break;
339    case MVT::f32: ObjSize = 4;                break;
340    case MVT::f64: ObjSize = ArgIncrement = 8; break;
341    }
342    // Create the frame index object for this incoming parameter...
343    int FI = MFI->CreateFixedObject(ObjSize, ArgOffset);
344
345    // Create the SelectionDAG nodes corresponding to a load from this parameter
346    SDOperand FIN = DAG.getFrameIndex(FI, MVT::i32);
347
348    // Don't codegen dead arguments.  FIXME: remove this check when we can nuke
349    // dead loads.
350    SDOperand ArgValue;
351    if (!I->use_empty())
352      ArgValue = DAG.getLoad(ObjectVT, DAG.getEntryNode(), FIN,
353                             DAG.getSrcValue(NULL));
354    else {
355      if (MVT::isInteger(ObjectVT))
356        ArgValue = DAG.getConstant(0, ObjectVT);
357      else
358        ArgValue = DAG.getConstantFP(0, ObjectVT);
359    }
360    ArgValues.push_back(ArgValue);
361
362    ArgOffset += ArgIncrement;   // Move on to the next argument...
363  }
364
365  // If the function takes variable number of arguments, make a frame index for
366  // the start of the first vararg value... for expansion of llvm.va_start.
367  if (F.isVarArg())
368    VarArgsFrameIndex = MFI->CreateFixedObject(1, ArgOffset);
369  ReturnAddrIndex = 0;     // No return address slot generated yet.
370  BytesToPopOnReturn = 0;  // Callee pops nothing.
371  BytesCallerReserves = ArgOffset;
372
373  // Finally, inform the code generator which regs we return values in.
374  switch (getValueType(F.getReturnType())) {
375  default: assert(0 && "Unknown type!");
376  case MVT::isVoid: break;
377  case MVT::i1:
378  case MVT::i8:
379  case MVT::i16:
380  case MVT::i32:
381    MF.addLiveOut(X86::EAX);
382    break;
383  case MVT::i64:
384    MF.addLiveOut(X86::EAX);
385    MF.addLiveOut(X86::EDX);
386    break;
387  case MVT::f32:
388  case MVT::f64:
389    MF.addLiveOut(X86::ST0);
390    break;
391  }
392  return ArgValues;
393}
394
395std::pair<SDOperand, SDOperand>
396X86TargetLowering::LowerCCCCallTo(SDOperand Chain, const Type *RetTy,
397                                  bool isVarArg, bool isTailCall,
398                                  SDOperand Callee, ArgListTy &Args,
399                                  SelectionDAG &DAG) {
400  // Count how many bytes are to be pushed on the stack.
401  unsigned NumBytes = 0;
402
403  if (Args.empty()) {
404    // Save zero bytes.
405    Chain = DAG.getNode(ISD::CALLSEQ_START, MVT::Other, Chain,
406                        DAG.getConstant(0, getPointerTy()));
407  } else {
408    for (unsigned i = 0, e = Args.size(); i != e; ++i)
409      switch (getValueType(Args[i].second)) {
410      default: assert(0 && "Unknown value type!");
411      case MVT::i1:
412      case MVT::i8:
413      case MVT::i16:
414      case MVT::i32:
415      case MVT::f32:
416        NumBytes += 4;
417        break;
418      case MVT::i64:
419      case MVT::f64:
420        NumBytes += 8;
421        break;
422      }
423
424    Chain = DAG.getNode(ISD::CALLSEQ_START, MVT::Other, Chain,
425                        DAG.getConstant(NumBytes, getPointerTy()));
426
427    // Arguments go on the stack in reverse order, as specified by the ABI.
428    unsigned ArgOffset = 0;
429    SDOperand StackPtr = DAG.getRegister(X86::ESP, MVT::i32);
430    std::vector<SDOperand> Stores;
431
432    for (unsigned i = 0, e = Args.size(); i != e; ++i) {
433      SDOperand PtrOff = DAG.getConstant(ArgOffset, getPointerTy());
434      PtrOff = DAG.getNode(ISD::ADD, MVT::i32, StackPtr, PtrOff);
435
436      switch (getValueType(Args[i].second)) {
437      default: assert(0 && "Unexpected ValueType for argument!");
438      case MVT::i1:
439      case MVT::i8:
440      case MVT::i16:
441        // Promote the integer to 32 bits.  If the input type is signed use a
442        // sign extend, otherwise use a zero extend.
443        if (Args[i].second->isSigned())
444          Args[i].first =DAG.getNode(ISD::SIGN_EXTEND, MVT::i32, Args[i].first);
445        else
446          Args[i].first =DAG.getNode(ISD::ZERO_EXTEND, MVT::i32, Args[i].first);
447
448        // FALL THROUGH
449      case MVT::i32:
450      case MVT::f32:
451        Stores.push_back(DAG.getNode(ISD::STORE, MVT::Other, Chain,
452                                     Args[i].first, PtrOff,
453                                     DAG.getSrcValue(NULL)));
454        ArgOffset += 4;
455        break;
456      case MVT::i64:
457      case MVT::f64:
458        Stores.push_back(DAG.getNode(ISD::STORE, MVT::Other, Chain,
459                                     Args[i].first, PtrOff,
460                                     DAG.getSrcValue(NULL)));
461        ArgOffset += 8;
462        break;
463      }
464    }
465    Chain = DAG.getNode(ISD::TokenFactor, MVT::Other, Stores);
466  }
467
468  std::vector<MVT::ValueType> RetVals;
469  MVT::ValueType RetTyVT = getValueType(RetTy);
470  RetVals.push_back(MVT::Other);
471
472  // The result values produced have to be legal.  Promote the result.
473  switch (RetTyVT) {
474  case MVT::isVoid: break;
475  default:
476    RetVals.push_back(RetTyVT);
477    break;
478  case MVT::i1:
479  case MVT::i8:
480  case MVT::i16:
481    RetVals.push_back(MVT::i32);
482    break;
483  case MVT::f32:
484    if (X86ScalarSSE)
485      RetVals.push_back(MVT::f32);
486    else
487      RetVals.push_back(MVT::f64);
488    break;
489  case MVT::i64:
490    RetVals.push_back(MVT::i32);
491    RetVals.push_back(MVT::i32);
492    break;
493  }
494
495  if (X86DAGIsel) {
496    std::vector<MVT::ValueType> NodeTys;
497    NodeTys.push_back(MVT::Other);   // Returns a chain
498    NodeTys.push_back(MVT::Flag);    // Returns a flag for retval copy to use.
499    std::vector<SDOperand> Ops;
500    Ops.push_back(Chain);
501    Ops.push_back(Callee);
502
503    // FIXME: Do not generate X86ISD::TAILCALL for now.
504    Chain = DAG.getNode(X86ISD::CALL, NodeTys, Ops);
505    SDOperand InFlag = Chain.getValue(1);
506
507    SDOperand RetVal;
508    if (RetTyVT != MVT::isVoid) {
509      switch (RetTyVT) {
510      default: assert(0 && "Unknown value type to return!");
511      case MVT::i1:
512      case MVT::i8:
513        RetVal = DAG.getCopyFromReg(Chain, X86::AL, MVT::i8, InFlag);
514        Chain = RetVal.getValue(1);
515        break;
516      case MVT::i16:
517        RetVal = DAG.getCopyFromReg(Chain, X86::AX, MVT::i16, InFlag);
518        Chain = RetVal.getValue(1);
519        break;
520      case MVT::i32:
521        RetVal = DAG.getCopyFromReg(Chain, X86::EAX, MVT::i32, InFlag);
522        Chain = RetVal.getValue(1);
523        break;
524      case MVT::i64: {
525        SDOperand Lo = DAG.getCopyFromReg(Chain, X86::EAX, MVT::i32, InFlag);
526        SDOperand Hi = DAG.getCopyFromReg(Lo.getValue(1), X86::EDX, MVT::i32,
527                                          Lo.getValue(2));
528        RetVal = DAG.getNode(ISD::BUILD_PAIR, MVT::i64, Lo, Hi);
529        Chain = Hi.getValue(1);
530        break;
531      }
532      case MVT::f32:
533      case MVT::f64: {
534        std::vector<MVT::ValueType> Tys;
535        Tys.push_back(MVT::f64);
536        Tys.push_back(MVT::Other);
537        std::vector<SDOperand> Ops;
538        Ops.push_back(Chain);
539        Ops.push_back(InFlag);
540        RetVal = DAG.getNode(X86ISD::FP_GET_RESULT, Tys, Ops);
541        Chain = RetVal.getValue(1);
542        if (X86ScalarSSE) {
543          unsigned Size = MVT::getSizeInBits(MVT::f64)/8;
544          MachineFunction &MF = DAG.getMachineFunction();
545          int SSFI = MF.getFrameInfo()->CreateStackObject(Size, Size);
546          SDOperand StackSlot = DAG.getFrameIndex(SSFI, getPointerTy());
547          Tys.clear();
548          Tys.push_back(MVT::Other);
549          Ops.clear();
550          Ops.push_back(Chain);
551          Ops.push_back(RetVal);
552          Ops.push_back(StackSlot);
553          Ops.push_back(DAG.getValueType(RetTyVT));
554          Chain = DAG.getNode(X86ISD::FST, Tys, Ops);
555          RetVal = DAG.getLoad(RetTyVT, Chain, StackSlot,
556                               DAG.getSrcValue(NULL));
557          Chain = RetVal.getValue(1);
558        } else if (RetTyVT == MVT::f32)
559          RetVal = DAG.getNode(ISD::FP_ROUND, MVT::f32, RetVal);
560        break;
561      }
562      }
563    }
564
565    Chain = DAG.getNode(ISD::CALLSEQ_END, MVT::Other, Chain,
566                        DAG.getConstant(NumBytes, getPointerTy()),
567                        DAG.getConstant(0, getPointerTy()));
568    return std::make_pair(RetVal, Chain);
569  } else {
570    std::vector<SDOperand> Ops;
571    Ops.push_back(Chain);
572    Ops.push_back(Callee);
573    Ops.push_back(DAG.getConstant(NumBytes, getPointerTy()));
574    Ops.push_back(DAG.getConstant(0, getPointerTy()));
575
576    SDOperand TheCall = DAG.getNode(isTailCall ? X86ISD::TAILCALL : X86ISD::CALL,
577                                    RetVals, Ops);
578
579    SDOperand ResultVal;
580    switch (RetTyVT) {
581    case MVT::isVoid: break;
582    default:
583      ResultVal = TheCall.getValue(1);
584      break;
585    case MVT::i1:
586    case MVT::i8:
587    case MVT::i16:
588      ResultVal = DAG.getNode(ISD::TRUNCATE, RetTyVT, TheCall.getValue(1));
589      break;
590    case MVT::f32:
591      // FIXME: we would really like to remember that this FP_ROUND operation is
592      // okay to eliminate if we allow excess FP precision.
593      ResultVal = DAG.getNode(ISD::FP_ROUND, MVT::f32, TheCall.getValue(1));
594      break;
595    case MVT::i64:
596      ResultVal = DAG.getNode(ISD::BUILD_PAIR, MVT::i64, TheCall.getValue(1),
597                              TheCall.getValue(2));
598      break;
599    }
600
601    Chain = DAG.getNode(ISD::CALLSEQ_END, MVT::Other, TheCall);
602    return std::make_pair(ResultVal, Chain);
603  }
604}
605
606SDOperand
607X86TargetLowering::LowerVAStart(SDOperand Chain, SDOperand VAListP,
608                                Value *VAListV, SelectionDAG &DAG) {
609  // vastart just stores the address of the VarArgsFrameIndex slot.
610  SDOperand FR = DAG.getFrameIndex(VarArgsFrameIndex, MVT::i32);
611  return DAG.getNode(ISD::STORE, MVT::Other, Chain, FR, VAListP,
612                     DAG.getSrcValue(VAListV));
613}
614
615
616std::pair<SDOperand,SDOperand>
617X86TargetLowering::LowerVAArg(SDOperand Chain, SDOperand VAListP,
618                              Value *VAListV, const Type *ArgTy,
619                              SelectionDAG &DAG) {
620  MVT::ValueType ArgVT = getValueType(ArgTy);
621  SDOperand Val = DAG.getLoad(MVT::i32, Chain,
622                              VAListP, DAG.getSrcValue(VAListV));
623  SDOperand Result = DAG.getLoad(ArgVT, Chain, Val,
624                                 DAG.getSrcValue(NULL));
625  unsigned Amt;
626  if (ArgVT == MVT::i32)
627    Amt = 4;
628  else {
629    assert((ArgVT == MVT::i64 || ArgVT == MVT::f64) &&
630           "Other types should have been promoted for varargs!");
631    Amt = 8;
632  }
633  Val = DAG.getNode(ISD::ADD, Val.getValueType(), Val,
634                    DAG.getConstant(Amt, Val.getValueType()));
635  Chain = DAG.getNode(ISD::STORE, MVT::Other, Chain,
636                      Val, VAListP, DAG.getSrcValue(VAListV));
637  return std::make_pair(Result, Chain);
638}
639
640//===----------------------------------------------------------------------===//
641//                    Fast Calling Convention implementation
642//===----------------------------------------------------------------------===//
643//
644// The X86 'fast' calling convention passes up to two integer arguments in
645// registers (an appropriate portion of EAX/EDX), passes arguments in C order,
646// and requires that the callee pop its arguments off the stack (allowing proper
647// tail calls), and has the same return value conventions as C calling convs.
648//
649// This calling convention always arranges for the callee pop value to be 8n+4
650// bytes, which is needed for tail recursion elimination and stack alignment
651// reasons.
652//
653// Note that this can be enhanced in the future to pass fp vals in registers
654// (when we have a global fp allocator) and do other tricks.
655//
656
657/// AddLiveIn - This helper function adds the specified physical register to the
658/// MachineFunction as a live in value.  It also creates a corresponding virtual
659/// register for it.
660static unsigned AddLiveIn(MachineFunction &MF, unsigned PReg,
661                          TargetRegisterClass *RC) {
662  assert(RC->contains(PReg) && "Not the correct regclass!");
663  unsigned VReg = MF.getSSARegMap()->createVirtualRegister(RC);
664  MF.addLiveIn(PReg, VReg);
665  return VReg;
666}
667
668
669std::vector<SDOperand>
670X86TargetLowering::LowerFastCCArguments(Function &F, SelectionDAG &DAG) {
671  std::vector<SDOperand> ArgValues;
672
673  MachineFunction &MF = DAG.getMachineFunction();
674  MachineFrameInfo *MFI = MF.getFrameInfo();
675
676  // Add DAG nodes to load the arguments...  On entry to a function the stack
677  // frame looks like this:
678  //
679  // [ESP] -- return address
680  // [ESP + 4] -- first nonreg argument (leftmost lexically)
681  // [ESP + 8] -- second nonreg argument, if first argument is 4 bytes in size
682  //    ...
683  unsigned ArgOffset = 0;   // Frame mechanisms handle retaddr slot
684
685  // Keep track of the number of integer regs passed so far.  This can be either
686  // 0 (neither EAX or EDX used), 1 (EAX is used) or 2 (EAX and EDX are both
687  // used).
688  unsigned NumIntRegs = 0;
689
690  for (Function::arg_iterator I = F.arg_begin(), E = F.arg_end(); I != E; ++I) {
691    MVT::ValueType ObjectVT = getValueType(I->getType());
692    unsigned ArgIncrement = 4;
693    unsigned ObjSize = 0;
694    SDOperand ArgValue;
695
696    switch (ObjectVT) {
697    default: assert(0 && "Unhandled argument type!");
698    case MVT::i1:
699    case MVT::i8:
700      if (NumIntRegs < 2) {
701        if (!I->use_empty()) {
702          unsigned VReg = AddLiveIn(MF, NumIntRegs ? X86::DL : X86::AL,
703                                    X86::R8RegisterClass);
704          ArgValue = DAG.getCopyFromReg(DAG.getRoot(), VReg, MVT::i8);
705          DAG.setRoot(ArgValue.getValue(1));
706          if (ObjectVT == MVT::i1)
707            // FIXME: Should insert a assertzext here.
708            ArgValue = DAG.getNode(ISD::TRUNCATE, MVT::i1, ArgValue);
709        }
710        ++NumIntRegs;
711        break;
712      }
713
714      ObjSize = 1;
715      break;
716    case MVT::i16:
717      if (NumIntRegs < 2) {
718        if (!I->use_empty()) {
719          unsigned VReg = AddLiveIn(MF, NumIntRegs ? X86::DX : X86::AX,
720                                    X86::R16RegisterClass);
721          ArgValue = DAG.getCopyFromReg(DAG.getRoot(), VReg, MVT::i16);
722          DAG.setRoot(ArgValue.getValue(1));
723        }
724        ++NumIntRegs;
725        break;
726      }
727      ObjSize = 2;
728      break;
729    case MVT::i32:
730      if (NumIntRegs < 2) {
731        if (!I->use_empty()) {
732          unsigned VReg = AddLiveIn(MF,NumIntRegs ? X86::EDX : X86::EAX,
733                                    X86::R32RegisterClass);
734          ArgValue = DAG.getCopyFromReg(DAG.getRoot(), VReg, MVT::i32);
735          DAG.setRoot(ArgValue.getValue(1));
736        }
737        ++NumIntRegs;
738        break;
739      }
740      ObjSize = 4;
741      break;
742    case MVT::i64:
743      if (NumIntRegs == 0) {
744        if (!I->use_empty()) {
745          unsigned BotReg = AddLiveIn(MF, X86::EAX, X86::R32RegisterClass);
746          unsigned TopReg = AddLiveIn(MF, X86::EDX, X86::R32RegisterClass);
747
748          SDOperand Low = DAG.getCopyFromReg(DAG.getRoot(), BotReg, MVT::i32);
749          SDOperand Hi  = DAG.getCopyFromReg(Low.getValue(1), TopReg, MVT::i32);
750          DAG.setRoot(Hi.getValue(1));
751
752          ArgValue = DAG.getNode(ISD::BUILD_PAIR, MVT::i64, Low, Hi);
753        }
754        NumIntRegs = 2;
755        break;
756      } else if (NumIntRegs == 1) {
757        if (!I->use_empty()) {
758          unsigned BotReg = AddLiveIn(MF, X86::EDX, X86::R32RegisterClass);
759          SDOperand Low = DAG.getCopyFromReg(DAG.getRoot(), BotReg, MVT::i32);
760          DAG.setRoot(Low.getValue(1));
761
762          // Load the high part from memory.
763          // Create the frame index object for this incoming parameter...
764          int FI = MFI->CreateFixedObject(4, ArgOffset);
765          SDOperand FIN = DAG.getFrameIndex(FI, MVT::i32);
766          SDOperand Hi = DAG.getLoad(MVT::i32, DAG.getEntryNode(), FIN,
767                                     DAG.getSrcValue(NULL));
768          ArgValue = DAG.getNode(ISD::BUILD_PAIR, MVT::i64, Low, Hi);
769        }
770        ArgOffset += 4;
771        NumIntRegs = 2;
772        break;
773      }
774      ObjSize = ArgIncrement = 8;
775      break;
776    case MVT::f32: ObjSize = 4;                break;
777    case MVT::f64: ObjSize = ArgIncrement = 8; break;
778    }
779
780    // Don't codegen dead arguments.  FIXME: remove this check when we can nuke
781    // dead loads.
782    if (ObjSize && !I->use_empty()) {
783      // Create the frame index object for this incoming parameter...
784      int FI = MFI->CreateFixedObject(ObjSize, ArgOffset);
785
786      // Create the SelectionDAG nodes corresponding to a load from this
787      // parameter.
788      SDOperand FIN = DAG.getFrameIndex(FI, MVT::i32);
789
790      ArgValue = DAG.getLoad(ObjectVT, DAG.getEntryNode(), FIN,
791                             DAG.getSrcValue(NULL));
792    } else if (ArgValue.Val == 0) {
793      if (MVT::isInteger(ObjectVT))
794        ArgValue = DAG.getConstant(0, ObjectVT);
795      else
796        ArgValue = DAG.getConstantFP(0, ObjectVT);
797    }
798    ArgValues.push_back(ArgValue);
799
800    if (ObjSize)
801      ArgOffset += ArgIncrement;   // Move on to the next argument.
802  }
803
804  // Make sure the instruction takes 8n+4 bytes to make sure the start of the
805  // arguments and the arguments after the retaddr has been pushed are aligned.
806  if ((ArgOffset & 7) == 0)
807    ArgOffset += 4;
808
809  VarArgsFrameIndex = 0xAAAAAAA;   // fastcc functions can't have varargs.
810  ReturnAddrIndex = 0;             // No return address slot generated yet.
811  BytesToPopOnReturn = ArgOffset;  // Callee pops all stack arguments.
812  BytesCallerReserves = 0;
813
814  // Finally, inform the code generator which regs we return values in.
815  switch (getValueType(F.getReturnType())) {
816  default: assert(0 && "Unknown type!");
817  case MVT::isVoid: break;
818  case MVT::i1:
819  case MVT::i8:
820  case MVT::i16:
821  case MVT::i32:
822    MF.addLiveOut(X86::EAX);
823    break;
824  case MVT::i64:
825    MF.addLiveOut(X86::EAX);
826    MF.addLiveOut(X86::EDX);
827    break;
828  case MVT::f32:
829  case MVT::f64:
830    MF.addLiveOut(X86::ST0);
831    break;
832  }
833  return ArgValues;
834}
835
836std::pair<SDOperand, SDOperand>
837X86TargetLowering::LowerFastCCCallTo(SDOperand Chain, const Type *RetTy,
838                                     bool isTailCall, SDOperand Callee,
839                                     ArgListTy &Args, SelectionDAG &DAG) {
840  // Count how many bytes are to be pushed on the stack.
841  unsigned NumBytes = 0;
842
843  // Keep track of the number of integer regs passed so far.  This can be either
844  // 0 (neither EAX or EDX used), 1 (EAX is used) or 2 (EAX and EDX are both
845  // used).
846  unsigned NumIntRegs = 0;
847
848  for (unsigned i = 0, e = Args.size(); i != e; ++i)
849    switch (getValueType(Args[i].second)) {
850    default: assert(0 && "Unknown value type!");
851    case MVT::i1:
852    case MVT::i8:
853    case MVT::i16:
854    case MVT::i32:
855      if (NumIntRegs < 2) {
856        ++NumIntRegs;
857        break;
858      }
859      // fall through
860    case MVT::f32:
861      NumBytes += 4;
862      break;
863    case MVT::i64:
864      if (NumIntRegs == 0) {
865        NumIntRegs = 2;
866        break;
867      } else if (NumIntRegs == 1) {
868        NumIntRegs = 2;
869        NumBytes += 4;
870        break;
871      }
872
873      // fall through
874    case MVT::f64:
875      NumBytes += 8;
876      break;
877    }
878
879  // Make sure the instruction takes 8n+4 bytes to make sure the start of the
880  // arguments and the arguments after the retaddr has been pushed are aligned.
881  if ((NumBytes & 7) == 0)
882    NumBytes += 4;
883
884  Chain = DAG.getNode(ISD::CALLSEQ_START, MVT::Other, Chain,
885                      DAG.getConstant(NumBytes, getPointerTy()));
886
887  // Arguments go on the stack in reverse order, as specified by the ABI.
888  unsigned ArgOffset = 0;
889  SDOperand StackPtr = DAG.getCopyFromReg(DAG.getEntryNode(),
890                                          X86::ESP, MVT::i32);
891  NumIntRegs = 0;
892  std::vector<SDOperand> Stores;
893  std::vector<SDOperand> RegValuesToPass;
894  for (unsigned i = 0, e = Args.size(); i != e; ++i) {
895    switch (getValueType(Args[i].second)) {
896    default: assert(0 && "Unexpected ValueType for argument!");
897    case MVT::i1:
898      Args[i].first = DAG.getNode(ISD::ANY_EXTEND, MVT::i8, Args[i].first);
899      // Fall through.
900    case MVT::i8:
901    case MVT::i16:
902    case MVT::i32:
903      if (NumIntRegs < 2) {
904        RegValuesToPass.push_back(Args[i].first);
905        ++NumIntRegs;
906        break;
907      }
908      // Fall through
909    case MVT::f32: {
910      SDOperand PtrOff = DAG.getConstant(ArgOffset, getPointerTy());
911      PtrOff = DAG.getNode(ISD::ADD, MVT::i32, StackPtr, PtrOff);
912      Stores.push_back(DAG.getNode(ISD::STORE, MVT::Other, Chain,
913                                   Args[i].first, PtrOff,
914                                   DAG.getSrcValue(NULL)));
915      ArgOffset += 4;
916      break;
917    }
918    case MVT::i64:
919      if (NumIntRegs < 2) {    // Can pass part of it in regs?
920        SDOperand Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, MVT::i32,
921                                   Args[i].first, DAG.getConstant(1, MVT::i32));
922        SDOperand Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, MVT::i32,
923                                   Args[i].first, DAG.getConstant(0, MVT::i32));
924        RegValuesToPass.push_back(Lo);
925        ++NumIntRegs;
926        if (NumIntRegs < 2) {   // Pass both parts in regs?
927          RegValuesToPass.push_back(Hi);
928          ++NumIntRegs;
929        } else {
930          // Pass the high part in memory.
931          SDOperand PtrOff = DAG.getConstant(ArgOffset, getPointerTy());
932          PtrOff = DAG.getNode(ISD::ADD, MVT::i32, StackPtr, PtrOff);
933          Stores.push_back(DAG.getNode(ISD::STORE, MVT::Other, Chain,
934                                       Hi, PtrOff, DAG.getSrcValue(NULL)));
935          ArgOffset += 4;
936        }
937        break;
938      }
939      // Fall through
940    case MVT::f64:
941      SDOperand PtrOff = DAG.getConstant(ArgOffset, getPointerTy());
942      PtrOff = DAG.getNode(ISD::ADD, MVT::i32, StackPtr, PtrOff);
943      Stores.push_back(DAG.getNode(ISD::STORE, MVT::Other, Chain,
944                                   Args[i].first, PtrOff,
945                                   DAG.getSrcValue(NULL)));
946      ArgOffset += 8;
947      break;
948    }
949  }
950  if (!Stores.empty())
951    Chain = DAG.getNode(ISD::TokenFactor, MVT::Other, Stores);
952
953  // Make sure the instruction takes 8n+4 bytes to make sure the start of the
954  // arguments and the arguments after the retaddr has been pushed are aligned.
955  if ((ArgOffset & 7) == 0)
956    ArgOffset += 4;
957
958  std::vector<MVT::ValueType> RetVals;
959  MVT::ValueType RetTyVT = getValueType(RetTy);
960
961  RetVals.push_back(MVT::Other);
962
963  // The result values produced have to be legal.  Promote the result.
964  switch (RetTyVT) {
965  case MVT::isVoid: break;
966  default:
967    RetVals.push_back(RetTyVT);
968    break;
969  case MVT::i1:
970  case MVT::i8:
971  case MVT::i16:
972    RetVals.push_back(MVT::i32);
973    break;
974  case MVT::f32:
975    if (X86ScalarSSE)
976      RetVals.push_back(MVT::f32);
977    else
978      RetVals.push_back(MVT::f64);
979    break;
980  case MVT::i64:
981    RetVals.push_back(MVT::i32);
982    RetVals.push_back(MVT::i32);
983    break;
984  }
985
986  if (X86DAGIsel) {
987    // Build a sequence of copy-to-reg nodes chained together with token chain
988    // and flag operands which copy the outgoing args into registers.
989    SDOperand InFlag;
990    for (unsigned i = 0, e = RegValuesToPass.size(); i != e; ++i) {
991      unsigned CCReg;
992      SDOperand RegToPass = RegValuesToPass[i];
993      switch (RegToPass.getValueType()) {
994      default: assert(0 && "Bad thing to pass in regs");
995      case MVT::i8:
996        CCReg = (i == 0) ? X86::AL  : X86::DL;
997        break;
998      case MVT::i16:
999        CCReg = (i == 0) ? X86::AX  : X86::DX;
1000        break;
1001      case MVT::i32:
1002        CCReg = (i == 0) ? X86::EAX : X86::EDX;
1003        break;
1004      }
1005
1006      Chain = DAG.getCopyToReg(Chain, CCReg, RegToPass, InFlag);
1007      InFlag = Chain.getValue(1);
1008    }
1009
1010    std::vector<MVT::ValueType> NodeTys;
1011    NodeTys.push_back(MVT::Other);   // Returns a chain
1012    NodeTys.push_back(MVT::Flag);    // Returns a flag for retval copy to use.
1013    std::vector<SDOperand> Ops;
1014    Ops.push_back(Chain);
1015    Ops.push_back(Callee);
1016    if (InFlag.Val)
1017      Ops.push_back(InFlag);
1018
1019    // FIXME: Do not generate X86ISD::TAILCALL for now.
1020    Chain = DAG.getNode(X86ISD::CALL, NodeTys, Ops);
1021    InFlag = Chain.getValue(1);
1022
1023    SDOperand RetVal;
1024    if (RetTyVT != MVT::isVoid) {
1025      switch (RetTyVT) {
1026      default: assert(0 && "Unknown value type to return!");
1027      case MVT::i1:
1028      case MVT::i8:
1029        RetVal = DAG.getCopyFromReg(Chain, X86::AL, MVT::i8, InFlag);
1030        Chain = RetVal.getValue(1);
1031        break;
1032      case MVT::i16:
1033        RetVal = DAG.getCopyFromReg(Chain, X86::AX, MVT::i16, InFlag);
1034        Chain = RetVal.getValue(1);
1035        break;
1036      case MVT::i32:
1037        RetVal = DAG.getCopyFromReg(Chain, X86::EAX, MVT::i32, InFlag);
1038        Chain = RetVal.getValue(1);
1039        break;
1040      case MVT::i64: {
1041        SDOperand Lo = DAG.getCopyFromReg(Chain, X86::EAX, MVT::i32, InFlag);
1042        SDOperand Hi = DAG.getCopyFromReg(Lo.getValue(1), X86::EDX, MVT::i32,
1043                                          Lo.getValue(2));
1044        RetVal = DAG.getNode(ISD::BUILD_PAIR, MVT::i64, Lo, Hi);
1045        Chain = Hi.getValue(1);
1046        break;
1047      }
1048      case MVT::f32:
1049      case MVT::f64: {
1050        std::vector<MVT::ValueType> Tys;
1051        Tys.push_back(MVT::f64);
1052        Tys.push_back(MVT::Other);
1053        std::vector<SDOperand> Ops;
1054        Ops.push_back(Chain);
1055        Ops.push_back(InFlag);
1056        RetVal = DAG.getNode(X86ISD::FP_GET_RESULT, Tys, Ops);
1057        Chain = RetVal.getValue(1);
1058        if (X86ScalarSSE) {
1059          unsigned Size = MVT::getSizeInBits(MVT::f64)/8;
1060          MachineFunction &MF = DAG.getMachineFunction();
1061          int SSFI = MF.getFrameInfo()->CreateStackObject(Size, Size);
1062          SDOperand StackSlot = DAG.getFrameIndex(SSFI, getPointerTy());
1063          Tys.clear();
1064          Tys.push_back(MVT::Other);
1065          Ops.clear();
1066          Ops.push_back(Chain);
1067          Ops.push_back(RetVal);
1068          Ops.push_back(StackSlot);
1069          Ops.push_back(DAG.getValueType(RetTyVT));
1070          Chain = DAG.getNode(X86ISD::FST, Tys, Ops);
1071          RetVal = DAG.getLoad(RetTyVT, Chain, StackSlot,
1072                               DAG.getSrcValue(NULL));
1073          Chain = RetVal.getValue(1);
1074        } else if (RetTyVT == MVT::f32)
1075          RetVal = DAG.getNode(ISD::FP_ROUND, MVT::f32, RetVal);
1076        break;
1077      }
1078      }
1079    }
1080
1081    Chain = DAG.getNode(ISD::CALLSEQ_END, MVT::Other, Chain,
1082                        DAG.getConstant(ArgOffset, getPointerTy()),
1083                        DAG.getConstant(ArgOffset, getPointerTy()));
1084    return std::make_pair(RetVal, Chain);
1085  } else {
1086    std::vector<SDOperand> Ops;
1087    Ops.push_back(Chain);
1088    Ops.push_back(Callee);
1089    Ops.push_back(DAG.getConstant(ArgOffset, getPointerTy()));
1090    // Callee pops all arg values on the stack.
1091    Ops.push_back(DAG.getConstant(ArgOffset, getPointerTy()));
1092
1093    // Pass register arguments as needed.
1094    Ops.insert(Ops.end(), RegValuesToPass.begin(), RegValuesToPass.end());
1095
1096    SDOperand TheCall = DAG.getNode(isTailCall ? X86ISD::TAILCALL : X86ISD::CALL,
1097                                    RetVals, Ops);
1098    Chain = DAG.getNode(ISD::CALLSEQ_END, MVT::Other, TheCall);
1099
1100    SDOperand ResultVal;
1101    switch (RetTyVT) {
1102    case MVT::isVoid: break;
1103    default:
1104      ResultVal = TheCall.getValue(1);
1105      break;
1106    case MVT::i1:
1107    case MVT::i8:
1108    case MVT::i16:
1109      ResultVal = DAG.getNode(ISD::TRUNCATE, RetTyVT, TheCall.getValue(1));
1110      break;
1111    case MVT::f32:
1112      // FIXME: we would really like to remember that this FP_ROUND operation is
1113      // okay to eliminate if we allow excess FP precision.
1114      ResultVal = DAG.getNode(ISD::FP_ROUND, MVT::f32, TheCall.getValue(1));
1115      break;
1116    case MVT::i64:
1117      ResultVal = DAG.getNode(ISD::BUILD_PAIR, MVT::i64, TheCall.getValue(1),
1118                              TheCall.getValue(2));
1119      break;
1120    }
1121
1122    return std::make_pair(ResultVal, Chain);
1123  }
1124}
1125
1126SDOperand X86TargetLowering::getReturnAddressFrameIndex(SelectionDAG &DAG) {
1127  if (ReturnAddrIndex == 0) {
1128    // Set up a frame object for the return address.
1129    MachineFunction &MF = DAG.getMachineFunction();
1130    ReturnAddrIndex = MF.getFrameInfo()->CreateFixedObject(4, -4);
1131  }
1132
1133  return DAG.getFrameIndex(ReturnAddrIndex, MVT::i32);
1134}
1135
1136
1137
1138std::pair<SDOperand, SDOperand> X86TargetLowering::
1139LowerFrameReturnAddress(bool isFrameAddress, SDOperand Chain, unsigned Depth,
1140                        SelectionDAG &DAG) {
1141  SDOperand Result;
1142  if (Depth)        // Depths > 0 not supported yet!
1143    Result = DAG.getConstant(0, getPointerTy());
1144  else {
1145    SDOperand RetAddrFI = getReturnAddressFrameIndex(DAG);
1146    if (!isFrameAddress)
1147      // Just load the return address
1148      Result = DAG.getLoad(MVT::i32, DAG.getEntryNode(), RetAddrFI,
1149                           DAG.getSrcValue(NULL));
1150    else
1151      Result = DAG.getNode(ISD::SUB, MVT::i32, RetAddrFI,
1152                           DAG.getConstant(4, MVT::i32));
1153  }
1154  return std::make_pair(Result, Chain);
1155}
1156
1157/// getCondBrOpcodeForX86CC - Returns the X86 conditional branch opcode
1158/// which corresponds to the condition code.
1159static unsigned getCondBrOpcodeForX86CC(unsigned X86CC) {
1160  switch (X86CC) {
1161  default: assert(0 && "Unknown X86 conditional code!");
1162  case X86ISD::COND_A:  return X86::JA;
1163  case X86ISD::COND_AE: return X86::JAE;
1164  case X86ISD::COND_B:  return X86::JB;
1165  case X86ISD::COND_BE: return X86::JBE;
1166  case X86ISD::COND_E:  return X86::JE;
1167  case X86ISD::COND_G:  return X86::JG;
1168  case X86ISD::COND_GE: return X86::JGE;
1169  case X86ISD::COND_L:  return X86::JL;
1170  case X86ISD::COND_LE: return X86::JLE;
1171  case X86ISD::COND_NE: return X86::JNE;
1172  case X86ISD::COND_NO: return X86::JNO;
1173  case X86ISD::COND_NP: return X86::JNP;
1174  case X86ISD::COND_NS: return X86::JNS;
1175  case X86ISD::COND_O:  return X86::JO;
1176  case X86ISD::COND_P:  return X86::JP;
1177  case X86ISD::COND_S:  return X86::JS;
1178  }
1179}
1180
1181/// getX86CC - do a one to one translation of a ISD::CondCode to the X86
1182/// specific condition code. It returns a X86ISD::COND_INVALID if it cannot
1183/// do a direct translation.
1184static unsigned getX86CC(SDOperand CC, bool isFP) {
1185  ISD::CondCode SetCCOpcode = cast<CondCodeSDNode>(CC)->get();
1186  unsigned X86CC = X86ISD::COND_INVALID;
1187  if (!isFP) {
1188    switch (SetCCOpcode) {
1189    default: break;
1190    case ISD::SETEQ:  X86CC = X86ISD::COND_E;  break;
1191    case ISD::SETGT:  X86CC = X86ISD::COND_G;  break;
1192    case ISD::SETGE:  X86CC = X86ISD::COND_GE; break;
1193    case ISD::SETLT:  X86CC = X86ISD::COND_L;  break;
1194    case ISD::SETLE:  X86CC = X86ISD::COND_LE; break;
1195    case ISD::SETNE:  X86CC = X86ISD::COND_NE; break;
1196    case ISD::SETULT: X86CC = X86ISD::COND_B;  break;
1197    case ISD::SETUGT: X86CC = X86ISD::COND_A;  break;
1198    case ISD::SETULE: X86CC = X86ISD::COND_BE; break;
1199    case ISD::SETUGE: X86CC = X86ISD::COND_AE; break;
1200    }
1201  } else {
1202    // On a floating point condition, the flags are set as follows:
1203    // ZF  PF  CF   op
1204    //  0 | 0 | 0 | X > Y
1205    //  0 | 0 | 1 | X < Y
1206    //  1 | 0 | 0 | X == Y
1207    //  1 | 1 | 1 | unordered
1208    switch (SetCCOpcode) {
1209    default: break;
1210    case ISD::SETUEQ:
1211    case ISD::SETEQ: X86CC = X86ISD::COND_E;  break;
1212    case ISD::SETOGT:
1213    case ISD::SETGT: X86CC = X86ISD::COND_A;  break;
1214    case ISD::SETOGE:
1215    case ISD::SETGE: X86CC = X86ISD::COND_AE; break;
1216    case ISD::SETULT:
1217    case ISD::SETLT: X86CC = X86ISD::COND_B;  break;
1218    case ISD::SETULE:
1219    case ISD::SETLE: X86CC = X86ISD::COND_BE; break;
1220    case ISD::SETONE:
1221    case ISD::SETNE: X86CC = X86ISD::COND_NE; break;
1222    case ISD::SETUO: X86CC = X86ISD::COND_P;  break;
1223    case ISD::SETO:  X86CC = X86ISD::COND_NP; break;
1224    }
1225  }
1226  return X86CC;
1227}
1228
1229/// hasFPCMov - is there a floating point cmov for the specific X86 condition
1230/// code. Current x86 isa includes the following FP cmov instructions:
1231/// fcmovb, fcomvbe, fcomve, fcmovu, fcmovae, fcmova, fcmovne, fcmovnu.
1232static bool hasFPCMov(unsigned X86CC) {
1233  switch (X86CC) {
1234  default:
1235    return false;
1236  case X86ISD::COND_B:
1237  case X86ISD::COND_BE:
1238  case X86ISD::COND_E:
1239  case X86ISD::COND_P:
1240  case X86ISD::COND_A:
1241  case X86ISD::COND_AE:
1242  case X86ISD::COND_NE:
1243  case X86ISD::COND_NP:
1244    return true;
1245  }
1246}
1247
1248MachineBasicBlock *
1249X86TargetLowering::InsertAtEndOfBasicBlock(MachineInstr *MI,
1250                                           MachineBasicBlock *BB) {
1251  assert((MI->getOpcode() == X86::CMOV_FR32 ||
1252          MI->getOpcode() == X86::CMOV_FR64) &&
1253         "Unexpected instr type to insert");
1254
1255  // To "insert" a SELECT_CC instruction, we actually have to insert the diamond
1256  // control-flow pattern.  The incoming instruction knows the destination vreg
1257  // to set, the condition code register to branch on, the true/false values to
1258  // select between, and a branch opcode to use.
1259  const BasicBlock *LLVM_BB = BB->getBasicBlock();
1260  ilist<MachineBasicBlock>::iterator It = BB;
1261  ++It;
1262
1263  //  thisMBB:
1264  //  ...
1265  //   TrueVal = ...
1266  //   cmpTY ccX, r1, r2
1267  //   bCC copy1MBB
1268  //   fallthrough --> copy0MBB
1269  MachineBasicBlock *thisMBB = BB;
1270  MachineBasicBlock *copy0MBB = new MachineBasicBlock(LLVM_BB);
1271  MachineBasicBlock *sinkMBB = new MachineBasicBlock(LLVM_BB);
1272  unsigned Opc = getCondBrOpcodeForX86CC(MI->getOperand(3).getImmedValue());
1273  BuildMI(BB, Opc, 1).addMBB(sinkMBB);
1274  MachineFunction *F = BB->getParent();
1275  F->getBasicBlockList().insert(It, copy0MBB);
1276  F->getBasicBlockList().insert(It, sinkMBB);
1277  // Update machine-CFG edges
1278  BB->addSuccessor(copy0MBB);
1279  BB->addSuccessor(sinkMBB);
1280
1281  //  copy0MBB:
1282  //   %FalseValue = ...
1283  //   # fallthrough to sinkMBB
1284  BB = copy0MBB;
1285
1286  // Update machine-CFG edges
1287  BB->addSuccessor(sinkMBB);
1288
1289  //  sinkMBB:
1290  //   %Result = phi [ %FalseValue, copy0MBB ], [ %TrueValue, thisMBB ]
1291  //  ...
1292  BB = sinkMBB;
1293  BuildMI(BB, X86::PHI, 4, MI->getOperand(0).getReg())
1294    .addReg(MI->getOperand(1).getReg()).addMBB(copy0MBB)
1295    .addReg(MI->getOperand(2).getReg()).addMBB(thisMBB);
1296
1297  delete MI;   // The pseudo instruction is gone now.
1298  return BB;
1299}
1300
1301
1302//===----------------------------------------------------------------------===//
1303//                           X86 Custom Lowering Hooks
1304//===----------------------------------------------------------------------===//
1305
1306/// LowerOperation - Provide custom lowering hooks for some operations.
1307///
1308SDOperand X86TargetLowering::LowerOperation(SDOperand Op, SelectionDAG &DAG) {
1309  switch (Op.getOpcode()) {
1310  default: assert(0 && "Should not custom lower this!");
1311  case ISD::ADD_PARTS:
1312  case ISD::SUB_PARTS: {
1313    assert(Op.getNumOperands() == 4 && Op.getValueType() == MVT::i32 &&
1314           "Not an i64 add/sub!");
1315    bool isAdd = Op.getOpcode() == ISD::ADD_PARTS;
1316    std::vector<MVT::ValueType> Tys;
1317    Tys.push_back(MVT::i32);
1318    Tys.push_back(MVT::Flag);
1319    std::vector<SDOperand> Ops;
1320    Ops.push_back(Op.getOperand(0));
1321    Ops.push_back(Op.getOperand(2));
1322    SDOperand Lo = DAG.getNode(isAdd ? X86ISD::ADD_FLAG : X86ISD::SUB_FLAG,
1323                               Tys, Ops);
1324    SDOperand Hi = DAG.getNode(isAdd ? X86ISD::ADC : X86ISD::SBB, MVT::i32,
1325                               Op.getOperand(1), Op.getOperand(3),
1326                               Lo.getValue(1));
1327    Tys.clear();
1328    Tys.push_back(MVT::i32);
1329    Tys.push_back(MVT::i32);
1330    Ops.clear();
1331    Ops.push_back(Lo);
1332    Ops.push_back(Hi);
1333    return DAG.getNode(ISD::MERGE_VALUES, Tys, Ops);
1334  }
1335  case ISD::SHL_PARTS:
1336  case ISD::SRA_PARTS:
1337  case ISD::SRL_PARTS: {
1338    assert(Op.getNumOperands() == 3 && Op.getValueType() == MVT::i32 &&
1339           "Not an i64 shift!");
1340    bool isSRA = Op.getOpcode() == ISD::SRA_PARTS;
1341    SDOperand ShOpLo = Op.getOperand(0);
1342    SDOperand ShOpHi = Op.getOperand(1);
1343    SDOperand ShAmt  = Op.getOperand(2);
1344    SDOperand Tmp1 = isSRA ? DAG.getNode(ISD::SRA, MVT::i32, ShOpHi,
1345                                         DAG.getConstant(31, MVT::i32))
1346                           : DAG.getConstant(0, MVT::i32);
1347
1348    SDOperand Tmp2, Tmp3;
1349    if (Op.getOpcode() == ISD::SHL_PARTS) {
1350      Tmp2 = DAG.getNode(X86ISD::SHLD, MVT::i32, ShOpHi, ShOpLo, ShAmt);
1351      Tmp3 = DAG.getNode(ISD::SHL, MVT::i32, ShOpLo, ShAmt);
1352    } else {
1353      Tmp2 = DAG.getNode(X86ISD::SHRD, MVT::i32, ShOpLo, ShOpHi, ShAmt);
1354      Tmp3 = DAG.getNode(isSRA ? ISD::SRA : ISD::SHL, MVT::i32, ShOpHi, ShAmt);
1355    }
1356
1357    SDOperand InFlag = DAG.getNode(X86ISD::TEST, MVT::Flag,
1358                                   ShAmt, DAG.getConstant(32, MVT::i8));
1359
1360    SDOperand Hi, Lo;
1361    SDOperand CC = DAG.getConstant(X86ISD::COND_NE, MVT::i8);
1362
1363    std::vector<MVT::ValueType> Tys;
1364    Tys.push_back(MVT::i32);
1365    Tys.push_back(MVT::Flag);
1366    std::vector<SDOperand> Ops;
1367    if (Op.getOpcode() == ISD::SHL_PARTS) {
1368      Ops.push_back(Tmp2);
1369      Ops.push_back(Tmp3);
1370      Ops.push_back(CC);
1371      Ops.push_back(InFlag);
1372      Hi = DAG.getNode(X86ISD::CMOV, Tys, Ops);
1373      InFlag = Hi.getValue(1);
1374
1375      Ops.clear();
1376      Ops.push_back(Tmp3);
1377      Ops.push_back(Tmp1);
1378      Ops.push_back(CC);
1379      Ops.push_back(InFlag);
1380      Lo = DAG.getNode(X86ISD::CMOV, Tys, Ops);
1381    } else {
1382      Ops.push_back(Tmp2);
1383      Ops.push_back(Tmp3);
1384      Ops.push_back(CC);
1385      Ops.push_back(InFlag);
1386      Lo = DAG.getNode(X86ISD::CMOV, Tys, Ops);
1387      InFlag = Lo.getValue(1);
1388
1389      Ops.clear();
1390      Ops.push_back(Tmp3);
1391      Ops.push_back(Tmp1);
1392      Ops.push_back(CC);
1393      Ops.push_back(InFlag);
1394      Hi = DAG.getNode(X86ISD::CMOV, Tys, Ops);
1395    }
1396
1397    Tys.clear();
1398    Tys.push_back(MVT::i32);
1399    Tys.push_back(MVT::i32);
1400    Ops.clear();
1401    Ops.push_back(Lo);
1402    Ops.push_back(Hi);
1403    return DAG.getNode(ISD::MERGE_VALUES, Tys, Ops);
1404  }
1405  case ISD::SINT_TO_FP: {
1406    assert(Op.getValueType() == MVT::f64 &&
1407           Op.getOperand(0).getValueType() == MVT::i64 &&
1408           "Unknown SINT_TO_FP to lower!");
1409    // We lower sint64->FP into a store to a temporary stack slot, followed by a
1410    // FILD64m node.
1411    MachineFunction &MF = DAG.getMachineFunction();
1412    int SSFI = MF.getFrameInfo()->CreateStackObject(8, 8);
1413    SDOperand StackSlot = DAG.getFrameIndex(SSFI, getPointerTy());
1414    SDOperand Store = DAG.getNode(ISD::STORE, MVT::Other, DAG.getEntryNode(),
1415                           Op.getOperand(0), StackSlot, DAG.getSrcValue(NULL));
1416    std::vector<MVT::ValueType> RTs;
1417    RTs.push_back(MVT::f64);
1418    RTs.push_back(MVT::Other);
1419    std::vector<SDOperand> Ops;
1420    Ops.push_back(Store);
1421    Ops.push_back(StackSlot);
1422    return DAG.getNode(X86ISD::FILD64m, RTs, Ops);
1423  }
1424  case ISD::FP_TO_SINT: {
1425    assert(Op.getValueType() <= MVT::i64 && Op.getValueType() >= MVT::i16 &&
1426           Op.getOperand(0).getValueType() == MVT::f64 &&
1427           "Unknown FP_TO_SINT to lower!");
1428    // We lower FP->sint64 into FISTP64, followed by a load, all to a temporary
1429    // stack slot.
1430    MachineFunction &MF = DAG.getMachineFunction();
1431    unsigned MemSize = MVT::getSizeInBits(Op.getValueType())/8;
1432    int SSFI = MF.getFrameInfo()->CreateStackObject(MemSize, MemSize);
1433    SDOperand StackSlot = DAG.getFrameIndex(SSFI, getPointerTy());
1434
1435    unsigned Opc;
1436    switch (Op.getValueType()) {
1437    default: assert(0 && "Invalid FP_TO_SINT to lower!");
1438    case MVT::i16: Opc = X86ISD::FP_TO_INT16_IN_MEM; break;
1439    case MVT::i32: Opc = X86ISD::FP_TO_INT32_IN_MEM; break;
1440    case MVT::i64: Opc = X86ISD::FP_TO_INT64_IN_MEM; break;
1441    }
1442
1443    // Build the FP_TO_INT*_IN_MEM
1444    std::vector<SDOperand> Ops;
1445    Ops.push_back(DAG.getEntryNode());
1446    Ops.push_back(Op.getOperand(0));
1447    Ops.push_back(StackSlot);
1448    SDOperand FIST = DAG.getNode(Opc, MVT::Other, Ops);
1449
1450    // Load the result.
1451    return DAG.getLoad(Op.getValueType(), FIST, StackSlot,
1452                       DAG.getSrcValue(NULL));
1453  }
1454  case ISD::READCYCLECOUNTER: {
1455    std::vector<MVT::ValueType> Tys;
1456    Tys.push_back(MVT::Other);
1457    Tys.push_back(MVT::Flag);
1458    std::vector<SDOperand> Ops;
1459    Ops.push_back(Op.getOperand(0));
1460    SDOperand rd = DAG.getNode(X86ISD::RDTSC_DAG, Tys, Ops);
1461    Ops.clear();
1462    Ops.push_back(DAG.getCopyFromReg(rd, X86::EAX, MVT::i32, rd.getValue(1)));
1463    Ops.push_back(DAG.getCopyFromReg(Ops[0].getValue(1), X86::EDX,
1464                                     MVT::i32, Ops[0].getValue(2)));
1465    Ops.push_back(Ops[1].getValue(1));
1466    Tys[0] = Tys[1] = MVT::i32;
1467    Tys.push_back(MVT::Other);
1468    return DAG.getNode(ISD::MERGE_VALUES, Tys, Ops);
1469  }
1470  case ISD::SETCC: {
1471    assert(Op.getValueType() == MVT::i8 && "SetCC type must be 8-bit integer");
1472    SDOperand CC   = Op.getOperand(2);
1473    SDOperand Cond = DAG.getNode(X86ISD::CMP, MVT::Flag,
1474                                 Op.getOperand(0), Op.getOperand(1));
1475    ISD::CondCode SetCCOpcode = cast<CondCodeSDNode>(CC)->get();
1476    bool isFP = MVT::isFloatingPoint(Op.getOperand(1).getValueType());
1477    unsigned X86CC = getX86CC(CC, isFP);
1478    if (X86CC != X86ISD::COND_INVALID) {
1479      return DAG.getNode(X86ISD::SETCC, MVT::i8,
1480                         DAG.getConstant(X86CC, MVT::i8), Cond);
1481    } else {
1482      assert(isFP && "Illegal integer SetCC!");
1483
1484      std::vector<MVT::ValueType> Tys;
1485      std::vector<SDOperand> Ops;
1486      switch (SetCCOpcode) {
1487      default: assert(false && "Illegal floating point SetCC!");
1488      case ISD::SETOEQ: {  // !PF & ZF
1489        Tys.push_back(MVT::i8);
1490        Tys.push_back(MVT::Flag);
1491        Ops.push_back(DAG.getConstant(X86ISD::COND_NP, MVT::i8));
1492        Ops.push_back(Cond);
1493        SDOperand Tmp1 = DAG.getNode(X86ISD::SETCC, Tys, Ops);
1494        SDOperand Tmp2 = DAG.getNode(X86ISD::SETCC, MVT::i8,
1495                                     DAG.getConstant(X86ISD::COND_E, MVT::i8),
1496                                     Tmp1.getValue(1));
1497        return DAG.getNode(ISD::AND, MVT::i8, Tmp1, Tmp2);
1498      }
1499      case ISD::SETOLT: {  // !PF & CF
1500        Tys.push_back(MVT::i8);
1501        Tys.push_back(MVT::Flag);
1502        Ops.push_back(DAG.getConstant(X86ISD::COND_NP, MVT::i8));
1503        Ops.push_back(Cond);
1504        SDOperand Tmp1 = DAG.getNode(X86ISD::SETCC, Tys, Ops);
1505        SDOperand Tmp2 = DAG.getNode(X86ISD::SETCC, MVT::i8,
1506                                     DAG.getConstant(X86ISD::COND_B, MVT::i8),
1507                                     Tmp1.getValue(1));
1508        return DAG.getNode(ISD::AND, MVT::i8, Tmp1, Tmp2);
1509      }
1510      case ISD::SETOLE: {  // !PF & (CF || ZF)
1511        Tys.push_back(MVT::i8);
1512        Tys.push_back(MVT::Flag);
1513        Ops.push_back(DAG.getConstant(X86ISD::COND_NP, MVT::i8));
1514        Ops.push_back(Cond);
1515        SDOperand Tmp1 = DAG.getNode(X86ISD::SETCC, Tys, Ops);
1516        SDOperand Tmp2 = DAG.getNode(X86ISD::SETCC, MVT::i8,
1517                                     DAG.getConstant(X86ISD::COND_BE, MVT::i8),
1518                                     Tmp1.getValue(1));
1519        return DAG.getNode(ISD::AND, MVT::i8, Tmp1, Tmp2);
1520      }
1521      case ISD::SETUGT: {  // PF | (!ZF & !CF)
1522        Tys.push_back(MVT::i8);
1523        Tys.push_back(MVT::Flag);
1524        Ops.push_back(DAG.getConstant(X86ISD::COND_P, MVT::i8));
1525        Ops.push_back(Cond);
1526        SDOperand Tmp1 = DAG.getNode(X86ISD::SETCC, Tys, Ops);
1527        SDOperand Tmp2 = DAG.getNode(X86ISD::SETCC, MVT::i8,
1528                                     DAG.getConstant(X86ISD::COND_A, MVT::i8),
1529                                     Tmp1.getValue(1));
1530        return DAG.getNode(ISD::OR, MVT::i8, Tmp1, Tmp2);
1531      }
1532      case ISD::SETUGE: {  // PF | !CF
1533        Tys.push_back(MVT::i8);
1534        Tys.push_back(MVT::Flag);
1535        Ops.push_back(DAG.getConstant(X86ISD::COND_P, MVT::i8));
1536        Ops.push_back(Cond);
1537        SDOperand Tmp1 = DAG.getNode(X86ISD::SETCC, Tys, Ops);
1538        SDOperand Tmp2 = DAG.getNode(X86ISD::SETCC, MVT::i8,
1539                                     DAG.getConstant(X86ISD::COND_AE, MVT::i8),
1540                                     Tmp1.getValue(1));
1541        return DAG.getNode(ISD::OR, MVT::i8, Tmp1, Tmp2);
1542      }
1543      case ISD::SETUNE: {  // PF | !ZF
1544        Tys.push_back(MVT::i8);
1545        Tys.push_back(MVT::Flag);
1546        Ops.push_back(DAG.getConstant(X86ISD::COND_P, MVT::i8));
1547        Ops.push_back(Cond);
1548        SDOperand Tmp1 = DAG.getNode(X86ISD::SETCC, Tys, Ops);
1549        SDOperand Tmp2 = DAG.getNode(X86ISD::SETCC, MVT::i8,
1550                                     DAG.getConstant(X86ISD::COND_NE, MVT::i8),
1551                                     Tmp1.getValue(1));
1552        return DAG.getNode(ISD::OR, MVT::i8, Tmp1, Tmp2);
1553      }
1554      }
1555    }
1556  }
1557  case ISD::SELECT: {
1558    MVT::ValueType VT = Op.getValueType();
1559    bool isFP      = MVT::isFloatingPoint(VT);
1560    bool isFPStack = isFP && (X86Vector < SSE2);
1561    bool isFPSSE   = isFP && (X86Vector >= SSE2);
1562    bool isValid   = false;
1563    SDOperand Op0 = Op.getOperand(0);
1564    SDOperand Cond, CC;
1565    if (Op0.getOpcode() == X86ISD::SETCC) {
1566      CC   = Op0.getOperand(0);
1567      Cond = Op0.getOperand(1);
1568      isValid =
1569        !(isFPStack && !hasFPCMov(cast<ConstantSDNode>(CC)->getSignExtended()));
1570    } else if (Op0.getOpcode() == ISD::SETCC) {
1571      CC = Op0.getOperand(2);
1572      bool isFP = MVT::isFloatingPoint(Op0.getOperand(1).getValueType());
1573      unsigned X86CC = getX86CC(CC, isFP);
1574      CC = DAG.getConstant(X86CC, MVT::i8);
1575      Cond = DAG.getNode(X86ISD::CMP, MVT::Flag,
1576                         Op0.getOperand(0), Op0.getOperand(1));
1577      isValid = true;
1578    }
1579
1580    if (!isValid) {
1581      CC = DAG.getConstant(X86ISD::COND_E, MVT::i8);
1582      Cond = DAG.getNode(X86ISD::TEST, MVT::Flag, Op0, Op0);
1583    }
1584
1585    std::vector<MVT::ValueType> Tys;
1586    Tys.push_back(Op.getValueType());
1587    Tys.push_back(MVT::Flag);
1588    std::vector<SDOperand> Ops;
1589    Ops.push_back(Op.getOperand(1));
1590    Ops.push_back(Op.getOperand(2));
1591    Ops.push_back(CC);
1592    Ops.push_back(Cond);
1593    return DAG.getNode(X86ISD::CMOV, Tys, Ops);
1594  }
1595  case ISD::BRCOND: {
1596    SDOperand Cond  = Op.getOperand(1);
1597    SDOperand Dest  = Op.getOperand(2);
1598    SDOperand CC;
1599    // TODO: handle Cond == OR / AND / XOR
1600    if (Cond.getOpcode() == X86ISD::SETCC) {
1601      CC = Cond.getOperand(0);
1602      Cond = Cond.getOperand(1);
1603    } else if (Cond.getOpcode() == ISD::SETCC) {
1604      CC = Cond.getOperand(2);
1605      bool isFP = MVT::isFloatingPoint(Cond.getOperand(1).getValueType());
1606      unsigned X86CC = getX86CC(CC, isFP);
1607      CC = DAG.getConstant(X86CC, MVT::i8);
1608      Cond = DAG.getNode(X86ISD::CMP, MVT::Flag,
1609                         Cond.getOperand(0), Cond.getOperand(1));
1610    } else {
1611      CC = DAG.getConstant(X86ISD::COND_NE, MVT::i8);
1612      Cond = DAG.getNode(X86ISD::TEST, MVT::Flag, Cond, Cond);
1613    }
1614    return DAG.getNode(X86ISD::BRCOND, Op.getValueType(),
1615                       Op.getOperand(0), Op.getOperand(2), CC, Cond);
1616  }
1617  case ISD::RET: {
1618    // Can only be return void.
1619    return DAG.getNode(X86ISD::RET_FLAG, MVT::Other, Op.getOperand(0),
1620                       DAG.getConstant(getBytesToPopOnReturn(), MVT::i16));
1621  }
1622  case ISD::MEMSET: {
1623    SDOperand InFlag;
1624    SDOperand Chain = Op.getOperand(0);
1625    unsigned Align =
1626      (unsigned)cast<ConstantSDNode>(Op.getOperand(4))->getValue();
1627    if (Align == 0) Align = 1;
1628
1629    MVT::ValueType AVT;
1630    SDOperand Count;
1631    if (ConstantSDNode *ValC = dyn_cast<ConstantSDNode>(Op.getOperand(2))) {
1632      unsigned ValReg;
1633      unsigned Val = ValC->getValue() & 255;
1634
1635      // If the value is a constant, then we can potentially use larger sets.
1636      switch (Align & 3) {
1637      case 2:   // WORD aligned
1638        AVT = MVT::i16;
1639        if (ConstantSDNode *I = dyn_cast<ConstantSDNode>(Op.getOperand(3)))
1640          Count = DAG.getConstant(I->getValue() / 2, MVT::i32);
1641        else
1642          Count = DAG.getNode(ISD::SRL, MVT::i32, Op.getOperand(3),
1643                              DAG.getConstant(1, MVT::i8));
1644        Val    = (Val << 8) | Val;
1645        ValReg = X86::AX;
1646        break;
1647      case 0:   // DWORD aligned
1648        AVT = MVT::i32;
1649        if (ConstantSDNode *I = dyn_cast<ConstantSDNode>(Op.getOperand(3)))
1650          Count = DAG.getConstant(I->getValue() / 4, MVT::i32);
1651        else
1652          Count = DAG.getNode(ISD::SRL, MVT::i32, Op.getOperand(3),
1653                              DAG.getConstant(2, MVT::i8));
1654        Val = (Val << 8)  | Val;
1655        Val = (Val << 16) | Val;
1656        ValReg = X86::EAX;
1657        break;
1658      default:  // Byte aligned
1659        AVT = MVT::i8;
1660        Count = Op.getOperand(3);
1661        ValReg = X86::AL;
1662        break;
1663      }
1664
1665      Chain  = DAG.getCopyToReg(Chain, ValReg, DAG.getConstant(Val, AVT),
1666                                InFlag);
1667      InFlag = Chain.getValue(1);
1668    } else {
1669      AVT    = MVT::i8;
1670      Count  = Op.getOperand(3);
1671      Chain  = DAG.getCopyToReg(Chain, X86::AL, Op.getOperand(2), InFlag);
1672      InFlag = Chain.getValue(1);
1673    }
1674
1675    Chain  = DAG.getCopyToReg(Chain, X86::ECX, Count, InFlag);
1676    InFlag = Chain.getValue(1);
1677    Chain  = DAG.getCopyToReg(Chain, X86::EDI, Op.getOperand(1), InFlag);
1678    InFlag = Chain.getValue(1);
1679
1680    return DAG.getNode(X86ISD::REP_STOS, MVT::Other, Chain,
1681                       DAG.getValueType(AVT), InFlag);
1682  }
1683  case ISD::MEMCPY: {
1684    SDOperand Chain = Op.getOperand(0);
1685    unsigned Align =
1686      (unsigned)cast<ConstantSDNode>(Op.getOperand(4))->getValue();
1687    if (Align == 0) Align = 1;
1688
1689    MVT::ValueType AVT;
1690    SDOperand Count;
1691    switch (Align & 3) {
1692    case 2:   // WORD aligned
1693      AVT = MVT::i16;
1694      if (ConstantSDNode *I = dyn_cast<ConstantSDNode>(Op.getOperand(3)))
1695        Count = DAG.getConstant(I->getValue() / 2, MVT::i32);
1696      else
1697        Count = DAG.getNode(ISD::SRL, MVT::i32, Op.getOperand(3),
1698                            DAG.getConstant(1, MVT::i8));
1699      break;
1700    case 0:   // DWORD aligned
1701      AVT = MVT::i32;
1702      if (ConstantSDNode *I = dyn_cast<ConstantSDNode>(Op.getOperand(3)))
1703        Count = DAG.getConstant(I->getValue() / 4, MVT::i32);
1704      else
1705        Count = DAG.getNode(ISD::SRL, MVT::i32, Op.getOperand(3),
1706                            DAG.getConstant(2, MVT::i8));
1707      break;
1708    default:  // Byte aligned
1709      AVT = MVT::i8;
1710      Count = Op.getOperand(3);
1711      break;
1712    }
1713
1714    SDOperand InFlag;
1715    Chain  = DAG.getCopyToReg(Chain, X86::ECX, Count, InFlag);
1716    InFlag = Chain.getValue(1);
1717    Chain  = DAG.getCopyToReg(Chain, X86::EDI, Op.getOperand(1), InFlag);
1718    InFlag = Chain.getValue(1);
1719    Chain  = DAG.getCopyToReg(Chain, X86::ESI, Op.getOperand(2), InFlag);
1720    InFlag = Chain.getValue(1);
1721
1722    return DAG.getNode(X86ISD::REP_MOVS, MVT::Other, Chain,
1723                       DAG.getValueType(AVT), InFlag);
1724  }
1725  case ISD::GlobalAddress: {
1726    SDOperand Result;
1727    GlobalValue *GV = cast<GlobalAddressSDNode>(Op)->getGlobal();
1728    // For Darwin, external and weak symbols are indirect, so we want to load
1729    // the value at address GV, not the value of GV itself.  This means that
1730    // the GlobalAddress must be in the base or index register of the address,
1731    // not the GV offset field.
1732    if (getTargetMachine().
1733        getSubtarget<X86Subtarget>().getIndirectExternAndWeakGlobals() &&
1734        (GV->hasWeakLinkage() || GV->isExternal()))
1735      Result = DAG.getLoad(MVT::i32, DAG.getEntryNode(),
1736                           DAG.getTargetGlobalAddress(GV, getPointerTy()),
1737                           DAG.getSrcValue(NULL));
1738    return Result;
1739  }
1740  }
1741}
1742
1743const char *X86TargetLowering::getTargetNodeName(unsigned Opcode) const {
1744  switch (Opcode) {
1745  default: return NULL;
1746  case X86ISD::ADD_FLAG:           return "X86ISD::ADD_FLAG";
1747  case X86ISD::SUB_FLAG:           return "X86ISD::SUB_FLAG";
1748  case X86ISD::ADC:                return "X86ISD::ADC";
1749  case X86ISD::SBB:                return "X86ISD::SBB";
1750  case X86ISD::SHLD:               return "X86ISD::SHLD";
1751  case X86ISD::SHRD:               return "X86ISD::SHRD";
1752  case X86ISD::FILD64m:            return "X86ISD::FILD64m";
1753  case X86ISD::FP_TO_INT16_IN_MEM: return "X86ISD::FP_TO_INT16_IN_MEM";
1754  case X86ISD::FP_TO_INT32_IN_MEM: return "X86ISD::FP_TO_INT32_IN_MEM";
1755  case X86ISD::FP_TO_INT64_IN_MEM: return "X86ISD::FP_TO_INT64_IN_MEM";
1756  case X86ISD::FLD:                return "X86ISD::FLD";
1757  case X86ISD::FST:                return "X86ISD::FST";
1758  case X86ISD::FP_GET_RESULT:      return "X86ISD::FP_GET_RESULT";
1759  case X86ISD::FP_SET_RESULT:      return "X86ISD::FP_SET_RESULT";
1760  case X86ISD::CALL:               return "X86ISD::CALL";
1761  case X86ISD::TAILCALL:           return "X86ISD::TAILCALL";
1762  case X86ISD::RDTSC_DAG:          return "X86ISD::RDTSC_DAG";
1763  case X86ISD::CMP:                return "X86ISD::CMP";
1764  case X86ISD::TEST:               return "X86ISD::TEST";
1765  case X86ISD::SETCC:              return "X86ISD::SETCC";
1766  case X86ISD::CMOV:               return "X86ISD::CMOV";
1767  case X86ISD::BRCOND:             return "X86ISD::BRCOND";
1768  case X86ISD::RET_FLAG:           return "X86ISD::RET_FLAG";
1769  case X86ISD::REP_STOS:           return "X86ISD::RET_STOS";
1770  case X86ISD::REP_MOVS:           return "X86ISD::RET_MOVS";
1771  }
1772}
1773
1774bool X86TargetLowering::isMaskedValueZeroForTargetNode(const SDOperand &Op,
1775                                                       uint64_t Mask) const {
1776
1777  unsigned Opc = Op.getOpcode();
1778
1779  switch (Opc) {
1780  default:
1781    assert(Opc >= ISD::BUILTIN_OP_END && "Expected a target specific node");
1782    break;
1783  case X86ISD::SETCC: return (Mask & 1) == 0;
1784  }
1785
1786  return false;
1787}
1788