X86ISelLowering.cpp revision 5ff12fc41a0c53811b59b06deb11d59f45f6df69 
1//===-- X86ISelLowering.cpp - X86 DAG Lowering Implementation -------------===//
2//
3//                     The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10// This file defines the interfaces that X86 uses to lower LLVM code into a
11// selection DAG.
12//
13//===----------------------------------------------------------------------===//
14
15#define DEBUG_TYPE "x86-isel"
16#include "X86.h"
17#include "X86InstrBuilder.h"
18#include "X86ISelLowering.h"
19#include "X86TargetMachine.h"
20#include "X86TargetObjectFile.h"
21#include "llvm/CallingConv.h"
22#include "llvm/Constants.h"
23#include "llvm/DerivedTypes.h"
24#include "llvm/GlobalAlias.h"
25#include "llvm/GlobalVariable.h"
26#include "llvm/Function.h"
27#include "llvm/Instructions.h"
28#include "llvm/Intrinsics.h"
29#include "llvm/LLVMContext.h"
30#include "llvm/CodeGen/MachineFrameInfo.h"
31#include "llvm/CodeGen/MachineFunction.h"
32#include "llvm/CodeGen/MachineInstrBuilder.h"
33#include "llvm/CodeGen/MachineJumpTableInfo.h"
34#include "llvm/CodeGen/MachineModuleInfo.h"
35#include "llvm/CodeGen/MachineRegisterInfo.h"
36#include "llvm/CodeGen/PseudoSourceValue.h"
37#include "llvm/MC/MCAsmInfo.h"
38#include "llvm/MC/MCContext.h"
39#include "llvm/MC/MCExpr.h"
40#include "llvm/MC/MCSymbol.h"
41#include "llvm/ADT/BitVector.h"
42#include "llvm/ADT/SmallSet.h"
43#include "llvm/ADT/Statistic.h"
44#include "llvm/ADT/StringExtras.h"
45#include "llvm/ADT/VectorExtras.h"
46#include "llvm/Support/CommandLine.h"
47#include "llvm/Support/Debug.h"
48#include "llvm/Support/Dwarf.h"
49#include "llvm/Support/ErrorHandling.h"
50#include "llvm/Support/MathExtras.h"
51#include "llvm/Support/raw_ostream.h"
52using namespace llvm;
53using namespace dwarf;
54
55STATISTIC(NumTailCalls, "Number of tail calls");
56
57static cl::opt<bool>
58DisableMMX("disable-mmx", cl::Hidden, cl::desc("Disable use of MMX"));
59
60// Forward declarations.
61static SDValue getMOVL(SelectionDAG &DAG, DebugLoc dl, EVT VT, SDValue V1,
62                       SDValue V2);
63
64static TargetLoweringObjectFile *createTLOF(X86TargetMachine &TM) {
65  switch (TM.getSubtarget<X86Subtarget>().TargetType) {
66  default: llvm_unreachable("unknown subtarget type");
67  case X86Subtarget::isDarwin:
68    if (TM.getSubtarget<X86Subtarget>().is64Bit())
69      return new X8664_MachoTargetObjectFile();
70    return new TargetLoweringObjectFileMachO();
71  case X86Subtarget::isELF:
72   if (TM.getSubtarget<X86Subtarget>().is64Bit())
73     return new X8664_ELFTargetObjectFile(TM);
74    return new X8632_ELFTargetObjectFile(TM);
75  case X86Subtarget::isMingw:
76  case X86Subtarget::isCygwin:
77  case X86Subtarget::isWindows:
78    return new TargetLoweringObjectFileCOFF();
79  }
80}
81
82X86TargetLowering::X86TargetLowering(X86TargetMachine &TM)
83  : TargetLowering(TM, createTLOF(TM)) {
84  Subtarget = &TM.getSubtarget<X86Subtarget>();
85  X86ScalarSSEf64 = Subtarget->hasSSE2();
86  X86ScalarSSEf32 = Subtarget->hasSSE1();
87  X86StackPtr = Subtarget->is64Bit() ? X86::RSP : X86::ESP;
88
89  RegInfo = TM.getRegisterInfo();
90  TD = getTargetData();
91
92  // Set up the TargetLowering object.
93
94  // X86 is weird, it always uses i8 for shift amounts and setcc results.
95  setShiftAmountType(MVT::i8);
96  setBooleanContents(ZeroOrOneBooleanContent);
97  setSchedulingPreference(Sched::RegPressure);
98  setStackPointerRegisterToSaveRestore(X86StackPtr);
99
100  if (Subtarget->isTargetDarwin()) {
101    // Darwin should use _setjmp/_longjmp instead of setjmp/longjmp.
102    setUseUnderscoreSetJmp(false);
103    setUseUnderscoreLongJmp(false);
104  } else if (Subtarget->isTargetMingw()) {
105    // MS runtime is weird: it exports _setjmp, but longjmp!
106    setUseUnderscoreSetJmp(true);
107    setUseUnderscoreLongJmp(false);
108  } else {
109    setUseUnderscoreSetJmp(true);
110    setUseUnderscoreLongJmp(true);
111  }
112
113  // Set up the register classes.
114  addRegisterClass(MVT::i8, X86::GR8RegisterClass);
115  addRegisterClass(MVT::i16, X86::GR16RegisterClass);
116  addRegisterClass(MVT::i32, X86::GR32RegisterClass);
117  if (Subtarget->is64Bit())
118    addRegisterClass(MVT::i64, X86::GR64RegisterClass);
119
120  setLoadExtAction(ISD::SEXTLOAD, MVT::i1, Promote);
121
122  // We don't accept any truncstore of integer registers.
123  setTruncStoreAction(MVT::i64, MVT::i32, Expand);
124  setTruncStoreAction(MVT::i64, MVT::i16, Expand);
125  setTruncStoreAction(MVT::i64, MVT::i8 , Expand);
126  setTruncStoreAction(MVT::i32, MVT::i16, Expand);
127  setTruncStoreAction(MVT::i32, MVT::i8 , Expand);
128  setTruncStoreAction(MVT::i16, MVT::i8,  Expand);
129
130  // SETOEQ and SETUNE require checking two conditions.
131  setCondCodeAction(ISD::SETOEQ, MVT::f32, Expand);
132  setCondCodeAction(ISD::SETOEQ, MVT::f64, Expand);
133  setCondCodeAction(ISD::SETOEQ, MVT::f80, Expand);
134  setCondCodeAction(ISD::SETUNE, MVT::f32, Expand);
135  setCondCodeAction(ISD::SETUNE, MVT::f64, Expand);
136  setCondCodeAction(ISD::SETUNE, MVT::f80, Expand);
137
138  // Promote all UINT_TO_FP to larger SINT_TO_FP's, as X86 doesn't have this
139  // operation.
140  setOperationAction(ISD::UINT_TO_FP       , MVT::i1   , Promote);
141  setOperationAction(ISD::UINT_TO_FP       , MVT::i8   , Promote);
142  setOperationAction(ISD::UINT_TO_FP       , MVT::i16  , Promote);
143
144  if (Subtarget->is64Bit()) {
145    setOperationAction(ISD::UINT_TO_FP     , MVT::i32  , Promote);
146    setOperationAction(ISD::UINT_TO_FP     , MVT::i64  , Expand);
147  } else if (!UseSoftFloat) {
148    // We have an algorithm for SSE2->double, and we turn this into a
149    // 64-bit FILD followed by conditional FADD for other targets.
150    setOperationAction(ISD::UINT_TO_FP     , MVT::i64  , Custom);
151    // We have an algorithm for SSE2, and we turn this into a 64-bit
152    // FILD for other targets.
153    setOperationAction(ISD::UINT_TO_FP     , MVT::i32  , Custom);
154  }
155
156  // Promote i1/i8 SINT_TO_FP to larger SINT_TO_FP's, as X86 doesn't have
157  // this operation.
158  setOperationAction(ISD::SINT_TO_FP       , MVT::i1   , Promote);
159  setOperationAction(ISD::SINT_TO_FP       , MVT::i8   , Promote);
160
161  if (!UseSoftFloat) {
162    // SSE has no i16 to fp conversion, only i32
163    if (X86ScalarSSEf32) {
164      setOperationAction(ISD::SINT_TO_FP     , MVT::i16  , Promote);
165      // f32 and f64 cases are Legal, f80 case is not
166      setOperationAction(ISD::SINT_TO_FP     , MVT::i32  , Custom);
167    } else {
168      setOperationAction(ISD::SINT_TO_FP     , MVT::i16  , Custom);
169      setOperationAction(ISD::SINT_TO_FP     , MVT::i32  , Custom);
170    }
171  } else {
172    setOperationAction(ISD::SINT_TO_FP     , MVT::i16  , Promote);
173    setOperationAction(ISD::SINT_TO_FP     , MVT::i32  , Promote);
174  }
175
176  // In 32-bit mode these are custom lowered.  In 64-bit mode F32 and F64
177  // are Legal, f80 is custom lowered.
178  setOperationAction(ISD::FP_TO_SINT     , MVT::i64  , Custom);
179  setOperationAction(ISD::SINT_TO_FP     , MVT::i64  , Custom);
180
181  // Promote i1/i8 FP_TO_SINT to larger FP_TO_SINTS's, as X86 doesn't have
182  // this operation.
183  setOperationAction(ISD::FP_TO_SINT       , MVT::i1   , Promote);
184  setOperationAction(ISD::FP_TO_SINT       , MVT::i8   , Promote);
185
186  if (X86ScalarSSEf32) {
187    setOperationAction(ISD::FP_TO_SINT     , MVT::i16  , Promote);
188    // f32 and f64 cases are Legal, f80 case is not
189    setOperationAction(ISD::FP_TO_SINT     , MVT::i32  , Custom);
190  } else {
191    setOperationAction(ISD::FP_TO_SINT     , MVT::i16  , Custom);
192    setOperationAction(ISD::FP_TO_SINT     , MVT::i32  , Custom);
193  }
194
195  // Handle FP_TO_UINT by promoting the destination to a larger signed
196  // conversion.
197  setOperationAction(ISD::FP_TO_UINT       , MVT::i1   , Promote);
198  setOperationAction(ISD::FP_TO_UINT       , MVT::i8   , Promote);
199  setOperationAction(ISD::FP_TO_UINT       , MVT::i16  , Promote);
200
201  if (Subtarget->is64Bit()) {
202    setOperationAction(ISD::FP_TO_UINT     , MVT::i64  , Expand);
203    setOperationAction(ISD::FP_TO_UINT     , MVT::i32  , Promote);
204  } else if (!UseSoftFloat) {
205    if (X86ScalarSSEf32 && !Subtarget->hasSSE3())
206      // Expand FP_TO_UINT into a select.
207      // FIXME: We would like to use a Custom expander here eventually to do
208      // the optimal thing for SSE vs. the default expansion in the legalizer.
209      setOperationAction(ISD::FP_TO_UINT   , MVT::i32  , Expand);
210    else
211      // With SSE3 we can use fisttpll to convert to a signed i64; without
212      // SSE, we're stuck with a fistpll.
213      setOperationAction(ISD::FP_TO_UINT   , MVT::i32  , Custom);
214  }
215
216  // TODO: when we have SSE, these could be more efficient, by using movd/movq.
217  if (!X86ScalarSSEf64) {
218    setOperationAction(ISD::BIT_CONVERT      , MVT::f32  , Expand);
219    setOperationAction(ISD::BIT_CONVERT      , MVT::i32  , Expand);
220    if (Subtarget->is64Bit()) {
221      setOperationAction(ISD::BIT_CONVERT    , MVT::f64  , Expand);
222      // Without SSE, i64->f64 goes through memory; i64->MMX is Legal.
223      if (Subtarget->hasMMX() && !DisableMMX)
224        setOperationAction(ISD::BIT_CONVERT    , MVT::i64  , Custom);
225      else
226        setOperationAction(ISD::BIT_CONVERT    , MVT::i64  , Expand);
227    }
228  }
229
230  // Scalar integer divide and remainder are lowered to use operations that
231  // produce two results, to match the available instructions. This exposes
232  // the two-result form to trivial CSE, which is able to combine x/y and x%y
233  // into a single instruction.
234  //
235  // Scalar integer multiply-high is also lowered to use two-result
236  // operations, to match the available instructions. However, plain multiply
237  // (low) operations are left as Legal, as there are single-result
238  // instructions for this in x86. Using the two-result multiply instructions
239  // when both high and low results are needed must be arranged by dagcombine.
240  setOperationAction(ISD::MULHS           , MVT::i8    , Expand);
241  setOperationAction(ISD::MULHU           , MVT::i8    , Expand);
242  setOperationAction(ISD::SDIV            , MVT::i8    , Expand);
243  setOperationAction(ISD::UDIV            , MVT::i8    , Expand);
244  setOperationAction(ISD::SREM            , MVT::i8    , Expand);
245  setOperationAction(ISD::UREM            , MVT::i8    , Expand);
246  setOperationAction(ISD::MULHS           , MVT::i16   , Expand);
247  setOperationAction(ISD::MULHU           , MVT::i16   , Expand);
248  setOperationAction(ISD::SDIV            , MVT::i16   , Expand);
249  setOperationAction(ISD::UDIV            , MVT::i16   , Expand);
250  setOperationAction(ISD::SREM            , MVT::i16   , Expand);
251  setOperationAction(ISD::UREM            , MVT::i16   , Expand);
252  setOperationAction(ISD::MULHS           , MVT::i32   , Expand);
253  setOperationAction(ISD::MULHU           , MVT::i32   , Expand);
254  setOperationAction(ISD::SDIV            , MVT::i32   , Expand);
255  setOperationAction(ISD::UDIV            , MVT::i32   , Expand);
256  setOperationAction(ISD::SREM            , MVT::i32   , Expand);
257  setOperationAction(ISD::UREM            , MVT::i32   , Expand);
258  setOperationAction(ISD::MULHS           , MVT::i64   , Expand);
259  setOperationAction(ISD::MULHU           , MVT::i64   , Expand);
260  setOperationAction(ISD::SDIV            , MVT::i64   , Expand);
261  setOperationAction(ISD::UDIV            , MVT::i64   , Expand);
262  setOperationAction(ISD::SREM            , MVT::i64   , Expand);
263  setOperationAction(ISD::UREM            , MVT::i64   , Expand);
264
265  setOperationAction(ISD::BR_JT            , MVT::Other, Expand);
266  setOperationAction(ISD::BRCOND           , MVT::Other, Custom);
267  setOperationAction(ISD::BR_CC            , MVT::Other, Expand);
268  setOperationAction(ISD::SELECT_CC        , MVT::Other, Expand);
269  if (Subtarget->is64Bit())
270    setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i32, Legal);
271  setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i16  , Legal);
272  setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i8   , Legal);
273  setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i1   , Expand);
274  setOperationAction(ISD::FP_ROUND_INREG   , MVT::f32  , Expand);
275  setOperationAction(ISD::FREM             , MVT::f32  , Expand);
276  setOperationAction(ISD::FREM             , MVT::f64  , Expand);
277  setOperationAction(ISD::FREM             , MVT::f80  , Expand);
278  setOperationAction(ISD::FLT_ROUNDS_      , MVT::i32  , Custom);
279
280  setOperationAction(ISD::CTPOP            , MVT::i8   , Expand);
281  setOperationAction(ISD::CTTZ             , MVT::i8   , Custom);
282  setOperationAction(ISD::CTLZ             , MVT::i8   , Custom);
283  setOperationAction(ISD::CTPOP            , MVT::i16  , Expand);
284  setOperationAction(ISD::CTTZ             , MVT::i16  , Custom);
285  setOperationAction(ISD::CTLZ             , MVT::i16  , Custom);
286  setOperationAction(ISD::CTPOP            , MVT::i32  , Expand);
287  setOperationAction(ISD::CTTZ             , MVT::i32  , Custom);
288  setOperationAction(ISD::CTLZ             , MVT::i32  , Custom);
289  if (Subtarget->is64Bit()) {
290    setOperationAction(ISD::CTPOP          , MVT::i64  , Expand);
291    setOperationAction(ISD::CTTZ           , MVT::i64  , Custom);
292    setOperationAction(ISD::CTLZ           , MVT::i64  , Custom);
293  }
294
295  setOperationAction(ISD::READCYCLECOUNTER , MVT::i64  , Custom);
296  setOperationAction(ISD::BSWAP            , MVT::i16  , Expand);
297
298  // These should be promoted to a larger select which is supported.
299  setOperationAction(ISD::SELECT          , MVT::i1   , Promote);
300  // X86 wants to expand cmov itself.
301  setOperationAction(ISD::SELECT          , MVT::i8   , Custom);
302  setOperationAction(ISD::SELECT        , MVT::i16  , Custom);
303  setOperationAction(ISD::SELECT          , MVT::i32  , Custom);
304  setOperationAction(ISD::SELECT          , MVT::f32  , Custom);
305  setOperationAction(ISD::SELECT          , MVT::f64  , Custom);
306  setOperationAction(ISD::SELECT          , MVT::f80  , Custom);
307  setOperationAction(ISD::SETCC           , MVT::i8   , Custom);
308  setOperationAction(ISD::SETCC           , MVT::i16  , Custom);
309  setOperationAction(ISD::SETCC           , MVT::i32  , Custom);
310  setOperationAction(ISD::SETCC           , MVT::f32  , Custom);
311  setOperationAction(ISD::SETCC           , MVT::f64  , Custom);
312  setOperationAction(ISD::SETCC           , MVT::f80  , Custom);
313  if (Subtarget->is64Bit()) {
314    setOperationAction(ISD::SELECT        , MVT::i64  , Custom);
315    setOperationAction(ISD::SETCC         , MVT::i64  , Custom);
316  }
317  setOperationAction(ISD::EH_RETURN       , MVT::Other, Custom);
318
319  // Darwin ABI issue.
320  setOperationAction(ISD::ConstantPool    , MVT::i32  , Custom);
321  setOperationAction(ISD::JumpTable       , MVT::i32  , Custom);
322  setOperationAction(ISD::GlobalAddress   , MVT::i32  , Custom);
323  setOperationAction(ISD::GlobalTLSAddress, MVT::i32  , Custom);
324  if (Subtarget->is64Bit())
325    setOperationAction(ISD::GlobalTLSAddress, MVT::i64, Custom);
326  setOperationAction(ISD::ExternalSymbol  , MVT::i32  , Custom);
327  setOperationAction(ISD::BlockAddress    , MVT::i32  , Custom);
328  if (Subtarget->is64Bit()) {
329    setOperationAction(ISD::ConstantPool  , MVT::i64  , Custom);
330    setOperationAction(ISD::JumpTable     , MVT::i64  , Custom);
331    setOperationAction(ISD::GlobalAddress , MVT::i64  , Custom);
332    setOperationAction(ISD::ExternalSymbol, MVT::i64  , Custom);
333    setOperationAction(ISD::BlockAddress  , MVT::i64  , Custom);
334  }
335  // 64-bit addm sub, shl, sra, srl (iff 32-bit x86)
336  setOperationAction(ISD::SHL_PARTS       , MVT::i32  , Custom);
337  setOperationAction(ISD::SRA_PARTS       , MVT::i32  , Custom);
338  setOperationAction(ISD::SRL_PARTS       , MVT::i32  , Custom);
339  if (Subtarget->is64Bit()) {
340    setOperationAction(ISD::SHL_PARTS     , MVT::i64  , Custom);
341    setOperationAction(ISD::SRA_PARTS     , MVT::i64  , Custom);
342    setOperationAction(ISD::SRL_PARTS     , MVT::i64  , Custom);
343  }
344
345  if (Subtarget->hasSSE1())
346    setOperationAction(ISD::PREFETCH      , MVT::Other, Legal);
347
348  if (!Subtarget->hasSSE2())
349    setOperationAction(ISD::MEMBARRIER    , MVT::Other, Expand);
350
351  // Expand certain atomics
352  setOperationAction(ISD::ATOMIC_CMP_SWAP, MVT::i8, Custom);
353  setOperationAction(ISD::ATOMIC_CMP_SWAP, MVT::i16, Custom);
354  setOperationAction(ISD::ATOMIC_CMP_SWAP, MVT::i32, Custom);
355  setOperationAction(ISD::ATOMIC_CMP_SWAP, MVT::i64, Custom);
356
357  setOperationAction(ISD::ATOMIC_LOAD_SUB, MVT::i8, Custom);
358  setOperationAction(ISD::ATOMIC_LOAD_SUB, MVT::i16, Custom);
359  setOperationAction(ISD::ATOMIC_LOAD_SUB, MVT::i32, Custom);
360  setOperationAction(ISD::ATOMIC_LOAD_SUB, MVT::i64, Custom);
361
362  if (!Subtarget->is64Bit()) {
363    setOperationAction(ISD::ATOMIC_LOAD_ADD, MVT::i64, Custom);
364    setOperationAction(ISD::ATOMIC_LOAD_SUB, MVT::i64, Custom);
365    setOperationAction(ISD::ATOMIC_LOAD_AND, MVT::i64, Custom);
366    setOperationAction(ISD::ATOMIC_LOAD_OR, MVT::i64, Custom);
367    setOperationAction(ISD::ATOMIC_LOAD_XOR, MVT::i64, Custom);
368    setOperationAction(ISD::ATOMIC_LOAD_NAND, MVT::i64, Custom);
369    setOperationAction(ISD::ATOMIC_SWAP, MVT::i64, Custom);
370  }
371
372  // FIXME - use subtarget debug flags
373  if (!Subtarget->isTargetDarwin() &&
374      !Subtarget->isTargetELF() &&
375      !Subtarget->isTargetCygMing()) {
376    setOperationAction(ISD::EH_LABEL, MVT::Other, Expand);
377  }
378
379  setOperationAction(ISD::EXCEPTIONADDR, MVT::i64, Expand);
380  setOperationAction(ISD::EHSELECTION,   MVT::i64, Expand);
381  setOperationAction(ISD::EXCEPTIONADDR, MVT::i32, Expand);
382  setOperationAction(ISD::EHSELECTION,   MVT::i32, Expand);
383  if (Subtarget->is64Bit()) {
384    setExceptionPointerRegister(X86::RAX);
385    setExceptionSelectorRegister(X86::RDX);
386  } else {
387    setExceptionPointerRegister(X86::EAX);
388    setExceptionSelectorRegister(X86::EDX);
389  }
390  setOperationAction(ISD::FRAME_TO_ARGS_OFFSET, MVT::i32, Custom);
391  setOperationAction(ISD::FRAME_TO_ARGS_OFFSET, MVT::i64, Custom);
392
393  setOperationAction(ISD::TRAMPOLINE, MVT::Other, Custom);
394
395  setOperationAction(ISD::TRAP, MVT::Other, Legal);
396
397  // VASTART needs to be custom lowered to use the VarArgsFrameIndex
398  setOperationAction(ISD::VASTART           , MVT::Other, Custom);
399  setOperationAction(ISD::VAEND             , MVT::Other, Expand);
400  if (Subtarget->is64Bit()) {
401    setOperationAction(ISD::VAARG           , MVT::Other, Custom);
402    setOperationAction(ISD::VACOPY          , MVT::Other, Custom);
403  } else {
404    setOperationAction(ISD::VAARG           , MVT::Other, Expand);
405    setOperationAction(ISD::VACOPY          , MVT::Other, Expand);
406  }
407
408  setOperationAction(ISD::STACKSAVE,          MVT::Other, Expand);
409  setOperationAction(ISD::STACKRESTORE,       MVT::Other, Expand);
410  if (Subtarget->is64Bit())
411    setOperationAction(ISD::DYNAMIC_STACKALLOC, MVT::i64, Expand);
412  if (Subtarget->isTargetCygMing())
413    setOperationAction(ISD::DYNAMIC_STACKALLOC, MVT::i32, Custom);
414  else
415    setOperationAction(ISD::DYNAMIC_STACKALLOC, MVT::i32, Expand);
416
417  if (!UseSoftFloat && X86ScalarSSEf64) {
418    // f32 and f64 use SSE.
419    // Set up the FP register classes.
420    addRegisterClass(MVT::f32, X86::FR32RegisterClass);
421    addRegisterClass(MVT::f64, X86::FR64RegisterClass);
422
423    // Use ANDPD to simulate FABS.
424    setOperationAction(ISD::FABS , MVT::f64, Custom);
425    setOperationAction(ISD::FABS , MVT::f32, Custom);
426
427    // Use XORP to simulate FNEG.
428    setOperationAction(ISD::FNEG , MVT::f64, Custom);
429    setOperationAction(ISD::FNEG , MVT::f32, Custom);
430
431    // Use ANDPD and ORPD to simulate FCOPYSIGN.
432    setOperationAction(ISD::FCOPYSIGN, MVT::f64, Custom);
433    setOperationAction(ISD::FCOPYSIGN, MVT::f32, Custom);
434
435    // We don't support sin/cos/fmod
436    setOperationAction(ISD::FSIN , MVT::f64, Expand);
437    setOperationAction(ISD::FCOS , MVT::f64, Expand);
438    setOperationAction(ISD::FSIN , MVT::f32, Expand);
439    setOperationAction(ISD::FCOS , MVT::f32, Expand);
440
441    // Expand FP immediates into loads from the stack, except for the special
442    // cases we handle.
443    addLegalFPImmediate(APFloat(+0.0)); // xorpd
444    addLegalFPImmediate(APFloat(+0.0f)); // xorps
445  } else if (!UseSoftFloat && X86ScalarSSEf32) {
446    // Use SSE for f32, x87 for f64.
447    // Set up the FP register classes.
448    addRegisterClass(MVT::f32, X86::FR32RegisterClass);
449    addRegisterClass(MVT::f64, X86::RFP64RegisterClass);
450
451    // Use ANDPS to simulate FABS.
452    setOperationAction(ISD::FABS , MVT::f32, Custom);
453
454    // Use XORP to simulate FNEG.
455    setOperationAction(ISD::FNEG , MVT::f32, Custom);
456
457    setOperationAction(ISD::UNDEF,     MVT::f64, Expand);
458
459    // Use ANDPS and ORPS to simulate FCOPYSIGN.
460    setOperationAction(ISD::FCOPYSIGN, MVT::f64, Expand);
461    setOperationAction(ISD::FCOPYSIGN, MVT::f32, Custom);
462
463    // We don't support sin/cos/fmod
464    setOperationAction(ISD::FSIN , MVT::f32, Expand);
465    setOperationAction(ISD::FCOS , MVT::f32, Expand);
466
467    // Special cases we handle for FP constants.
468    addLegalFPImmediate(APFloat(+0.0f)); // xorps
469    addLegalFPImmediate(APFloat(+0.0)); // FLD0
470    addLegalFPImmediate(APFloat(+1.0)); // FLD1
471    addLegalFPImmediate(APFloat(-0.0)); // FLD0/FCHS
472    addLegalFPImmediate(APFloat(-1.0)); // FLD1/FCHS
473
474    if (!UnsafeFPMath) {
475      setOperationAction(ISD::FSIN           , MVT::f64  , Expand);
476      setOperationAction(ISD::FCOS           , MVT::f64  , Expand);
477    }
478  } else if (!UseSoftFloat) {
479    // f32 and f64 in x87.
480    // Set up the FP register classes.
481    addRegisterClass(MVT::f64, X86::RFP64RegisterClass);
482    addRegisterClass(MVT::f32, X86::RFP32RegisterClass);
483
484    setOperationAction(ISD::UNDEF,     MVT::f64, Expand);
485    setOperationAction(ISD::UNDEF,     MVT::f32, Expand);
486    setOperationAction(ISD::FCOPYSIGN, MVT::f64, Expand);
487    setOperationAction(ISD::FCOPYSIGN, MVT::f32, Expand);
488
489    if (!UnsafeFPMath) {
490      setOperationAction(ISD::FSIN           , MVT::f64  , Expand);
491      setOperationAction(ISD::FCOS           , MVT::f64  , Expand);
492    }
493    addLegalFPImmediate(APFloat(+0.0)); // FLD0
494    addLegalFPImmediate(APFloat(+1.0)); // FLD1
495    addLegalFPImmediate(APFloat(-0.0)); // FLD0/FCHS
496    addLegalFPImmediate(APFloat(-1.0)); // FLD1/FCHS
497    addLegalFPImmediate(APFloat(+0.0f)); // FLD0
498    addLegalFPImmediate(APFloat(+1.0f)); // FLD1
499    addLegalFPImmediate(APFloat(-0.0f)); // FLD0/FCHS
500    addLegalFPImmediate(APFloat(-1.0f)); // FLD1/FCHS
501  }
502
503  // Long double always uses X87.
504  if (!UseSoftFloat) {
505    addRegisterClass(MVT::f80, X86::RFP80RegisterClass);
506    setOperationAction(ISD::UNDEF,     MVT::f80, Expand);
507    setOperationAction(ISD::FCOPYSIGN, MVT::f80, Expand);
508    {
509      bool ignored;
510      APFloat TmpFlt(+0.0);
511      TmpFlt.convert(APFloat::x87DoubleExtended, APFloat::rmNearestTiesToEven,
512                     &ignored);
513      addLegalFPImmediate(TmpFlt);  // FLD0
514      TmpFlt.changeSign();
515      addLegalFPImmediate(TmpFlt);  // FLD0/FCHS
516      APFloat TmpFlt2(+1.0);
517      TmpFlt2.convert(APFloat::x87DoubleExtended, APFloat::rmNearestTiesToEven,
518                      &ignored);
519      addLegalFPImmediate(TmpFlt2);  // FLD1
520      TmpFlt2.changeSign();
521      addLegalFPImmediate(TmpFlt2);  // FLD1/FCHS
522    }
523
524    if (!UnsafeFPMath) {
525      setOperationAction(ISD::FSIN           , MVT::f80  , Expand);
526      setOperationAction(ISD::FCOS           , MVT::f80  , Expand);
527    }
528  }
529
530  // Always use a library call for pow.
531  setOperationAction(ISD::FPOW             , MVT::f32  , Expand);
532  setOperationAction(ISD::FPOW             , MVT::f64  , Expand);
533  setOperationAction(ISD::FPOW             , MVT::f80  , Expand);
534
535  setOperationAction(ISD::FLOG, MVT::f80, Expand);
536  setOperationAction(ISD::FLOG2, MVT::f80, Expand);
537  setOperationAction(ISD::FLOG10, MVT::f80, Expand);
538  setOperationAction(ISD::FEXP, MVT::f80, Expand);
539  setOperationAction(ISD::FEXP2, MVT::f80, Expand);
540
541  // First set operation action for all vector types to either promote
542  // (for widening) or expand (for scalarization). Then we will selectively
543  // turn on ones that can be effectively codegen'd.
544  for (unsigned VT = (unsigned)MVT::FIRST_VECTOR_VALUETYPE;
545       VT <= (unsigned)MVT::LAST_VECTOR_VALUETYPE; ++VT) {
546    setOperationAction(ISD::ADD , (MVT::SimpleValueType)VT, Expand);
547    setOperationAction(ISD::SUB , (MVT::SimpleValueType)VT, Expand);
548    setOperationAction(ISD::FADD, (MVT::SimpleValueType)VT, Expand);
549    setOperationAction(ISD::FNEG, (MVT::SimpleValueType)VT, Expand);
550    setOperationAction(ISD::FSUB, (MVT::SimpleValueType)VT, Expand);
551    setOperationAction(ISD::MUL , (MVT::SimpleValueType)VT, Expand);
552    setOperationAction(ISD::FMUL, (MVT::SimpleValueType)VT, Expand);
553    setOperationAction(ISD::SDIV, (MVT::SimpleValueType)VT, Expand);
554    setOperationAction(ISD::UDIV, (MVT::SimpleValueType)VT, Expand);
555    setOperationAction(ISD::FDIV, (MVT::SimpleValueType)VT, Expand);
556    setOperationAction(ISD::SREM, (MVT::SimpleValueType)VT, Expand);
557    setOperationAction(ISD::UREM, (MVT::SimpleValueType)VT, Expand);
558    setOperationAction(ISD::LOAD, (MVT::SimpleValueType)VT, Expand);
559    setOperationAction(ISD::VECTOR_SHUFFLE, (MVT::SimpleValueType)VT, Expand);
560    setOperationAction(ISD::EXTRACT_VECTOR_ELT,(MVT::SimpleValueType)VT,Expand);
561    setOperationAction(ISD::EXTRACT_SUBVECTOR,(MVT::SimpleValueType)VT,Expand);
562    setOperationAction(ISD::INSERT_VECTOR_ELT,(MVT::SimpleValueType)VT, Expand);
563    setOperationAction(ISD::FABS, (MVT::SimpleValueType)VT, Expand);
564    setOperationAction(ISD::FSIN, (MVT::SimpleValueType)VT, Expand);
565    setOperationAction(ISD::FCOS, (MVT::SimpleValueType)VT, Expand);
566    setOperationAction(ISD::FREM, (MVT::SimpleValueType)VT, Expand);
567    setOperationAction(ISD::FPOWI, (MVT::SimpleValueType)VT, Expand);
568    setOperationAction(ISD::FSQRT, (MVT::SimpleValueType)VT, Expand);
569    setOperationAction(ISD::FCOPYSIGN, (MVT::SimpleValueType)VT, Expand);
570    setOperationAction(ISD::SMUL_LOHI, (MVT::SimpleValueType)VT, Expand);
571    setOperationAction(ISD::UMUL_LOHI, (MVT::SimpleValueType)VT, Expand);
572    setOperationAction(ISD::SDIVREM, (MVT::SimpleValueType)VT, Expand);
573    setOperationAction(ISD::UDIVREM, (MVT::SimpleValueType)VT, Expand);
574    setOperationAction(ISD::FPOW, (MVT::SimpleValueType)VT, Expand);
575    setOperationAction(ISD::CTPOP, (MVT::SimpleValueType)VT, Expand);
576    setOperationAction(ISD::CTTZ, (MVT::SimpleValueType)VT, Expand);
577    setOperationAction(ISD::CTLZ, (MVT::SimpleValueType)VT, Expand);
578    setOperationAction(ISD::SHL, (MVT::SimpleValueType)VT, Expand);
579    setOperationAction(ISD::SRA, (MVT::SimpleValueType)VT, Expand);
580    setOperationAction(ISD::SRL, (MVT::SimpleValueType)VT, Expand);
581    setOperationAction(ISD::ROTL, (MVT::SimpleValueType)VT, Expand);
582    setOperationAction(ISD::ROTR, (MVT::SimpleValueType)VT, Expand);
583    setOperationAction(ISD::BSWAP, (MVT::SimpleValueType)VT, Expand);
584    setOperationAction(ISD::VSETCC, (MVT::SimpleValueType)VT, Expand);
585    setOperationAction(ISD::FLOG, (MVT::SimpleValueType)VT, Expand);
586    setOperationAction(ISD::FLOG2, (MVT::SimpleValueType)VT, Expand);
587    setOperationAction(ISD::FLOG10, (MVT::SimpleValueType)VT, Expand);
588    setOperationAction(ISD::FEXP, (MVT::SimpleValueType)VT, Expand);
589    setOperationAction(ISD::FEXP2, (MVT::SimpleValueType)VT, Expand);
590    setOperationAction(ISD::FP_TO_UINT, (MVT::SimpleValueType)VT, Expand);
591    setOperationAction(ISD::FP_TO_SINT, (MVT::SimpleValueType)VT, Expand);
592    setOperationAction(ISD::UINT_TO_FP, (MVT::SimpleValueType)VT, Expand);
593    setOperationAction(ISD::SINT_TO_FP, (MVT::SimpleValueType)VT, Expand);
594    setOperationAction(ISD::SIGN_EXTEND_INREG, (MVT::SimpleValueType)VT,Expand);
595    setOperationAction(ISD::TRUNCATE,  (MVT::SimpleValueType)VT, Expand);
596    setOperationAction(ISD::SIGN_EXTEND,  (MVT::SimpleValueType)VT, Expand);
597    setOperationAction(ISD::ZERO_EXTEND,  (MVT::SimpleValueType)VT, Expand);
598    setOperationAction(ISD::ANY_EXTEND,  (MVT::SimpleValueType)VT, Expand);
599    for (unsigned InnerVT = (unsigned)MVT::FIRST_VECTOR_VALUETYPE;
600         InnerVT <= (unsigned)MVT::LAST_VECTOR_VALUETYPE; ++InnerVT)
601      setTruncStoreAction((MVT::SimpleValueType)VT,
602                          (MVT::SimpleValueType)InnerVT, Expand);
603    setLoadExtAction(ISD::SEXTLOAD, (MVT::SimpleValueType)VT, Expand);
604    setLoadExtAction(ISD::ZEXTLOAD, (MVT::SimpleValueType)VT, Expand);
605    setLoadExtAction(ISD::EXTLOAD, (MVT::SimpleValueType)VT, Expand);
606  }
607
608  // FIXME: In order to prevent SSE instructions being expanded to MMX ones
609  // with -msoft-float, disable use of MMX as well.
610  if (!UseSoftFloat && !DisableMMX && Subtarget->hasMMX()) {
611    addRegisterClass(MVT::v8i8,  X86::VR64RegisterClass, false);
612    addRegisterClass(MVT::v4i16, X86::VR64RegisterClass, false);
613    addRegisterClass(MVT::v2i32, X86::VR64RegisterClass, false);
614    addRegisterClass(MVT::v2f32, X86::VR64RegisterClass, false);
615    addRegisterClass(MVT::v1i64, X86::VR64RegisterClass, false);
616
617    setOperationAction(ISD::ADD,                MVT::v8i8,  Legal);
618    setOperationAction(ISD::ADD,                MVT::v4i16, Legal);
619    setOperationAction(ISD::ADD,                MVT::v2i32, Legal);
620    setOperationAction(ISD::ADD,                MVT::v1i64, Legal);
621
622    setOperationAction(ISD::SUB,                MVT::v8i8,  Legal);
623    setOperationAction(ISD::SUB,                MVT::v4i16, Legal);
624    setOperationAction(ISD::SUB,                MVT::v2i32, Legal);
625    setOperationAction(ISD::SUB,                MVT::v1i64, Legal);
626
627    setOperationAction(ISD::MULHS,              MVT::v4i16, Legal);
628    setOperationAction(ISD::MUL,                MVT::v4i16, Legal);
629
630    setOperationAction(ISD::AND,                MVT::v8i8,  Promote);
631    AddPromotedToType (ISD::AND,                MVT::v8i8,  MVT::v1i64);
632    setOperationAction(ISD::AND,                MVT::v4i16, Promote);
633    AddPromotedToType (ISD::AND,                MVT::v4i16, MVT::v1i64);
634    setOperationAction(ISD::AND,                MVT::v2i32, Promote);
635    AddPromotedToType (ISD::AND,                MVT::v2i32, MVT::v1i64);
636    setOperationAction(ISD::AND,                MVT::v1i64, Legal);
637
638    setOperationAction(ISD::OR,                 MVT::v8i8,  Promote);
639    AddPromotedToType (ISD::OR,                 MVT::v8i8,  MVT::v1i64);
640    setOperationAction(ISD::OR,                 MVT::v4i16, Promote);
641    AddPromotedToType (ISD::OR,                 MVT::v4i16, MVT::v1i64);
642    setOperationAction(ISD::OR,                 MVT::v2i32, Promote);
643    AddPromotedToType (ISD::OR,                 MVT::v2i32, MVT::v1i64);
644    setOperationAction(ISD::OR,                 MVT::v1i64, Legal);
645
646    setOperationAction(ISD::XOR,                MVT::v8i8,  Promote);
647    AddPromotedToType (ISD::XOR,                MVT::v8i8,  MVT::v1i64);
648    setOperationAction(ISD::XOR,                MVT::v4i16, Promote);
649    AddPromotedToType (ISD::XOR,                MVT::v4i16, MVT::v1i64);
650    setOperationAction(ISD::XOR,                MVT::v2i32, Promote);
651    AddPromotedToType (ISD::XOR,                MVT::v2i32, MVT::v1i64);
652    setOperationAction(ISD::XOR,                MVT::v1i64, Legal);
653
654    setOperationAction(ISD::LOAD,               MVT::v8i8,  Promote);
655    AddPromotedToType (ISD::LOAD,               MVT::v8i8,  MVT::v1i64);
656    setOperationAction(ISD::LOAD,               MVT::v4i16, Promote);
657    AddPromotedToType (ISD::LOAD,               MVT::v4i16, MVT::v1i64);
658    setOperationAction(ISD::LOAD,               MVT::v2i32, Promote);
659    AddPromotedToType (ISD::LOAD,               MVT::v2i32, MVT::v1i64);
660    setOperationAction(ISD::LOAD,               MVT::v2f32, Promote);
661    AddPromotedToType (ISD::LOAD,               MVT::v2f32, MVT::v1i64);
662    setOperationAction(ISD::LOAD,               MVT::v1i64, Legal);
663
664    setOperationAction(ISD::BUILD_VECTOR,       MVT::v8i8,  Custom);
665    setOperationAction(ISD::BUILD_VECTOR,       MVT::v4i16, Custom);
666    setOperationAction(ISD::BUILD_VECTOR,       MVT::v2i32, Custom);
667    setOperationAction(ISD::BUILD_VECTOR,       MVT::v2f32, Custom);
668    setOperationAction(ISD::BUILD_VECTOR,       MVT::v1i64, Custom);
669
670    setOperationAction(ISD::VECTOR_SHUFFLE,     MVT::v8i8,  Custom);
671    setOperationAction(ISD::VECTOR_SHUFFLE,     MVT::v4i16, Custom);
672    setOperationAction(ISD::VECTOR_SHUFFLE,     MVT::v2i32, Custom);
673    setOperationAction(ISD::VECTOR_SHUFFLE,     MVT::v1i64, Custom);
674
675    setOperationAction(ISD::SCALAR_TO_VECTOR,   MVT::v2f32, Custom);
676    setOperationAction(ISD::SCALAR_TO_VECTOR,   MVT::v8i8,  Custom);
677    setOperationAction(ISD::SCALAR_TO_VECTOR,   MVT::v4i16, Custom);
678    setOperationAction(ISD::SCALAR_TO_VECTOR,   MVT::v1i64, Custom);
679
680    setOperationAction(ISD::INSERT_VECTOR_ELT,  MVT::v4i16, Custom);
681
682    setOperationAction(ISD::SELECT,             MVT::v8i8, Promote);
683    setOperationAction(ISD::SELECT,             MVT::v4i16, Promote);
684    setOperationAction(ISD::SELECT,             MVT::v2i32, Promote);
685    setOperationAction(ISD::SELECT,             MVT::v1i64, Custom);
686    setOperationAction(ISD::VSETCC,             MVT::v8i8, Custom);
687    setOperationAction(ISD::VSETCC,             MVT::v4i16, Custom);
688    setOperationAction(ISD::VSETCC,             MVT::v2i32, Custom);
689
690    if (!X86ScalarSSEf64 && Subtarget->is64Bit()) {
691      setOperationAction(ISD::BIT_CONVERT,        MVT::v8i8,  Custom);
692      setOperationAction(ISD::BIT_CONVERT,        MVT::v4i16, Custom);
693      setOperationAction(ISD::BIT_CONVERT,        MVT::v2i32, Custom);
694      setOperationAction(ISD::BIT_CONVERT,        MVT::v2f32, Custom);
695      setOperationAction(ISD::BIT_CONVERT,        MVT::v1i64, Custom);
696    }
697  }
698
699  if (!UseSoftFloat && Subtarget->hasSSE1()) {
700    addRegisterClass(MVT::v4f32, X86::VR128RegisterClass);
701
702    setOperationAction(ISD::FADD,               MVT::v4f32, Legal);
703    setOperationAction(ISD::FSUB,               MVT::v4f32, Legal);
704    setOperationAction(ISD::FMUL,               MVT::v4f32, Legal);
705    setOperationAction(ISD::FDIV,               MVT::v4f32, Legal);
706    setOperationAction(ISD::FSQRT,              MVT::v4f32, Legal);
707    setOperationAction(ISD::FNEG,               MVT::v4f32, Custom);
708    setOperationAction(ISD::LOAD,               MVT::v4f32, Legal);
709    setOperationAction(ISD::BUILD_VECTOR,       MVT::v4f32, Custom);
710    setOperationAction(ISD::VECTOR_SHUFFLE,     MVT::v4f32, Custom);
711    setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v4f32, Custom);
712    setOperationAction(ISD::SELECT,             MVT::v4f32, Custom);
713    setOperationAction(ISD::VSETCC,             MVT::v4f32, Custom);
714  }
715
716  if (!UseSoftFloat && Subtarget->hasSSE2()) {
717    addRegisterClass(MVT::v2f64, X86::VR128RegisterClass);
718
719    // FIXME: Unfortunately -soft-float and -no-implicit-float means XMM
720    // registers cannot be used even for integer operations.
721    addRegisterClass(MVT::v16i8, X86::VR128RegisterClass);
722    addRegisterClass(MVT::v8i16, X86::VR128RegisterClass);
723    addRegisterClass(MVT::v4i32, X86::VR128RegisterClass);
724    addRegisterClass(MVT::v2i64, X86::VR128RegisterClass);
725
726    setOperationAction(ISD::ADD,                MVT::v16i8, Legal);
727    setOperationAction(ISD::ADD,                MVT::v8i16, Legal);
728    setOperationAction(ISD::ADD,                MVT::v4i32, Legal);
729    setOperationAction(ISD::ADD,                MVT::v2i64, Legal);
730    setOperationAction(ISD::MUL,                MVT::v2i64, Custom);
731    setOperationAction(ISD::SUB,                MVT::v16i8, Legal);
732    setOperationAction(ISD::SUB,                MVT::v8i16, Legal);
733    setOperationAction(ISD::SUB,                MVT::v4i32, Legal);
734    setOperationAction(ISD::SUB,                MVT::v2i64, Legal);
735    setOperationAction(ISD::MUL,                MVT::v8i16, Legal);
736    setOperationAction(ISD::FADD,               MVT::v2f64, Legal);
737    setOperationAction(ISD::FSUB,               MVT::v2f64, Legal);
738    setOperationAction(ISD::FMUL,               MVT::v2f64, Legal);
739    setOperationAction(ISD::FDIV,               MVT::v2f64, Legal);
740    setOperationAction(ISD::FSQRT,              MVT::v2f64, Legal);
741    setOperationAction(ISD::FNEG,               MVT::v2f64, Custom);
742
743    setOperationAction(ISD::VSETCC,             MVT::v2f64, Custom);
744    setOperationAction(ISD::VSETCC,             MVT::v16i8, Custom);
745    setOperationAction(ISD::VSETCC,             MVT::v8i16, Custom);
746    setOperationAction(ISD::VSETCC,             MVT::v4i32, Custom);
747
748    setOperationAction(ISD::SCALAR_TO_VECTOR,   MVT::v16i8, Custom);
749    setOperationAction(ISD::SCALAR_TO_VECTOR,   MVT::v8i16, Custom);
750    setOperationAction(ISD::INSERT_VECTOR_ELT,  MVT::v8i16, Custom);
751    setOperationAction(ISD::INSERT_VECTOR_ELT,  MVT::v4i32, Custom);
752    setOperationAction(ISD::INSERT_VECTOR_ELT,  MVT::v4f32, Custom);
753
754    setOperationAction(ISD::CONCAT_VECTORS,     MVT::v2f64, Custom);
755    setOperationAction(ISD::CONCAT_VECTORS,     MVT::v2i64, Custom);
756    setOperationAction(ISD::CONCAT_VECTORS,     MVT::v16i8, Custom);
757    setOperationAction(ISD::CONCAT_VECTORS,     MVT::v8i16, Custom);
758    setOperationAction(ISD::CONCAT_VECTORS,     MVT::v4i32, Custom);
759
760    // Custom lower build_vector, vector_shuffle, and extract_vector_elt.
761    for (unsigned i = (unsigned)MVT::v16i8; i != (unsigned)MVT::v2i64; ++i) {
762      EVT VT = (MVT::SimpleValueType)i;
763      // Do not attempt to custom lower non-power-of-2 vectors
764      if (!isPowerOf2_32(VT.getVectorNumElements()))
765        continue;
766      // Do not attempt to custom lower non-128-bit vectors
767      if (!VT.is128BitVector())
768        continue;
769      setOperationAction(ISD::BUILD_VECTOR,
770                         VT.getSimpleVT().SimpleTy, Custom);
771      setOperationAction(ISD::VECTOR_SHUFFLE,
772                         VT.getSimpleVT().SimpleTy, Custom);
773      setOperationAction(ISD::EXTRACT_VECTOR_ELT,
774                         VT.getSimpleVT().SimpleTy, Custom);
775    }
776
777    setOperationAction(ISD::BUILD_VECTOR,       MVT::v2f64, Custom);
778    setOperationAction(ISD::BUILD_VECTOR,       MVT::v2i64, Custom);
779    setOperationAction(ISD::VECTOR_SHUFFLE,     MVT::v2f64, Custom);
780    setOperationAction(ISD::VECTOR_SHUFFLE,     MVT::v2i64, Custom);
781    setOperationAction(ISD::INSERT_VECTOR_ELT,  MVT::v2f64, Custom);
782    setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v2f64, Custom);
783
784    if (Subtarget->is64Bit()) {
785      setOperationAction(ISD::INSERT_VECTOR_ELT,  MVT::v2i64, Custom);
786      setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v2i64, Custom);
787    }
788
789    // Promote v16i8, v8i16, v4i32 load, select, and, or, xor to v2i64.
790    for (unsigned i = (unsigned)MVT::v16i8; i != (unsigned)MVT::v2i64; i++) {
791      MVT::SimpleValueType SVT = (MVT::SimpleValueType)i;
792      EVT VT = SVT;
793
794      // Do not attempt to promote non-128-bit vectors
795      if (!VT.is128BitVector()) {
796        continue;
797      }
798
799      setOperationAction(ISD::AND,    SVT, Promote);
800      AddPromotedToType (ISD::AND,    SVT, MVT::v2i64);
801      setOperationAction(ISD::OR,     SVT, Promote);
802      AddPromotedToType (ISD::OR,     SVT, MVT::v2i64);
803      setOperationAction(ISD::XOR,    SVT, Promote);
804      AddPromotedToType (ISD::XOR,    SVT, MVT::v2i64);
805      setOperationAction(ISD::LOAD,   SVT, Promote);
806      AddPromotedToType (ISD::LOAD,   SVT, MVT::v2i64);
807      setOperationAction(ISD::SELECT, SVT, Promote);
808      AddPromotedToType (ISD::SELECT, SVT, MVT::v2i64);
809    }
810
811    setTruncStoreAction(MVT::f64, MVT::f32, Expand);
812
813    // Custom lower v2i64 and v2f64 selects.
814    setOperationAction(ISD::LOAD,               MVT::v2f64, Legal);
815    setOperationAction(ISD::LOAD,               MVT::v2i64, Legal);
816    setOperationAction(ISD::SELECT,             MVT::v2f64, Custom);
817    setOperationAction(ISD::SELECT,             MVT::v2i64, Custom);
818
819    setOperationAction(ISD::FP_TO_SINT,         MVT::v4i32, Legal);
820    setOperationAction(ISD::SINT_TO_FP,         MVT::v4i32, Legal);
821    if (!DisableMMX && Subtarget->hasMMX()) {
822      setOperationAction(ISD::FP_TO_SINT,         MVT::v2i32, Custom);
823      setOperationAction(ISD::SINT_TO_FP,         MVT::v2i32, Custom);
824    }
825  }
826
827  if (Subtarget->hasSSE41()) {
828    setOperationAction(ISD::FFLOOR,             MVT::f32,   Legal);
829    setOperationAction(ISD::FCEIL,              MVT::f32,   Legal);
830    setOperationAction(ISD::FTRUNC,             MVT::f32,   Legal);
831    setOperationAction(ISD::FRINT,              MVT::f32,   Legal);
832    setOperationAction(ISD::FNEARBYINT,         MVT::f32,   Legal);
833    setOperationAction(ISD::FFLOOR,             MVT::f64,   Legal);
834    setOperationAction(ISD::FCEIL,              MVT::f64,   Legal);
835    setOperationAction(ISD::FTRUNC,             MVT::f64,   Legal);
836    setOperationAction(ISD::FRINT,              MVT::f64,   Legal);
837    setOperationAction(ISD::FNEARBYINT,         MVT::f64,   Legal);
838
839    // FIXME: Do we need to handle scalar-to-vector here?
840    setOperationAction(ISD::MUL,                MVT::v4i32, Legal);
841
842    // i8 and i16 vectors are custom , because the source register and source
843    // source memory operand types are not the same width.  f32 vectors are
844    // custom since the immediate controlling the insert encodes additional
845    // information.
846    setOperationAction(ISD::INSERT_VECTOR_ELT,  MVT::v16i8, Custom);
847    setOperationAction(ISD::INSERT_VECTOR_ELT,  MVT::v8i16, Custom);
848    setOperationAction(ISD::INSERT_VECTOR_ELT,  MVT::v4i32, Custom);
849    setOperationAction(ISD::INSERT_VECTOR_ELT,  MVT::v4f32, Custom);
850
851    setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v16i8, Custom);
852    setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v8i16, Custom);
853    setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v4i32, Custom);
854    setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v4f32, Custom);
855
856    if (Subtarget->is64Bit()) {
857      setOperationAction(ISD::INSERT_VECTOR_ELT,  MVT::v2i64, Legal);
858      setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v2i64, Legal);
859    }
860  }
861
862  if (Subtarget->hasSSE42()) {
863    setOperationAction(ISD::VSETCC,             MVT::v2i64, Custom);
864  }
865
866  if (!UseSoftFloat && Subtarget->hasAVX()) {
867    addRegisterClass(MVT::v8f32, X86::VR256RegisterClass);
868    addRegisterClass(MVT::v4f64, X86::VR256RegisterClass);
869    addRegisterClass(MVT::v8i32, X86::VR256RegisterClass);
870    addRegisterClass(MVT::v4i64, X86::VR256RegisterClass);
871
872    setOperationAction(ISD::LOAD,               MVT::v8f32, Legal);
873    setOperationAction(ISD::LOAD,               MVT::v8i32, Legal);
874    setOperationAction(ISD::LOAD,               MVT::v4f64, Legal);
875    setOperationAction(ISD::LOAD,               MVT::v4i64, Legal);
876    setOperationAction(ISD::FADD,               MVT::v8f32, Legal);
877    setOperationAction(ISD::FSUB,               MVT::v8f32, Legal);
878    setOperationAction(ISD::FMUL,               MVT::v8f32, Legal);
879    setOperationAction(ISD::FDIV,               MVT::v8f32, Legal);
880    setOperationAction(ISD::FSQRT,              MVT::v8f32, Legal);
881    setOperationAction(ISD::FNEG,               MVT::v8f32, Custom);
882    //setOperationAction(ISD::BUILD_VECTOR,       MVT::v8f32, Custom);
883    //setOperationAction(ISD::VECTOR_SHUFFLE,     MVT::v8f32, Custom);
884    //setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v8f32, Custom);
885    //setOperationAction(ISD::SELECT,             MVT::v8f32, Custom);
886    //setOperationAction(ISD::VSETCC,             MVT::v8f32, Custom);
887
888    // Operations to consider commented out -v16i16 v32i8
889    //setOperationAction(ISD::ADD,                MVT::v16i16, Legal);
890    setOperationAction(ISD::ADD,                MVT::v8i32, Custom);
891    setOperationAction(ISD::ADD,                MVT::v4i64, Custom);
892    //setOperationAction(ISD::SUB,                MVT::v32i8, Legal);
893    //setOperationAction(ISD::SUB,                MVT::v16i16, Legal);
894    setOperationAction(ISD::SUB,                MVT::v8i32, Custom);
895    setOperationAction(ISD::SUB,                MVT::v4i64, Custom);
896    //setOperationAction(ISD::MUL,                MVT::v16i16, Legal);
897    setOperationAction(ISD::FADD,               MVT::v4f64, Legal);
898    setOperationAction(ISD::FSUB,               MVT::v4f64, Legal);
899    setOperationAction(ISD::FMUL,               MVT::v4f64, Legal);
900    setOperationAction(ISD::FDIV,               MVT::v4f64, Legal);
901    setOperationAction(ISD::FSQRT,              MVT::v4f64, Legal);
902    setOperationAction(ISD::FNEG,               MVT::v4f64, Custom);
903
904    setOperationAction(ISD::VSETCC,             MVT::v4f64, Custom);
905    // setOperationAction(ISD::VSETCC,             MVT::v32i8, Custom);
906    // setOperationAction(ISD::VSETCC,             MVT::v16i16, Custom);
907    setOperationAction(ISD::VSETCC,             MVT::v8i32, Custom);
908
909    // setOperationAction(ISD::SCALAR_TO_VECTOR,   MVT::v32i8, Custom);
910    // setOperationAction(ISD::SCALAR_TO_VECTOR,   MVT::v16i16, Custom);
911    // setOperationAction(ISD::INSERT_VECTOR_ELT,  MVT::v16i16, Custom);
912    setOperationAction(ISD::INSERT_VECTOR_ELT,  MVT::v8i32, Custom);
913    setOperationAction(ISD::INSERT_VECTOR_ELT,  MVT::v8f32, Custom);
914
915    setOperationAction(ISD::BUILD_VECTOR,       MVT::v4f64, Custom);
916    setOperationAction(ISD::BUILD_VECTOR,       MVT::v4i64, Custom);
917    setOperationAction(ISD::VECTOR_SHUFFLE,     MVT::v4f64, Custom);
918    setOperationAction(ISD::VECTOR_SHUFFLE,     MVT::v4i64, Custom);
919    setOperationAction(ISD::INSERT_VECTOR_ELT,  MVT::v4f64, Custom);
920    setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v4f64, Custom);
921
922#if 0
923    // Not sure we want to do this since there are no 256-bit integer
924    // operations in AVX
925
926    // Custom lower build_vector, vector_shuffle, and extract_vector_elt.
927    // This includes 256-bit vectors
928    for (unsigned i = (unsigned)MVT::v16i8; i != (unsigned)MVT::v4i64; ++i) {
929      EVT VT = (MVT::SimpleValueType)i;
930
931      // Do not attempt to custom lower non-power-of-2 vectors
932      if (!isPowerOf2_32(VT.getVectorNumElements()))
933        continue;
934
935      setOperationAction(ISD::BUILD_VECTOR,       VT, Custom);
936      setOperationAction(ISD::VECTOR_SHUFFLE,     VT, Custom);
937      setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
938    }
939
940    if (Subtarget->is64Bit()) {
941      setOperationAction(ISD::INSERT_VECTOR_ELT,  MVT::v4i64, Custom);
942      setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v4i64, Custom);
943    }
944#endif
945
946#if 0
947    // Not sure we want to do this since there are no 256-bit integer
948    // operations in AVX
949
950    // Promote v32i8, v16i16, v8i32 load, select, and, or, xor to v4i64.
951    // Including 256-bit vectors
952    for (unsigned i = (unsigned)MVT::v16i8; i != (unsigned)MVT::v4i64; i++) {
953      EVT VT = (MVT::SimpleValueType)i;
954
955      if (!VT.is256BitVector()) {
956        continue;
957      }
958      setOperationAction(ISD::AND,    VT, Promote);
959      AddPromotedToType (ISD::AND,    VT, MVT::v4i64);
960      setOperationAction(ISD::OR,     VT, Promote);
961      AddPromotedToType (ISD::OR,     VT, MVT::v4i64);
962      setOperationAction(ISD::XOR,    VT, Promote);
963      AddPromotedToType (ISD::XOR,    VT, MVT::v4i64);
964      setOperationAction(ISD::LOAD,   VT, Promote);
965      AddPromotedToType (ISD::LOAD,   VT, MVT::v4i64);
966      setOperationAction(ISD::SELECT, VT, Promote);
967      AddPromotedToType (ISD::SELECT, VT, MVT::v4i64);
968    }
969
970    setTruncStoreAction(MVT::f64, MVT::f32, Expand);
971#endif
972  }
973
974  // We want to custom lower some of our intrinsics.
975  setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::Other, Custom);
976
977  // Add/Sub/Mul with overflow operations are custom lowered.
978  setOperationAction(ISD::SADDO, MVT::i32, Custom);
979  setOperationAction(ISD::UADDO, MVT::i32, Custom);
980  setOperationAction(ISD::SSUBO, MVT::i32, Custom);
981  setOperationAction(ISD::USUBO, MVT::i32, Custom);
982  setOperationAction(ISD::SMULO, MVT::i32, Custom);
983
984  // Only custom-lower 64-bit SADDO and friends on 64-bit because we don't
985  // handle type legalization for these operations here.
986  //
987  // FIXME: We really should do custom legalization for addition and
988  // subtraction on x86-32 once PR3203 is fixed.  We really can't do much better
989  // than generic legalization for 64-bit multiplication-with-overflow, though.
990  if (Subtarget->is64Bit()) {
991    setOperationAction(ISD::SADDO, MVT::i64, Custom);
992    setOperationAction(ISD::UADDO, MVT::i64, Custom);
993    setOperationAction(ISD::SSUBO, MVT::i64, Custom);
994    setOperationAction(ISD::USUBO, MVT::i64, Custom);
995    setOperationAction(ISD::SMULO, MVT::i64, Custom);
996  }
997
998  if (!Subtarget->is64Bit()) {
999    // These libcalls are not available in 32-bit.
1000    setLibcallName(RTLIB::SHL_I128, 0);
1001    setLibcallName(RTLIB::SRL_I128, 0);
1002    setLibcallName(RTLIB::SRA_I128, 0);
1003  }
1004
1005  // We have target-specific dag combine patterns for the following nodes:
1006  setTargetDAGCombine(ISD::VECTOR_SHUFFLE);
1007  setTargetDAGCombine(ISD::EXTRACT_VECTOR_ELT);
1008  setTargetDAGCombine(ISD::BUILD_VECTOR);
1009  setTargetDAGCombine(ISD::SELECT);
1010  setTargetDAGCombine(ISD::SHL);
1011  setTargetDAGCombine(ISD::SRA);
1012  setTargetDAGCombine(ISD::SRL);
1013  setTargetDAGCombine(ISD::OR);
1014  setTargetDAGCombine(ISD::STORE);
1015  setTargetDAGCombine(ISD::MEMBARRIER);
1016  setTargetDAGCombine(ISD::ZERO_EXTEND);
1017  if (Subtarget->is64Bit())
1018    setTargetDAGCombine(ISD::MUL);
1019
1020  computeRegisterProperties();
1021
1022  // FIXME: These should be based on subtarget info. Plus, the values should
1023  // be smaller when we are in optimizing for size mode.
1024  maxStoresPerMemset = 16; // For @llvm.memset -> sequence of stores
1025  maxStoresPerMemcpy = 8; // For @llvm.memcpy -> sequence of stores
1026  maxStoresPerMemmove = 3; // For @llvm.memmove -> sequence of stores
1027  setPrefLoopAlignment(16);
1028  benefitFromCodePlacementOpt = true;
1029}
1030
1031
1032MVT::SimpleValueType X86TargetLowering::getSetCCResultType(EVT VT) const {
1033  return MVT::i8;
1034}
1035
1036
1037/// getMaxByValAlign - Helper for getByValTypeAlignment to determine
1038/// the desired ByVal argument alignment.
1039static void getMaxByValAlign(const Type *Ty, unsigned &MaxAlign) {
1040  if (MaxAlign == 16)
1041    return;
1042  if (const VectorType *VTy = dyn_cast<VectorType>(Ty)) {
1043    if (VTy->getBitWidth() == 128)
1044      MaxAlign = 16;
1045  } else if (const ArrayType *ATy = dyn_cast<ArrayType>(Ty)) {
1046    unsigned EltAlign = 0;
1047    getMaxByValAlign(ATy->getElementType(), EltAlign);
1048    if (EltAlign > MaxAlign)
1049      MaxAlign = EltAlign;
1050  } else if (const StructType *STy = dyn_cast<StructType>(Ty)) {
1051    for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) {
1052      unsigned EltAlign = 0;
1053      getMaxByValAlign(STy->getElementType(i), EltAlign);
1054      if (EltAlign > MaxAlign)
1055        MaxAlign = EltAlign;
1056      if (MaxAlign == 16)
1057        break;
1058    }
1059  }
1060  return;
1061}
1062
1063/// getByValTypeAlignment - Return the desired alignment for ByVal aggregate
1064/// function arguments in the caller parameter area. For X86, aggregates
1065/// that contain SSE vectors are placed at 16-byte boundaries while the rest
1066/// are at 4-byte boundaries.
1067unsigned X86TargetLowering::getByValTypeAlignment(const Type *Ty) const {
1068  if (Subtarget->is64Bit()) {
1069    // Max of 8 and alignment of type.
1070    unsigned TyAlign = TD->getABITypeAlignment(Ty);
1071    if (TyAlign > 8)
1072      return TyAlign;
1073    return 8;
1074  }
1075
1076  unsigned Align = 4;
1077  if (Subtarget->hasSSE1())
1078    getMaxByValAlign(Ty, Align);
1079  return Align;
1080}
1081
1082/// getOptimalMemOpType - Returns the target specific optimal type for load
1083/// and store operations as a result of memset, memcpy, and memmove
1084/// lowering. If DstAlign is zero that means it's safe to destination
1085/// alignment can satisfy any constraint. Similarly if SrcAlign is zero it
1086/// means there isn't a need to check it against alignment requirement,
1087/// probably because the source does not need to be loaded. If
1088/// 'NonScalarIntSafe' is true, that means it's safe to return a
1089/// non-scalar-integer type, e.g. empty string source, constant, or loaded
1090/// from memory. 'MemcpyStrSrc' indicates whether the memcpy source is
1091/// constant so it does not need to be loaded.
1092/// It returns EVT::Other if the type should be determined using generic
1093/// target-independent logic.
1094EVT
1095X86TargetLowering::getOptimalMemOpType(uint64_t Size,
1096                                       unsigned DstAlign, unsigned SrcAlign,
1097                                       bool NonScalarIntSafe,
1098                                       bool MemcpyStrSrc,
1099                                       MachineFunction &MF) const {
1100  // FIXME: This turns off use of xmm stores for memset/memcpy on targets like
1101  // linux.  This is because the stack realignment code can't handle certain
1102  // cases like PR2962.  This should be removed when PR2962 is fixed.
1103  const Function *F = MF.getFunction();
1104  if (NonScalarIntSafe &&
1105      !F->hasFnAttr(Attribute::NoImplicitFloat)) {
1106    if (Size >= 16 &&
1107        (Subtarget->isUnalignedMemAccessFast() ||
1108         ((DstAlign == 0 || DstAlign >= 16) &&
1109          (SrcAlign == 0 || SrcAlign >= 16))) &&
1110        Subtarget->getStackAlignment() >= 16) {
1111      if (Subtarget->hasSSE2())
1112        return MVT::v4i32;
1113      if (Subtarget->hasSSE1())
1114        return MVT::v4f32;
1115    } else if (!MemcpyStrSrc && Size >= 8 &&
1116               !Subtarget->is64Bit() &&
1117               Subtarget->getStackAlignment() >= 8 &&
1118               Subtarget->hasSSE2()) {
1119      // Do not use f64 to lower memcpy if source is string constant. It's
1120      // better to use i32 to avoid the loads.
1121      return MVT::f64;
1122    }
1123  }
1124  if (Subtarget->is64Bit() && Size >= 8)
1125    return MVT::i64;
1126  return MVT::i32;
1127}
1128
1129/// getJumpTableEncoding - Return the entry encoding for a jump table in the
1130/// current function.  The returned value is a member of the
1131/// MachineJumpTableInfo::JTEntryKind enum.
1132unsigned X86TargetLowering::getJumpTableEncoding() const {
1133  // In GOT pic mode, each entry in the jump table is emitted as a @GOTOFF
1134  // symbol.
1135  if (getTargetMachine().getRelocationModel() == Reloc::PIC_ &&
1136      Subtarget->isPICStyleGOT())
1137    return MachineJumpTableInfo::EK_Custom32;
1138
1139  // Otherwise, use the normal jump table encoding heuristics.
1140  return TargetLowering::getJumpTableEncoding();
1141}
1142
1143/// getPICBaseSymbol - Return the X86-32 PIC base.
1144MCSymbol *
1145X86TargetLowering::getPICBaseSymbol(const MachineFunction *MF,
1146                                    MCContext &Ctx) const {
1147  const MCAsmInfo &MAI = *getTargetMachine().getMCAsmInfo();
1148  return Ctx.GetOrCreateSymbol(Twine(MAI.getPrivateGlobalPrefix())+
1149                               Twine(MF->getFunctionNumber())+"$pb");
1150}
1151
1152
1153const MCExpr *
1154X86TargetLowering::LowerCustomJumpTableEntry(const MachineJumpTableInfo *MJTI,
1155                                             const MachineBasicBlock *MBB,
1156                                             unsigned uid,MCContext &Ctx) const{
1157  assert(getTargetMachine().getRelocationModel() == Reloc::PIC_ &&
1158         Subtarget->isPICStyleGOT());
1159  // In 32-bit ELF systems, our jump table entries are formed with @GOTOFF
1160  // entries.
1161  return MCSymbolRefExpr::Create(MBB->getSymbol(),
1162                                 MCSymbolRefExpr::VK_GOTOFF, Ctx);
1163}
1164
1165/// getPICJumpTableRelocaBase - Returns relocation base for the given PIC
1166/// jumptable.
1167SDValue X86TargetLowering::getPICJumpTableRelocBase(SDValue Table,
1168                                                    SelectionDAG &DAG) const {
1169  if (!Subtarget->is64Bit())
1170    // This doesn't have DebugLoc associated with it, but is not really the
1171    // same as a Register.
1172    return DAG.getNode(X86ISD::GlobalBaseReg, DebugLoc(), getPointerTy());
1173  return Table;
1174}
1175
1176/// getPICJumpTableRelocBaseExpr - This returns the relocation base for the
1177/// given PIC jumptable, the same as getPICJumpTableRelocBase, but as an
1178/// MCExpr.
1179const MCExpr *X86TargetLowering::
1180getPICJumpTableRelocBaseExpr(const MachineFunction *MF, unsigned JTI,
1181                             MCContext &Ctx) const {
1182  // X86-64 uses RIP relative addressing based on the jump table label.
1183  if (Subtarget->isPICStyleRIPRel())
1184    return TargetLowering::getPICJumpTableRelocBaseExpr(MF, JTI, Ctx);
1185
1186  // Otherwise, the reference is relative to the PIC base.
1187  return MCSymbolRefExpr::Create(getPICBaseSymbol(MF, Ctx), Ctx);
1188}
1189
1190/// getFunctionAlignment - Return the Log2 alignment of this function.
1191unsigned X86TargetLowering::getFunctionAlignment(const Function *F) const {
1192  return F->hasFnAttr(Attribute::OptimizeForSize) ? 0 : 4;
1193}
1194
1195//===----------------------------------------------------------------------===//
1196//               Return Value Calling Convention Implementation
1197//===----------------------------------------------------------------------===//
1198
1199#include "X86GenCallingConv.inc"
1200
1201bool
1202X86TargetLowering::CanLowerReturn(CallingConv::ID CallConv, bool isVarArg,
1203                        const SmallVectorImpl<EVT> &OutTys,
1204                        const SmallVectorImpl<ISD::ArgFlagsTy> &ArgsFlags,
1205                        SelectionDAG &DAG) const {
1206  SmallVector<CCValAssign, 16> RVLocs;
1207  CCState CCInfo(CallConv, isVarArg, getTargetMachine(),
1208                 RVLocs, *DAG.getContext());
1209  return CCInfo.CheckReturn(OutTys, ArgsFlags, RetCC_X86);
1210}
1211
1212SDValue
1213X86TargetLowering::LowerReturn(SDValue Chain,
1214                               CallingConv::ID CallConv, bool isVarArg,
1215                               const SmallVectorImpl<ISD::OutputArg> &Outs,
1216                               DebugLoc dl, SelectionDAG &DAG) const {
1217  MachineFunction &MF = DAG.getMachineFunction();
1218  X86MachineFunctionInfo *FuncInfo = MF.getInfo<X86MachineFunctionInfo>();
1219
1220  SmallVector<CCValAssign, 16> RVLocs;
1221  CCState CCInfo(CallConv, isVarArg, getTargetMachine(),
1222                 RVLocs, *DAG.getContext());
1223  CCInfo.AnalyzeReturn(Outs, RetCC_X86);
1224
1225  // Add the regs to the liveout set for the function.
1226  MachineRegisterInfo &MRI = DAG.getMachineFunction().getRegInfo();
1227  for (unsigned i = 0; i != RVLocs.size(); ++i)
1228    if (RVLocs[i].isRegLoc() && !MRI.isLiveOut(RVLocs[i].getLocReg()))
1229      MRI.addLiveOut(RVLocs[i].getLocReg());
1230
1231  SDValue Flag;
1232
1233  SmallVector<SDValue, 6> RetOps;
1234  RetOps.push_back(Chain); // Operand #0 = Chain (updated below)
1235  // Operand #1 = Bytes To Pop
1236  RetOps.push_back(DAG.getTargetConstant(FuncInfo->getBytesToPopOnReturn(),
1237                   MVT::i16));
1238
1239  // Copy the result values into the output registers.
1240  for (unsigned i = 0; i != RVLocs.size(); ++i) {
1241    CCValAssign &VA = RVLocs[i];
1242    assert(VA.isRegLoc() && "Can only return in registers!");
1243    SDValue ValToCopy = Outs[i].Val;
1244
1245    // Returns in ST0/ST1 are handled specially: these are pushed as operands to
1246    // the RET instruction and handled by the FP Stackifier.
1247    if (VA.getLocReg() == X86::ST0 ||
1248        VA.getLocReg() == X86::ST1) {
1249      // If this is a copy from an xmm register to ST(0), use an FPExtend to
1250      // change the value to the FP stack register class.
1251      if (isScalarFPTypeInSSEReg(VA.getValVT()))
1252        ValToCopy = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f80, ValToCopy);
1253      RetOps.push_back(ValToCopy);
1254      // Don't emit a copytoreg.
1255      continue;
1256    }
1257
1258    // 64-bit vector (MMX) values are returned in XMM0 / XMM1 except for v1i64
1259    // which is returned in RAX / RDX.
1260    if (Subtarget->is64Bit()) {
1261      EVT ValVT = ValToCopy.getValueType();
1262      if (ValVT.isVector() && ValVT.getSizeInBits() == 64) {
1263        ValToCopy = DAG.getNode(ISD::BIT_CONVERT, dl, MVT::i64, ValToCopy);
1264        if (VA.getLocReg() == X86::XMM0 || VA.getLocReg() == X86::XMM1)
1265          ValToCopy = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v2i64, ValToCopy);
1266      }
1267    }
1268
1269    Chain = DAG.getCopyToReg(Chain, dl, VA.getLocReg(), ValToCopy, Flag);
1270    Flag = Chain.getValue(1);
1271  }
1272
1273  // The x86-64 ABI for returning structs by value requires that we copy
1274  // the sret argument into %rax for the return. We saved the argument into
1275  // a virtual register in the entry block, so now we copy the value out
1276  // and into %rax.
1277  if (Subtarget->is64Bit() &&
1278      DAG.getMachineFunction().getFunction()->hasStructRetAttr()) {
1279    MachineFunction &MF = DAG.getMachineFunction();
1280    X86MachineFunctionInfo *FuncInfo = MF.getInfo<X86MachineFunctionInfo>();
1281    unsigned Reg = FuncInfo->getSRetReturnReg();
1282    assert(Reg &&
1283           "SRetReturnReg should have been set in LowerFormalArguments().");
1284    SDValue Val = DAG.getCopyFromReg(Chain, dl, Reg, getPointerTy());
1285
1286    Chain = DAG.getCopyToReg(Chain, dl, X86::RAX, Val, Flag);
1287    Flag = Chain.getValue(1);
1288
1289    // RAX now acts like a return value.
1290    MRI.addLiveOut(X86::RAX);
1291  }
1292
1293  RetOps[0] = Chain;  // Update chain.
1294
1295  // Add the flag if we have it.
1296  if (Flag.getNode())
1297    RetOps.push_back(Flag);
1298
1299  return DAG.getNode(X86ISD::RET_FLAG, dl,
1300                     MVT::Other, &RetOps[0], RetOps.size());
1301}
1302
1303/// LowerCallResult - Lower the result values of a call into the
1304/// appropriate copies out of appropriate physical registers.
1305///
1306SDValue
1307X86TargetLowering::LowerCallResult(SDValue Chain, SDValue InFlag,
1308                                   CallingConv::ID CallConv, bool isVarArg,
1309                                   const SmallVectorImpl<ISD::InputArg> &Ins,
1310                                   DebugLoc dl, SelectionDAG &DAG,
1311                                   SmallVectorImpl<SDValue> &InVals) const {
1312
1313  // Assign locations to each value returned by this call.
1314  SmallVector<CCValAssign, 16> RVLocs;
1315  bool Is64Bit = Subtarget->is64Bit();
1316  CCState CCInfo(CallConv, isVarArg, getTargetMachine(),
1317                 RVLocs, *DAG.getContext());
1318  CCInfo.AnalyzeCallResult(Ins, RetCC_X86);
1319
1320  // Copy all of the result registers out of their specified physreg.
1321  for (unsigned i = 0; i != RVLocs.size(); ++i) {
1322    CCValAssign &VA = RVLocs[i];
1323    EVT CopyVT = VA.getValVT();
1324
1325    // If this is x86-64, and we disabled SSE, we can't return FP values
1326    if ((CopyVT == MVT::f32 || CopyVT == MVT::f64) &&
1327        ((Is64Bit || Ins[i].Flags.isInReg()) && !Subtarget->hasSSE1())) {
1328      report_fatal_error("SSE register return with SSE disabled");
1329    }
1330
1331    // If this is a call to a function that returns an fp value on the floating
1332    // point stack, but where we prefer to use the value in xmm registers, copy
1333    // it out as F80 and use a truncate to move it from fp stack reg to xmm reg.
1334    if ((VA.getLocReg() == X86::ST0 ||
1335         VA.getLocReg() == X86::ST1) &&
1336        isScalarFPTypeInSSEReg(VA.getValVT())) {
1337      CopyVT = MVT::f80;
1338    }
1339
1340    SDValue Val;
1341    if (Is64Bit && CopyVT.isVector() && CopyVT.getSizeInBits() == 64) {
1342      // For x86-64, MMX values are returned in XMM0 / XMM1 except for v1i64.
1343      if (VA.getLocReg() == X86::XMM0 || VA.getLocReg() == X86::XMM1) {
1344        Chain = DAG.getCopyFromReg(Chain, dl, VA.getLocReg(),
1345                                   MVT::v2i64, InFlag).getValue(1);
1346        Val = Chain.getValue(0);
1347        Val = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i64,
1348                          Val, DAG.getConstant(0, MVT::i64));
1349      } else {
1350        Chain = DAG.getCopyFromReg(Chain, dl, VA.getLocReg(),
1351                                   MVT::i64, InFlag).getValue(1);
1352        Val = Chain.getValue(0);
1353      }
1354      Val = DAG.getNode(ISD::BIT_CONVERT, dl, CopyVT, Val);
1355    } else {
1356      Chain = DAG.getCopyFromReg(Chain, dl, VA.getLocReg(),
1357                                 CopyVT, InFlag).getValue(1);
1358      Val = Chain.getValue(0);
1359    }
1360    InFlag = Chain.getValue(2);
1361
1362    if (CopyVT != VA.getValVT()) {
1363      // Round the F80 the right size, which also moves to the appropriate xmm
1364      // register.
1365      Val = DAG.getNode(ISD::FP_ROUND, dl, VA.getValVT(), Val,
1366                        // This truncation won't change the value.
1367                        DAG.getIntPtrConstant(1));
1368    }
1369
1370    InVals.push_back(Val);
1371  }
1372
1373  return Chain;
1374}
1375
1376
1377//===----------------------------------------------------------------------===//
1378//                C & StdCall & Fast Calling Convention implementation
1379//===----------------------------------------------------------------------===//
1380//  StdCall calling convention seems to be standard for many Windows' API
1381//  routines and around. It differs from C calling convention just a little:
1382//  callee should clean up the stack, not caller. Symbols should be also
1383//  decorated in some fancy way :) It doesn't support any vector arguments.
1384//  For info on fast calling convention see Fast Calling Convention (tail call)
1385//  implementation LowerX86_32FastCCCallTo.
1386
1387/// CallIsStructReturn - Determines whether a call uses struct return
1388/// semantics.
1389static bool CallIsStructReturn(const SmallVectorImpl<ISD::OutputArg> &Outs) {
1390  if (Outs.empty())
1391    return false;
1392
1393  return Outs[0].Flags.isSRet();
1394}
1395
1396/// ArgsAreStructReturn - Determines whether a function uses struct
1397/// return semantics.
1398static bool
1399ArgsAreStructReturn(const SmallVectorImpl<ISD::InputArg> &Ins) {
1400  if (Ins.empty())
1401    return false;
1402
1403  return Ins[0].Flags.isSRet();
1404}
1405
1406/// CCAssignFnForNode - Selects the correct CCAssignFn for a the
1407/// given CallingConvention value.
1408CCAssignFn *X86TargetLowering::CCAssignFnForNode(CallingConv::ID CC) const {
1409  if (Subtarget->is64Bit()) {
1410    if (CC == CallingConv::GHC)
1411      return CC_X86_64_GHC;
1412    else if (Subtarget->isTargetWin64())
1413      return CC_X86_Win64_C;
1414    else
1415      return CC_X86_64_C;
1416  }
1417
1418  if (CC == CallingConv::X86_FastCall)
1419    return CC_X86_32_FastCall;
1420  else if (CC == CallingConv::X86_ThisCall)
1421    return CC_X86_32_ThisCall;
1422  else if (CC == CallingConv::Fast)
1423    return CC_X86_32_FastCC;
1424  else if (CC == CallingConv::GHC)
1425    return CC_X86_32_GHC;
1426  else
1427    return CC_X86_32_C;
1428}
1429
1430/// CreateCopyOfByValArgument - Make a copy of an aggregate at address specified
1431/// by "Src" to address "Dst" with size and alignment information specified by
1432/// the specific parameter attribute. The copy will be passed as a byval
1433/// function parameter.
1434static SDValue
1435CreateCopyOfByValArgument(SDValue Src, SDValue Dst, SDValue Chain,
1436                          ISD::ArgFlagsTy Flags, SelectionDAG &DAG,
1437                          DebugLoc dl) {
1438  SDValue SizeNode     = DAG.getConstant(Flags.getByValSize(), MVT::i32);
1439  return DAG.getMemcpy(Chain, dl, Dst, Src, SizeNode, Flags.getByValAlign(),
1440                       /*isVolatile*/false, /*AlwaysInline=*/true,
1441                       NULL, 0, NULL, 0);
1442}
1443
1444/// IsTailCallConvention - Return true if the calling convention is one that
1445/// supports tail call optimization.
1446static bool IsTailCallConvention(CallingConv::ID CC) {
1447  return (CC == CallingConv::Fast || CC == CallingConv::GHC);
1448}
1449
1450/// FuncIsMadeTailCallSafe - Return true if the function is being made into
1451/// a tailcall target by changing its ABI.
1452static bool FuncIsMadeTailCallSafe(CallingConv::ID CC) {
1453  return GuaranteedTailCallOpt && IsTailCallConvention(CC);
1454}
1455
1456SDValue
1457X86TargetLowering::LowerMemArgument(SDValue Chain,
1458                                    CallingConv::ID CallConv,
1459                                    const SmallVectorImpl<ISD::InputArg> &Ins,
1460                                    DebugLoc dl, SelectionDAG &DAG,
1461                                    const CCValAssign &VA,
1462                                    MachineFrameInfo *MFI,
1463                                    unsigned i) const {
1464  // Create the nodes corresponding to a load from this parameter slot.
1465  ISD::ArgFlagsTy Flags = Ins[i].Flags;
1466  bool AlwaysUseMutable = FuncIsMadeTailCallSafe(CallConv);
1467  bool isImmutable = !AlwaysUseMutable && !Flags.isByVal();
1468  EVT ValVT;
1469
1470  // If value is passed by pointer we have address passed instead of the value
1471  // itself.
1472  if (VA.getLocInfo() == CCValAssign::Indirect)
1473    ValVT = VA.getLocVT();
1474  else
1475    ValVT = VA.getValVT();
1476
1477  // FIXME: For now, all byval parameter objects are marked mutable. This can be
1478  // changed with more analysis.
1479  // In case of tail call optimization mark all arguments mutable. Since they
1480  // could be overwritten by lowering of arguments in case of a tail call.
1481  if (Flags.isByVal()) {
1482    int FI = MFI->CreateFixedObject(Flags.getByValSize(),
1483                                    VA.getLocMemOffset(), isImmutable, false);
1484    return DAG.getFrameIndex(FI, getPointerTy());
1485  } else {
1486    int FI = MFI->CreateFixedObject(ValVT.getSizeInBits()/8,
1487                                    VA.getLocMemOffset(), isImmutable, false);
1488    SDValue FIN = DAG.getFrameIndex(FI, getPointerTy());
1489    return DAG.getLoad(ValVT, dl, Chain, FIN,
1490                       PseudoSourceValue::getFixedStack(FI), 0,
1491                       false, false, 0);
1492  }
1493}
1494
1495SDValue
1496X86TargetLowering::LowerFormalArguments(SDValue Chain,
1497                                        CallingConv::ID CallConv,
1498                                        bool isVarArg,
1499                                      const SmallVectorImpl<ISD::InputArg> &Ins,
1500                                        DebugLoc dl,
1501                                        SelectionDAG &DAG,
1502                                        SmallVectorImpl<SDValue> &InVals)
1503                                          const {
1504  MachineFunction &MF = DAG.getMachineFunction();
1505  X86MachineFunctionInfo *FuncInfo = MF.getInfo<X86MachineFunctionInfo>();
1506
1507  const Function* Fn = MF.getFunction();
1508  if (Fn->hasExternalLinkage() &&
1509      Subtarget->isTargetCygMing() &&
1510      Fn->getName() == "main")
1511    FuncInfo->setForceFramePointer(true);
1512
1513  MachineFrameInfo *MFI = MF.getFrameInfo();
1514  bool Is64Bit = Subtarget->is64Bit();
1515  bool IsWin64 = Subtarget->isTargetWin64();
1516
1517  assert(!(isVarArg && IsTailCallConvention(CallConv)) &&
1518         "Var args not supported with calling convention fastcc or ghc");
1519
1520  // Assign locations to all of the incoming arguments.
1521  SmallVector<CCValAssign, 16> ArgLocs;
1522  CCState CCInfo(CallConv, isVarArg, getTargetMachine(),
1523                 ArgLocs, *DAG.getContext());
1524  CCInfo.AnalyzeFormalArguments(Ins, CCAssignFnForNode(CallConv));
1525
1526  unsigned LastVal = ~0U;
1527  SDValue ArgValue;
1528  for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) {
1529    CCValAssign &VA = ArgLocs[i];
1530    // TODO: If an arg is passed in two places (e.g. reg and stack), skip later
1531    // places.
1532    assert(VA.getValNo() != LastVal &&
1533           "Don't support value assigned to multiple locs yet");
1534    LastVal = VA.getValNo();
1535
1536    if (VA.isRegLoc()) {
1537      EVT RegVT = VA.getLocVT();
1538      TargetRegisterClass *RC = NULL;
1539      if (RegVT == MVT::i32)
1540        RC = X86::GR32RegisterClass;
1541      else if (Is64Bit && RegVT == MVT::i64)
1542        RC = X86::GR64RegisterClass;
1543      else if (RegVT == MVT::f32)
1544        RC = X86::FR32RegisterClass;
1545      else if (RegVT == MVT::f64)
1546        RC = X86::FR64RegisterClass;
1547      else if (RegVT.isVector() && RegVT.getSizeInBits() == 128)
1548        RC = X86::VR128RegisterClass;
1549      else if (RegVT.isVector() && RegVT.getSizeInBits() == 64)
1550        RC = X86::VR64RegisterClass;
1551      else
1552        llvm_unreachable("Unknown argument type!");
1553
1554      unsigned Reg = MF.addLiveIn(VA.getLocReg(), RC);
1555      ArgValue = DAG.getCopyFromReg(Chain, dl, Reg, RegVT);
1556
1557      // If this is an 8 or 16-bit value, it is really passed promoted to 32
1558      // bits.  Insert an assert[sz]ext to capture this, then truncate to the
1559      // right size.
1560      if (VA.getLocInfo() == CCValAssign::SExt)
1561        ArgValue = DAG.getNode(ISD::AssertSext, dl, RegVT, ArgValue,
1562                               DAG.getValueType(VA.getValVT()));
1563      else if (VA.getLocInfo() == CCValAssign::ZExt)
1564        ArgValue = DAG.getNode(ISD::AssertZext, dl, RegVT, ArgValue,
1565                               DAG.getValueType(VA.getValVT()));
1566      else if (VA.getLocInfo() == CCValAssign::BCvt)
1567        ArgValue = DAG.getNode(ISD::BIT_CONVERT, dl, VA.getValVT(), ArgValue);
1568
1569      if (VA.isExtInLoc()) {
1570        // Handle MMX values passed in XMM regs.
1571        if (RegVT.isVector()) {
1572          ArgValue = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i64,
1573                                 ArgValue, DAG.getConstant(0, MVT::i64));
1574          ArgValue = DAG.getNode(ISD::BIT_CONVERT, dl, VA.getValVT(), ArgValue);
1575        } else
1576          ArgValue = DAG.getNode(ISD::TRUNCATE, dl, VA.getValVT(), ArgValue);
1577      }
1578    } else {
1579      assert(VA.isMemLoc());
1580      ArgValue = LowerMemArgument(Chain, CallConv, Ins, dl, DAG, VA, MFI, i);
1581    }
1582
1583    // If value is passed via pointer - do a load.
1584    if (VA.getLocInfo() == CCValAssign::Indirect)
1585      ArgValue = DAG.getLoad(VA.getValVT(), dl, Chain, ArgValue, NULL, 0,
1586                             false, false, 0);
1587
1588    InVals.push_back(ArgValue);
1589  }
1590
1591  // The x86-64 ABI for returning structs by value requires that we copy
1592  // the sret argument into %rax for the return. Save the argument into
1593  // a virtual register so that we can access it from the return points.
1594  if (Is64Bit && MF.getFunction()->hasStructRetAttr()) {
1595    X86MachineFunctionInfo *FuncInfo = MF.getInfo<X86MachineFunctionInfo>();
1596    unsigned Reg = FuncInfo->getSRetReturnReg();
1597    if (!Reg) {
1598      Reg = MF.getRegInfo().createVirtualRegister(getRegClassFor(MVT::i64));
1599      FuncInfo->setSRetReturnReg(Reg);
1600    }
1601    SDValue Copy = DAG.getCopyToReg(DAG.getEntryNode(), dl, Reg, InVals[0]);
1602    Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Copy, Chain);
1603  }
1604
1605  unsigned StackSize = CCInfo.getNextStackOffset();
1606  // Align stack specially for tail calls.
1607  if (FuncIsMadeTailCallSafe(CallConv))
1608    StackSize = GetAlignedArgumentStackSize(StackSize, DAG);
1609
1610  // If the function takes variable number of arguments, make a frame index for
1611  // the start of the first vararg value... for expansion of llvm.va_start.
1612  if (isVarArg) {
1613    if (Is64Bit || (CallConv != CallingConv::X86_FastCall &&
1614                    CallConv != CallingConv::X86_ThisCall)) {
1615      FuncInfo->setVarArgsFrameIndex(MFI->CreateFixedObject(1, StackSize,
1616                                                            true, false));
1617    }
1618    if (Is64Bit) {
1619      unsigned TotalNumIntRegs = 0, TotalNumXMMRegs = 0;
1620
1621      // FIXME: We should really autogenerate these arrays
1622      static const unsigned GPR64ArgRegsWin64[] = {
1623        X86::RCX, X86::RDX, X86::R8,  X86::R9
1624      };
1625      static const unsigned XMMArgRegsWin64[] = {
1626        X86::XMM0, X86::XMM1, X86::XMM2, X86::XMM3
1627      };
1628      static const unsigned GPR64ArgRegs64Bit[] = {
1629        X86::RDI, X86::RSI, X86::RDX, X86::RCX, X86::R8, X86::R9
1630      };
1631      static const unsigned XMMArgRegs64Bit[] = {
1632        X86::XMM0, X86::XMM1, X86::XMM2, X86::XMM3,
1633        X86::XMM4, X86::XMM5, X86::XMM6, X86::XMM7
1634      };
1635      const unsigned *GPR64ArgRegs, *XMMArgRegs;
1636
1637      if (IsWin64) {
1638        TotalNumIntRegs = 4; TotalNumXMMRegs = 4;
1639        GPR64ArgRegs = GPR64ArgRegsWin64;
1640        XMMArgRegs = XMMArgRegsWin64;
1641      } else {
1642        TotalNumIntRegs = 6; TotalNumXMMRegs = 8;
1643        GPR64ArgRegs = GPR64ArgRegs64Bit;
1644        XMMArgRegs = XMMArgRegs64Bit;
1645      }
1646      unsigned NumIntRegs = CCInfo.getFirstUnallocated(GPR64ArgRegs,
1647                                                       TotalNumIntRegs);
1648      unsigned NumXMMRegs = CCInfo.getFirstUnallocated(XMMArgRegs,
1649                                                       TotalNumXMMRegs);
1650
1651      bool NoImplicitFloatOps = Fn->hasFnAttr(Attribute::NoImplicitFloat);
1652      assert(!(NumXMMRegs && !Subtarget->hasSSE1()) &&
1653             "SSE register cannot be used when SSE is disabled!");
1654      assert(!(NumXMMRegs && UseSoftFloat && NoImplicitFloatOps) &&
1655             "SSE register cannot be used when SSE is disabled!");
1656      if (UseSoftFloat || NoImplicitFloatOps || !Subtarget->hasSSE1())
1657        // Kernel mode asks for SSE to be disabled, so don't push them
1658        // on the stack.
1659        TotalNumXMMRegs = 0;
1660
1661      // For X86-64, if there are vararg parameters that are passed via
1662      // registers, then we must store them to their spots on the stack so they
1663      // may be loaded by deferencing the result of va_next.
1664      FuncInfo->setVarArgsGPOffset(NumIntRegs * 8);
1665      FuncInfo->setVarArgsFPOffset(TotalNumIntRegs * 8 + NumXMMRegs * 16);
1666      FuncInfo->setRegSaveFrameIndex(
1667        MFI->CreateStackObject(TotalNumIntRegs * 8 + TotalNumXMMRegs * 16, 16,
1668                               false));
1669
1670      // Store the integer parameter registers.
1671      SmallVector<SDValue, 8> MemOps;
1672      SDValue RSFIN = DAG.getFrameIndex(FuncInfo->getRegSaveFrameIndex(),
1673                                        getPointerTy());
1674      unsigned Offset = FuncInfo->getVarArgsGPOffset();
1675      for (; NumIntRegs != TotalNumIntRegs; ++NumIntRegs) {
1676        SDValue FIN = DAG.getNode(ISD::ADD, dl, getPointerTy(), RSFIN,
1677                                  DAG.getIntPtrConstant(Offset));
1678        unsigned VReg = MF.addLiveIn(GPR64ArgRegs[NumIntRegs],
1679                                     X86::GR64RegisterClass);
1680        SDValue Val = DAG.getCopyFromReg(Chain, dl, VReg, MVT::i64);
1681        SDValue Store =
1682          DAG.getStore(Val.getValue(1), dl, Val, FIN,
1683                       PseudoSourceValue::getFixedStack(
1684                         FuncInfo->getRegSaveFrameIndex()),
1685                       Offset, false, false, 0);
1686        MemOps.push_back(Store);
1687        Offset += 8;
1688      }
1689
1690      if (TotalNumXMMRegs != 0 && NumXMMRegs != TotalNumXMMRegs) {
1691        // Now store the XMM (fp + vector) parameter registers.
1692        SmallVector<SDValue, 11> SaveXMMOps;
1693        SaveXMMOps.push_back(Chain);
1694
1695        unsigned AL = MF.addLiveIn(X86::AL, X86::GR8RegisterClass);
1696        SDValue ALVal = DAG.getCopyFromReg(DAG.getEntryNode(), dl, AL, MVT::i8);
1697        SaveXMMOps.push_back(ALVal);
1698
1699        SaveXMMOps.push_back(DAG.getIntPtrConstant(
1700                               FuncInfo->getRegSaveFrameIndex()));
1701        SaveXMMOps.push_back(DAG.getIntPtrConstant(
1702                               FuncInfo->getVarArgsFPOffset()));
1703
1704        for (; NumXMMRegs != TotalNumXMMRegs; ++NumXMMRegs) {
1705          unsigned VReg = MF.addLiveIn(XMMArgRegs[NumXMMRegs],
1706                                       X86::VR128RegisterClass);
1707          SDValue Val = DAG.getCopyFromReg(Chain, dl, VReg, MVT::v4f32);
1708          SaveXMMOps.push_back(Val);
1709        }
1710        MemOps.push_back(DAG.getNode(X86ISD::VASTART_SAVE_XMM_REGS, dl,
1711                                     MVT::Other,
1712                                     &SaveXMMOps[0], SaveXMMOps.size()));
1713      }
1714
1715      if (!MemOps.empty())
1716        Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
1717                            &MemOps[0], MemOps.size());
1718    }
1719  }
1720
1721  // Some CCs need callee pop.
1722  if (Subtarget->IsCalleePop(isVarArg, CallConv)) {
1723    FuncInfo->setBytesToPopOnReturn(StackSize); // Callee pops everything.
1724  } else {
1725    FuncInfo->setBytesToPopOnReturn(0); // Callee pops nothing.
1726    // If this is an sret function, the return should pop the hidden pointer.
1727    if (!Is64Bit && !IsTailCallConvention(CallConv) && ArgsAreStructReturn(Ins))
1728      FuncInfo->setBytesToPopOnReturn(4);
1729  }
1730
1731  if (!Is64Bit) {
1732    // RegSaveFrameIndex is X86-64 only.
1733    FuncInfo->setRegSaveFrameIndex(0xAAAAAAA);
1734    if (CallConv == CallingConv::X86_FastCall ||
1735        CallConv == CallingConv::X86_ThisCall)
1736      // fastcc functions can't have varargs.
1737      FuncInfo->setVarArgsFrameIndex(0xAAAAAAA);
1738  }
1739
1740  return Chain;
1741}
1742
1743SDValue
1744X86TargetLowering::LowerMemOpCallTo(SDValue Chain,
1745                                    SDValue StackPtr, SDValue Arg,
1746                                    DebugLoc dl, SelectionDAG &DAG,
1747                                    const CCValAssign &VA,
1748                                    ISD::ArgFlagsTy Flags) const {
1749  const unsigned FirstStackArgOffset = (Subtarget->isTargetWin64() ? 32 : 0);
1750  unsigned LocMemOffset = FirstStackArgOffset + VA.getLocMemOffset();
1751  SDValue PtrOff = DAG.getIntPtrConstant(LocMemOffset);
1752  PtrOff = DAG.getNode(ISD::ADD, dl, getPointerTy(), StackPtr, PtrOff);
1753  if (Flags.isByVal()) {
1754    return CreateCopyOfByValArgument(Arg, PtrOff, Chain, Flags, DAG, dl);
1755  }
1756  return DAG.getStore(Chain, dl, Arg, PtrOff,
1757                      PseudoSourceValue::getStack(), LocMemOffset,
1758                      false, false, 0);
1759}
1760
1761/// EmitTailCallLoadRetAddr - Emit a load of return address if tail call
1762/// optimization is performed and it is required.
1763SDValue
1764X86TargetLowering::EmitTailCallLoadRetAddr(SelectionDAG &DAG,
1765                                           SDValue &OutRetAddr, SDValue Chain,
1766                                           bool IsTailCall, bool Is64Bit,
1767                                           int FPDiff, DebugLoc dl) const {
1768  // Adjust the Return address stack slot.
1769  EVT VT = getPointerTy();
1770  OutRetAddr = getReturnAddressFrameIndex(DAG);
1771
1772  // Load the "old" Return address.
1773  OutRetAddr = DAG.getLoad(VT, dl, Chain, OutRetAddr, NULL, 0, false, false, 0);
1774  return SDValue(OutRetAddr.getNode(), 1);
1775}
1776
1777/// EmitTailCallStoreRetAddr - Emit a store of the return adress if tail call
1778/// optimization is performed and it is required (FPDiff!=0).
1779static SDValue
1780EmitTailCallStoreRetAddr(SelectionDAG & DAG, MachineFunction &MF,
1781                         SDValue Chain, SDValue RetAddrFrIdx,
1782                         bool Is64Bit, int FPDiff, DebugLoc dl) {
1783  // Store the return address to the appropriate stack slot.
1784  if (!FPDiff) return Chain;
1785  // Calculate the new stack slot for the return address.
1786  int SlotSize = Is64Bit ? 8 : 4;
1787  int NewReturnAddrFI =
1788    MF.getFrameInfo()->CreateFixedObject(SlotSize, FPDiff-SlotSize, false, false);
1789  EVT VT = Is64Bit ? MVT::i64 : MVT::i32;
1790  SDValue NewRetAddrFrIdx = DAG.getFrameIndex(NewReturnAddrFI, VT);
1791  Chain = DAG.getStore(Chain, dl, RetAddrFrIdx, NewRetAddrFrIdx,
1792                       PseudoSourceValue::getFixedStack(NewReturnAddrFI), 0,
1793                       false, false, 0);
1794  return Chain;
1795}
1796
1797SDValue
1798X86TargetLowering::LowerCall(SDValue Chain, SDValue Callee,
1799                             CallingConv::ID CallConv, bool isVarArg,
1800                             bool &isTailCall,
1801                             const SmallVectorImpl<ISD::OutputArg> &Outs,
1802                             const SmallVectorImpl<ISD::InputArg> &Ins,
1803                             DebugLoc dl, SelectionDAG &DAG,
1804                             SmallVectorImpl<SDValue> &InVals) const {
1805  MachineFunction &MF = DAG.getMachineFunction();
1806  bool Is64Bit        = Subtarget->is64Bit();
1807  bool IsStructRet    = CallIsStructReturn(Outs);
1808  bool IsSibcall      = false;
1809
1810  if (isTailCall) {
1811    // Check if it's really possible to do a tail call.
1812    isTailCall = IsEligibleForTailCallOptimization(Callee, CallConv,
1813                    isVarArg, IsStructRet, MF.getFunction()->hasStructRetAttr(),
1814                                                   Outs, Ins, DAG);
1815
1816    // Sibcalls are automatically detected tailcalls which do not require
1817    // ABI changes.
1818    if (!GuaranteedTailCallOpt && isTailCall)
1819      IsSibcall = true;
1820
1821    if (isTailCall)
1822      ++NumTailCalls;
1823  }
1824
1825  assert(!(isVarArg && IsTailCallConvention(CallConv)) &&
1826         "Var args not supported with calling convention fastcc or ghc");
1827
1828  // Analyze operands of the call, assigning locations to each operand.
1829  SmallVector<CCValAssign, 16> ArgLocs;
1830  CCState CCInfo(CallConv, isVarArg, getTargetMachine(),
1831                 ArgLocs, *DAG.getContext());
1832  CCInfo.AnalyzeCallOperands(Outs, CCAssignFnForNode(CallConv));
1833
1834  // Get a count of how many bytes are to be pushed on the stack.
1835  unsigned NumBytes = CCInfo.getNextStackOffset();
1836  if (IsSibcall)
1837    // This is a sibcall. The memory operands are available in caller's
1838    // own caller's stack.
1839    NumBytes = 0;
1840  else if (GuaranteedTailCallOpt && IsTailCallConvention(CallConv))
1841    NumBytes = GetAlignedArgumentStackSize(NumBytes, DAG);
1842
1843  int FPDiff = 0;
1844  if (isTailCall && !IsSibcall) {
1845    // Lower arguments at fp - stackoffset + fpdiff.
1846    unsigned NumBytesCallerPushed =
1847      MF.getInfo<X86MachineFunctionInfo>()->getBytesToPopOnReturn();
1848    FPDiff = NumBytesCallerPushed - NumBytes;
1849
1850    // Set the delta of movement of the returnaddr stackslot.
1851    // But only set if delta is greater than previous delta.
1852    if (FPDiff < (MF.getInfo<X86MachineFunctionInfo>()->getTCReturnAddrDelta()))
1853      MF.getInfo<X86MachineFunctionInfo>()->setTCReturnAddrDelta(FPDiff);
1854  }
1855
1856  if (!IsSibcall)
1857    Chain = DAG.getCALLSEQ_START(Chain, DAG.getIntPtrConstant(NumBytes, true));
1858
1859  SDValue RetAddrFrIdx;
1860  // Load return adress for tail calls.
1861  if (isTailCall && FPDiff)
1862    Chain = EmitTailCallLoadRetAddr(DAG, RetAddrFrIdx, Chain, isTailCall,
1863                                    Is64Bit, FPDiff, dl);
1864
1865  SmallVector<std::pair<unsigned, SDValue>, 8> RegsToPass;
1866  SmallVector<SDValue, 8> MemOpChains;
1867  SDValue StackPtr;
1868
1869  // Walk the register/memloc assignments, inserting copies/loads.  In the case
1870  // of tail call optimization arguments are handle later.
1871  for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) {
1872    CCValAssign &VA = ArgLocs[i];
1873    EVT RegVT = VA.getLocVT();
1874    SDValue Arg = Outs[i].Val;
1875    ISD::ArgFlagsTy Flags = Outs[i].Flags;
1876    bool isByVal = Flags.isByVal();
1877
1878    // Promote the value if needed.
1879    switch (VA.getLocInfo()) {
1880    default: llvm_unreachable("Unknown loc info!");
1881    case CCValAssign::Full: break;
1882    case CCValAssign::SExt:
1883      Arg = DAG.getNode(ISD::SIGN_EXTEND, dl, RegVT, Arg);
1884      break;
1885    case CCValAssign::ZExt:
1886      Arg = DAG.getNode(ISD::ZERO_EXTEND, dl, RegVT, Arg);
1887      break;
1888    case CCValAssign::AExt:
1889      if (RegVT.isVector() && RegVT.getSizeInBits() == 128) {
1890        // Special case: passing MMX values in XMM registers.
1891        Arg = DAG.getNode(ISD::BIT_CONVERT, dl, MVT::i64, Arg);
1892        Arg = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v2i64, Arg);
1893        Arg = getMOVL(DAG, dl, MVT::v2i64, DAG.getUNDEF(MVT::v2i64), Arg);
1894      } else
1895        Arg = DAG.getNode(ISD::ANY_EXTEND, dl, RegVT, Arg);
1896      break;
1897    case CCValAssign::BCvt:
1898      Arg = DAG.getNode(ISD::BIT_CONVERT, dl, RegVT, Arg);
1899      break;
1900    case CCValAssign::Indirect: {
1901      // Store the argument.
1902      SDValue SpillSlot = DAG.CreateStackTemporary(VA.getValVT());
1903      int FI = cast<FrameIndexSDNode>(SpillSlot)->getIndex();
1904      Chain = DAG.getStore(Chain, dl, Arg, SpillSlot,
1905                           PseudoSourceValue::getFixedStack(FI), 0,
1906                           false, false, 0);
1907      Arg = SpillSlot;
1908      break;
1909    }
1910    }
1911
1912    if (VA.isRegLoc()) {
1913      RegsToPass.push_back(std::make_pair(VA.getLocReg(), Arg));
1914    } else if (!IsSibcall && (!isTailCall || isByVal)) {
1915      assert(VA.isMemLoc());
1916      if (StackPtr.getNode() == 0)
1917        StackPtr = DAG.getCopyFromReg(Chain, dl, X86StackPtr, getPointerTy());
1918      MemOpChains.push_back(LowerMemOpCallTo(Chain, StackPtr, Arg,
1919                                             dl, DAG, VA, Flags));
1920    }
1921  }
1922
1923  if (!MemOpChains.empty())
1924    Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
1925                        &MemOpChains[0], MemOpChains.size());
1926
1927  // Build a sequence of copy-to-reg nodes chained together with token chain
1928  // and flag operands which copy the outgoing args into registers.
1929  SDValue InFlag;
1930  // Tail call byval lowering might overwrite argument registers so in case of
1931  // tail call optimization the copies to registers are lowered later.
1932  if (!isTailCall)
1933    for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i) {
1934      Chain = DAG.getCopyToReg(Chain, dl, RegsToPass[i].first,
1935                               RegsToPass[i].second, InFlag);
1936      InFlag = Chain.getValue(1);
1937    }
1938
1939  if (Subtarget->isPICStyleGOT()) {
1940    // ELF / PIC requires GOT in the EBX register before function calls via PLT
1941    // GOT pointer.
1942    if (!isTailCall) {
1943      Chain = DAG.getCopyToReg(Chain, dl, X86::EBX,
1944                               DAG.getNode(X86ISD::GlobalBaseReg,
1945                                           DebugLoc(), getPointerTy()),
1946                               InFlag);
1947      InFlag = Chain.getValue(1);
1948    } else {
1949      // If we are tail calling and generating PIC/GOT style code load the
1950      // address of the callee into ECX. The value in ecx is used as target of
1951      // the tail jump. This is done to circumvent the ebx/callee-saved problem
1952      // for tail calls on PIC/GOT architectures. Normally we would just put the
1953      // address of GOT into ebx and then call target@PLT. But for tail calls
1954      // ebx would be restored (since ebx is callee saved) before jumping to the
1955      // target@PLT.
1956
1957      // Note: The actual moving to ECX is done further down.
1958      GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee);
1959      if (G && !G->getGlobal()->hasHiddenVisibility() &&
1960          !G->getGlobal()->hasProtectedVisibility())
1961        Callee = LowerGlobalAddress(Callee, DAG);
1962      else if (isa<ExternalSymbolSDNode>(Callee))
1963        Callee = LowerExternalSymbol(Callee, DAG);
1964    }
1965  }
1966
1967  if (Is64Bit && isVarArg) {
1968    // From AMD64 ABI document:
1969    // For calls that may call functions that use varargs or stdargs
1970    // (prototype-less calls or calls to functions containing ellipsis (...) in
1971    // the declaration) %al is used as hidden argument to specify the number
1972    // of SSE registers used. The contents of %al do not need to match exactly
1973    // the number of registers, but must be an ubound on the number of SSE
1974    // registers used and is in the range 0 - 8 inclusive.
1975
1976    // FIXME: Verify this on Win64
1977    // Count the number of XMM registers allocated.
1978    static const unsigned XMMArgRegs[] = {
1979      X86::XMM0, X86::XMM1, X86::XMM2, X86::XMM3,
1980      X86::XMM4, X86::XMM5, X86::XMM6, X86::XMM7
1981    };
1982    unsigned NumXMMRegs = CCInfo.getFirstUnallocated(XMMArgRegs, 8);
1983    assert((Subtarget->hasSSE1() || !NumXMMRegs)
1984           && "SSE registers cannot be used when SSE is disabled");
1985
1986    Chain = DAG.getCopyToReg(Chain, dl, X86::AL,
1987                             DAG.getConstant(NumXMMRegs, MVT::i8), InFlag);
1988    InFlag = Chain.getValue(1);
1989  }
1990
1991
1992  // For tail calls lower the arguments to the 'real' stack slot.
1993  if (isTailCall) {
1994    // Force all the incoming stack arguments to be loaded from the stack
1995    // before any new outgoing arguments are stored to the stack, because the
1996    // outgoing stack slots may alias the incoming argument stack slots, and
1997    // the alias isn't otherwise explicit. This is slightly more conservative
1998    // than necessary, because it means that each store effectively depends
1999    // on every argument instead of just those arguments it would clobber.
2000    SDValue ArgChain = DAG.getStackArgumentTokenFactor(Chain);
2001
2002    SmallVector<SDValue, 8> MemOpChains2;
2003    SDValue FIN;
2004    int FI = 0;
2005    // Do not flag preceeding copytoreg stuff together with the following stuff.
2006    InFlag = SDValue();
2007    if (GuaranteedTailCallOpt) {
2008      for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) {
2009        CCValAssign &VA = ArgLocs[i];
2010        if (VA.isRegLoc())
2011          continue;
2012        assert(VA.isMemLoc());
2013        SDValue Arg = Outs[i].Val;
2014        ISD::ArgFlagsTy Flags = Outs[i].Flags;
2015        // Create frame index.
2016        int32_t Offset = VA.getLocMemOffset()+FPDiff;
2017        uint32_t OpSize = (VA.getLocVT().getSizeInBits()+7)/8;
2018        FI = MF.getFrameInfo()->CreateFixedObject(OpSize, Offset, true, false);
2019        FIN = DAG.getFrameIndex(FI, getPointerTy());
2020
2021        if (Flags.isByVal()) {
2022          // Copy relative to framepointer.
2023          SDValue Source = DAG.getIntPtrConstant(VA.getLocMemOffset());
2024          if (StackPtr.getNode() == 0)
2025            StackPtr = DAG.getCopyFromReg(Chain, dl, X86StackPtr,
2026                                          getPointerTy());
2027          Source = DAG.getNode(ISD::ADD, dl, getPointerTy(), StackPtr, Source);
2028
2029          MemOpChains2.push_back(CreateCopyOfByValArgument(Source, FIN,
2030                                                           ArgChain,
2031                                                           Flags, DAG, dl));
2032        } else {
2033          // Store relative to framepointer.
2034          MemOpChains2.push_back(
2035            DAG.getStore(ArgChain, dl, Arg, FIN,
2036                         PseudoSourceValue::getFixedStack(FI), 0,
2037                         false, false, 0));
2038        }
2039      }
2040    }
2041
2042    if (!MemOpChains2.empty())
2043      Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
2044                          &MemOpChains2[0], MemOpChains2.size());
2045
2046    // Copy arguments to their registers.
2047    for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i) {
2048      Chain = DAG.getCopyToReg(Chain, dl, RegsToPass[i].first,
2049                               RegsToPass[i].second, InFlag);
2050      InFlag = Chain.getValue(1);
2051    }
2052    InFlag =SDValue();
2053
2054    // Store the return address to the appropriate stack slot.
2055    Chain = EmitTailCallStoreRetAddr(DAG, MF, Chain, RetAddrFrIdx, Is64Bit,
2056                                     FPDiff, dl);
2057  }
2058
2059  bool WasGlobalOrExternal = false;
2060  if (getTargetMachine().getCodeModel() == CodeModel::Large) {
2061    assert(Is64Bit && "Large code model is only legal in 64-bit mode.");
2062    // In the 64-bit large code model, we have to make all calls
2063    // through a register, since the call instruction's 32-bit
2064    // pc-relative offset may not be large enough to hold the whole
2065    // address.
2066  } else if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee)) {
2067    WasGlobalOrExternal = true;
2068    // If the callee is a GlobalAddress node (quite common, every direct call
2069    // is) turn it into a TargetGlobalAddress node so that legalize doesn't hack
2070    // it.
2071
2072    // We should use extra load for direct calls to dllimported functions in
2073    // non-JIT mode.
2074    const GlobalValue *GV = G->getGlobal();
2075    if (!GV->hasDLLImportLinkage()) {
2076      unsigned char OpFlags = 0;
2077
2078      // On ELF targets, in both X86-64 and X86-32 mode, direct calls to
2079      // external symbols most go through the PLT in PIC mode.  If the symbol
2080      // has hidden or protected visibility, or if it is static or local, then
2081      // we don't need to use the PLT - we can directly call it.
2082      if (Subtarget->isTargetELF() &&
2083          getTargetMachine().getRelocationModel() == Reloc::PIC_ &&
2084          GV->hasDefaultVisibility() && !GV->hasLocalLinkage()) {
2085        OpFlags = X86II::MO_PLT;
2086      } else if (Subtarget->isPICStyleStubAny() &&
2087               (GV->isDeclaration() || GV->isWeakForLinker()) &&
2088               Subtarget->getDarwinVers() < 9) {
2089        // PC-relative references to external symbols should go through $stub,
2090        // unless we're building with the leopard linker or later, which
2091        // automatically synthesizes these stubs.
2092        OpFlags = X86II::MO_DARWIN_STUB;
2093      }
2094
2095      Callee = DAG.getTargetGlobalAddress(GV, getPointerTy(),
2096                                          G->getOffset(), OpFlags);
2097    }
2098  } else if (ExternalSymbolSDNode *S = dyn_cast<ExternalSymbolSDNode>(Callee)) {
2099    WasGlobalOrExternal = true;
2100    unsigned char OpFlags = 0;
2101
2102    // On ELF targets, in either X86-64 or X86-32 mode, direct calls to external
2103    // symbols should go through the PLT.
2104    if (Subtarget->isTargetELF() &&
2105        getTargetMachine().getRelocationModel() == Reloc::PIC_) {
2106      OpFlags = X86II::MO_PLT;
2107    } else if (Subtarget->isPICStyleStubAny() &&
2108             Subtarget->getDarwinVers() < 9) {
2109      // PC-relative references to external symbols should go through $stub,
2110      // unless we're building with the leopard linker or later, which
2111      // automatically synthesizes these stubs.
2112      OpFlags = X86II::MO_DARWIN_STUB;
2113    }
2114
2115    Callee = DAG.getTargetExternalSymbol(S->getSymbol(), getPointerTy(),
2116                                         OpFlags);
2117  }
2118
2119  // Returns a chain & a flag for retval copy to use.
2120  SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Flag);
2121  SmallVector<SDValue, 8> Ops;
2122
2123  if (!IsSibcall && isTailCall) {
2124    Chain = DAG.getCALLSEQ_END(Chain, DAG.getIntPtrConstant(NumBytes, true),
2125                           DAG.getIntPtrConstant(0, true), InFlag);
2126    InFlag = Chain.getValue(1);
2127  }
2128
2129  Ops.push_back(Chain);
2130  Ops.push_back(Callee);
2131
2132  if (isTailCall)
2133    Ops.push_back(DAG.getConstant(FPDiff, MVT::i32));
2134
2135  // Add argument registers to the end of the list so that they are known live
2136  // into the call.
2137  for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i)
2138    Ops.push_back(DAG.getRegister(RegsToPass[i].first,
2139                                  RegsToPass[i].second.getValueType()));
2140
2141  // Add an implicit use GOT pointer in EBX.
2142  if (!isTailCall && Subtarget->isPICStyleGOT())
2143    Ops.push_back(DAG.getRegister(X86::EBX, getPointerTy()));
2144
2145  // Add an implicit use of AL for x86 vararg functions.
2146  if (Is64Bit && isVarArg)
2147    Ops.push_back(DAG.getRegister(X86::AL, MVT::i8));
2148
2149  if (InFlag.getNode())
2150    Ops.push_back(InFlag);
2151
2152  if (isTailCall) {
2153    // We used to do:
2154    //// If this is the first return lowered for this function, add the regs
2155    //// to the liveout set for the function.
2156    // This isn't right, although it's probably harmless on x86; liveouts
2157    // should be computed from returns not tail calls.  Consider a void
2158    // function making a tail call to a function returning int.
2159    return DAG.getNode(X86ISD::TC_RETURN, dl,
2160                       NodeTys, &Ops[0], Ops.size());
2161  }
2162
2163  Chain = DAG.getNode(X86ISD::CALL, dl, NodeTys, &Ops[0], Ops.size());
2164  InFlag = Chain.getValue(1);
2165
2166  // Create the CALLSEQ_END node.
2167  unsigned NumBytesForCalleeToPush;
2168  if (Subtarget->IsCalleePop(isVarArg, CallConv))
2169    NumBytesForCalleeToPush = NumBytes;    // Callee pops everything
2170  else if (!Is64Bit && !IsTailCallConvention(CallConv) && IsStructRet)
2171    // If this is a call to a struct-return function, the callee
2172    // pops the hidden struct pointer, so we have to push it back.
2173    // This is common for Darwin/X86, Linux & Mingw32 targets.
2174    NumBytesForCalleeToPush = 4;
2175  else
2176    NumBytesForCalleeToPush = 0;  // Callee pops nothing.
2177
2178  // Returns a flag for retval copy to use.
2179  if (!IsSibcall) {
2180    Chain = DAG.getCALLSEQ_END(Chain,
2181                               DAG.getIntPtrConstant(NumBytes, true),
2182                               DAG.getIntPtrConstant(NumBytesForCalleeToPush,
2183                                                     true),
2184                               InFlag);
2185    InFlag = Chain.getValue(1);
2186  }
2187
2188  // Handle result values, copying them out of physregs into vregs that we
2189  // return.
2190  return LowerCallResult(Chain, InFlag, CallConv, isVarArg,
2191                         Ins, dl, DAG, InVals);
2192}
2193
2194
2195//===----------------------------------------------------------------------===//
2196//                Fast Calling Convention (tail call) implementation
2197//===----------------------------------------------------------------------===//
2198
2199//  Like std call, callee cleans arguments, convention except that ECX is
2200//  reserved for storing the tail called function address. Only 2 registers are
2201//  free for argument passing (inreg). Tail call optimization is performed
2202//  provided:
2203//                * tailcallopt is enabled
2204//                * caller/callee are fastcc
2205//  On X86_64 architecture with GOT-style position independent code only local
2206//  (within module) calls are supported at the moment.
2207//  To keep the stack aligned according to platform abi the function
2208//  GetAlignedArgumentStackSize ensures that argument delta is always multiples
2209//  of stack alignment. (Dynamic linkers need this - darwin's dyld for example)
2210//  If a tail called function callee has more arguments than the caller the
2211//  caller needs to make sure that there is room to move the RETADDR to. This is
2212//  achieved by reserving an area the size of the argument delta right after the
2213//  original REtADDR, but before the saved framepointer or the spilled registers
2214//  e.g. caller(arg1, arg2) calls callee(arg1, arg2,arg3,arg4)
2215//  stack layout:
2216//    arg1
2217//    arg2
2218//    RETADDR
2219//    [ new RETADDR
2220//      move area ]
2221//    (possible EBP)
2222//    ESI
2223//    EDI
2224//    local1 ..
2225
2226/// GetAlignedArgumentStackSize - Make the stack size align e.g 16n + 12 aligned
2227/// for a 16 byte align requirement.
2228unsigned
2229X86TargetLowering::GetAlignedArgumentStackSize(unsigned StackSize,
2230                                               SelectionDAG& DAG) const {
2231  MachineFunction &MF = DAG.getMachineFunction();
2232  const TargetMachine &TM = MF.getTarget();
2233  const TargetFrameInfo &TFI = *TM.getFrameInfo();
2234  unsigned StackAlignment = TFI.getStackAlignment();
2235  uint64_t AlignMask = StackAlignment - 1;
2236  int64_t Offset = StackSize;
2237  uint64_t SlotSize = TD->getPointerSize();
2238  if ( (Offset & AlignMask) <= (StackAlignment - SlotSize) ) {
2239    // Number smaller than 12 so just add the difference.
2240    Offset += ((StackAlignment - SlotSize) - (Offset & AlignMask));
2241  } else {
2242    // Mask out lower bits, add stackalignment once plus the 12 bytes.
2243    Offset = ((~AlignMask) & Offset) + StackAlignment +
2244      (StackAlignment-SlotSize);
2245  }
2246  return Offset;
2247}
2248
2249/// MatchingStackOffset - Return true if the given stack call argument is
2250/// already available in the same position (relatively) of the caller's
2251/// incoming argument stack.
2252static
2253bool MatchingStackOffset(SDValue Arg, unsigned Offset, ISD::ArgFlagsTy Flags,
2254                         MachineFrameInfo *MFI, const MachineRegisterInfo *MRI,
2255                         const X86InstrInfo *TII) {
2256  unsigned Bytes = Arg.getValueType().getSizeInBits() / 8;
2257  int FI = INT_MAX;
2258  if (Arg.getOpcode() == ISD::CopyFromReg) {
2259    unsigned VR = cast<RegisterSDNode>(Arg.getOperand(1))->getReg();
2260    if (!VR || TargetRegisterInfo::isPhysicalRegister(VR))
2261      return false;
2262    MachineInstr *Def = MRI->getVRegDef(VR);
2263    if (!Def)
2264      return false;
2265    if (!Flags.isByVal()) {
2266      if (!TII->isLoadFromStackSlot(Def, FI))
2267        return false;
2268    } else {
2269      unsigned Opcode = Def->getOpcode();
2270      if ((Opcode == X86::LEA32r || Opcode == X86::LEA64r) &&
2271          Def->getOperand(1).isFI()) {
2272        FI = Def->getOperand(1).getIndex();
2273        Bytes = Flags.getByValSize();
2274      } else
2275        return false;
2276    }
2277  } else if (LoadSDNode *Ld = dyn_cast<LoadSDNode>(Arg)) {
2278    if (Flags.isByVal())
2279      // ByVal argument is passed in as a pointer but it's now being
2280      // dereferenced. e.g.
2281      // define @foo(%struct.X* %A) {
2282      //   tail call @bar(%struct.X* byval %A)
2283      // }
2284      return false;
2285    SDValue Ptr = Ld->getBasePtr();
2286    FrameIndexSDNode *FINode = dyn_cast<FrameIndexSDNode>(Ptr);
2287    if (!FINode)
2288      return false;
2289    FI = FINode->getIndex();
2290  } else
2291    return false;
2292
2293  assert(FI != INT_MAX);
2294  if (!MFI->isFixedObjectIndex(FI))
2295    return false;
2296  return Offset == MFI->getObjectOffset(FI) && Bytes == MFI->getObjectSize(FI);
2297}
2298
2299/// IsEligibleForTailCallOptimization - Check whether the call is eligible
2300/// for tail call optimization. Targets which want to do tail call
2301/// optimization should implement this function.
2302bool
2303X86TargetLowering::IsEligibleForTailCallOptimization(SDValue Callee,
2304                                                     CallingConv::ID CalleeCC,
2305                                                     bool isVarArg,
2306                                                     bool isCalleeStructRet,
2307                                                     bool isCallerStructRet,
2308                                    const SmallVectorImpl<ISD::OutputArg> &Outs,
2309                                    const SmallVectorImpl<ISD::InputArg> &Ins,
2310                                                     SelectionDAG& DAG) const {
2311  if (!IsTailCallConvention(CalleeCC) &&
2312      CalleeCC != CallingConv::C)
2313    return false;
2314
2315  // If -tailcallopt is specified, make fastcc functions tail-callable.
2316  const MachineFunction &MF = DAG.getMachineFunction();
2317  const Function *CallerF = DAG.getMachineFunction().getFunction();
2318  CallingConv::ID CallerCC = CallerF->getCallingConv();
2319  bool CCMatch = CallerCC == CalleeCC;
2320
2321  if (GuaranteedTailCallOpt) {
2322    if (IsTailCallConvention(CalleeCC) && CCMatch)
2323      return true;
2324    return false;
2325  }
2326
2327  // Look for obvious safe cases to perform tail call optimization that do not
2328  // require ABI changes. This is what gcc calls sibcall.
2329
2330  // Can't do sibcall if stack needs to be dynamically re-aligned. PEI needs to
2331  // emit a special epilogue.
2332  if (RegInfo->needsStackRealignment(MF))
2333    return false;
2334
2335  // Do not sibcall optimize vararg calls unless the call site is not passing any
2336  // arguments.
2337  if (isVarArg && !Outs.empty())
2338    return false;
2339
2340  // Also avoid sibcall optimization if either caller or callee uses struct
2341  // return semantics.
2342  if (isCalleeStructRet || isCallerStructRet)
2343    return false;
2344
2345  // If the call result is in ST0 / ST1, it needs to be popped off the x87 stack.
2346  // Therefore if it's not used by the call it is not safe to optimize this into
2347  // a sibcall.
2348  bool Unused = false;
2349  for (unsigned i = 0, e = Ins.size(); i != e; ++i) {
2350    if (!Ins[i].Used) {
2351      Unused = true;
2352      break;
2353    }
2354  }
2355  if (Unused) {
2356    SmallVector<CCValAssign, 16> RVLocs;
2357    CCState CCInfo(CalleeCC, false, getTargetMachine(),
2358                   RVLocs, *DAG.getContext());
2359    CCInfo.AnalyzeCallResult(Ins, RetCC_X86);
2360    for (unsigned i = 0, e = RVLocs.size(); i != e; ++i) {
2361      CCValAssign &VA = RVLocs[i];
2362      if (VA.getLocReg() == X86::ST0 || VA.getLocReg() == X86::ST1)
2363        return false;
2364    }
2365  }
2366
2367  // If the calling conventions do not match, then we'd better make sure the
2368  // results are returned in the same way as what the caller expects.
2369  if (!CCMatch) {
2370    SmallVector<CCValAssign, 16> RVLocs1;
2371    CCState CCInfo1(CalleeCC, false, getTargetMachine(),
2372                    RVLocs1, *DAG.getContext());
2373    CCInfo1.AnalyzeCallResult(Ins, RetCC_X86);
2374
2375    SmallVector<CCValAssign, 16> RVLocs2;
2376    CCState CCInfo2(CallerCC, false, getTargetMachine(),
2377                    RVLocs2, *DAG.getContext());
2378    CCInfo2.AnalyzeCallResult(Ins, RetCC_X86);
2379
2380    if (RVLocs1.size() != RVLocs2.size())
2381      return false;
2382    for (unsigned i = 0, e = RVLocs1.size(); i != e; ++i) {
2383      if (RVLocs1[i].isRegLoc() != RVLocs2[i].isRegLoc())
2384        return false;
2385      if (RVLocs1[i].getLocInfo() != RVLocs2[i].getLocInfo())
2386        return false;
2387      if (RVLocs1[i].isRegLoc()) {
2388        if (RVLocs1[i].getLocReg() != RVLocs2[i].getLocReg())
2389          return false;
2390      } else {
2391        if (RVLocs1[i].getLocMemOffset() != RVLocs2[i].getLocMemOffset())
2392          return false;
2393      }
2394    }
2395  }
2396
2397  // If the callee takes no arguments then go on to check the results of the
2398  // call.
2399  if (!Outs.empty()) {
2400    // Check if stack adjustment is needed. For now, do not do this if any
2401    // argument is passed on the stack.
2402    SmallVector<CCValAssign, 16> ArgLocs;
2403    CCState CCInfo(CalleeCC, isVarArg, getTargetMachine(),
2404                   ArgLocs, *DAG.getContext());
2405    CCInfo.AnalyzeCallOperands(Outs, CCAssignFnForNode(CalleeCC));
2406    if (CCInfo.getNextStackOffset()) {
2407      MachineFunction &MF = DAG.getMachineFunction();
2408      if (MF.getInfo<X86MachineFunctionInfo>()->getBytesToPopOnReturn())
2409        return false;
2410      if (Subtarget->isTargetWin64())
2411        // Win64 ABI has additional complications.
2412        return false;
2413
2414      // Check if the arguments are already laid out in the right way as
2415      // the caller's fixed stack objects.
2416      MachineFrameInfo *MFI = MF.getFrameInfo();
2417      const MachineRegisterInfo *MRI = &MF.getRegInfo();
2418      const X86InstrInfo *TII =
2419        ((X86TargetMachine&)getTargetMachine()).getInstrInfo();
2420      for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) {
2421        CCValAssign &VA = ArgLocs[i];
2422        EVT RegVT = VA.getLocVT();
2423        SDValue Arg = Outs[i].Val;
2424        ISD::ArgFlagsTy Flags = Outs[i].Flags;
2425        if (VA.getLocInfo() == CCValAssign::Indirect)
2426          return false;
2427        if (!VA.isRegLoc()) {
2428          if (!MatchingStackOffset(Arg, VA.getLocMemOffset(), Flags,
2429                                   MFI, MRI, TII))
2430            return false;
2431        }
2432      }
2433    }
2434
2435    // If the tailcall address may be in a register, then make sure it's
2436    // possible to register allocate for it. In 32-bit, the call address can
2437    // only target EAX, EDX, or ECX since the tail call must be scheduled after
2438    // callee-saved registers are restored. In 64-bit, it's RAX, RCX, RDX, RSI,
2439    // RDI, R8, R9, R11.
2440    if (!isa<GlobalAddressSDNode>(Callee) &&
2441        !isa<ExternalSymbolSDNode>(Callee)) {
2442      unsigned Limit = Subtarget->is64Bit() ? 8 : 3;
2443      unsigned NumInRegs = 0;
2444      for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) {
2445        CCValAssign &VA = ArgLocs[i];
2446        if (VA.isRegLoc()) {
2447          if (++NumInRegs == Limit)
2448            return false;
2449        }
2450      }
2451    }
2452  }
2453
2454  return true;
2455}
2456
2457FastISel *
2458X86TargetLowering::createFastISel(MachineFunction &mf,
2459                            DenseMap<const Value *, unsigned> &vm,
2460                            DenseMap<const BasicBlock*, MachineBasicBlock*> &bm,
2461                            DenseMap<const AllocaInst *, int> &am,
2462                            std::vector<std::pair<MachineInstr*, unsigned> > &pn
2463#ifndef NDEBUG
2464                          , SmallSet<const Instruction *, 8> &cil
2465#endif
2466                                  ) const {
2467  return X86::createFastISel(mf, vm, bm, am, pn
2468#ifndef NDEBUG
2469                             , cil
2470#endif
2471                             );
2472}
2473
2474
2475//===----------------------------------------------------------------------===//
2476//                           Other Lowering Hooks
2477//===----------------------------------------------------------------------===//
2478
2479
2480SDValue X86TargetLowering::getReturnAddressFrameIndex(SelectionDAG &DAG) const {
2481  MachineFunction &MF = DAG.getMachineFunction();
2482  X86MachineFunctionInfo *FuncInfo = MF.getInfo<X86MachineFunctionInfo>();
2483  int ReturnAddrIndex = FuncInfo->getRAIndex();
2484
2485  if (ReturnAddrIndex == 0) {
2486    // Set up a frame object for the return address.
2487    uint64_t SlotSize = TD->getPointerSize();
2488    ReturnAddrIndex = MF.getFrameInfo()->CreateFixedObject(SlotSize, -SlotSize,
2489                                                           false, false);
2490    FuncInfo->setRAIndex(ReturnAddrIndex);
2491  }
2492
2493  return DAG.getFrameIndex(ReturnAddrIndex, getPointerTy());
2494}
2495
2496
2497bool X86::isOffsetSuitableForCodeModel(int64_t Offset, CodeModel::Model M,
2498                                       bool hasSymbolicDisplacement) {
2499  // Offset should fit into 32 bit immediate field.
2500  if (!isInt<32>(Offset))
2501    return false;
2502
2503  // If we don't have a symbolic displacement - we don't have any extra
2504  // restrictions.
2505  if (!hasSymbolicDisplacement)
2506    return true;
2507
2508  // FIXME: Some tweaks might be needed for medium code model.
2509  if (M != CodeModel::Small && M != CodeModel::Kernel)
2510    return false;
2511
2512  // For small code model we assume that latest object is 16MB before end of 31
2513  // bits boundary. We may also accept pretty large negative constants knowing
2514  // that all objects are in the positive half of address space.
2515  if (M == CodeModel::Small && Offset < 16*1024*1024)
2516    return true;
2517
2518  // For kernel code model we know that all object resist in the negative half
2519  // of 32bits address space. We may not accept negative offsets, since they may
2520  // be just off and we may accept pretty large positive ones.
2521  if (M == CodeModel::Kernel && Offset > 0)
2522    return true;
2523
2524  return false;
2525}
2526
2527/// TranslateX86CC - do a one to one translation of a ISD::CondCode to the X86
2528/// specific condition code, returning the condition code and the LHS/RHS of the
2529/// comparison to make.
2530static unsigned TranslateX86CC(ISD::CondCode SetCCOpcode, bool isFP,
2531                               SDValue &LHS, SDValue &RHS, SelectionDAG &DAG) {
2532  if (!isFP) {
2533    if (ConstantSDNode *RHSC = dyn_cast<ConstantSDNode>(RHS)) {
2534      if (SetCCOpcode == ISD::SETGT && RHSC->isAllOnesValue()) {
2535        // X > -1   -> X == 0, jump !sign.
2536        RHS = DAG.getConstant(0, RHS.getValueType());
2537        return X86::COND_NS;
2538      } else if (SetCCOpcode == ISD::SETLT && RHSC->isNullValue()) {
2539        // X < 0   -> X == 0, jump on sign.
2540        return X86::COND_S;
2541      } else if (SetCCOpcode == ISD::SETLT && RHSC->getZExtValue() == 1) {
2542        // X < 1   -> X <= 0
2543        RHS = DAG.getConstant(0, RHS.getValueType());
2544        return X86::COND_LE;
2545      }
2546    }
2547
2548    switch (SetCCOpcode) {
2549    default: llvm_unreachable("Invalid integer condition!");
2550    case ISD::SETEQ:  return X86::COND_E;
2551    case ISD::SETGT:  return X86::COND_G;
2552    case ISD::SETGE:  return X86::COND_GE;
2553    case ISD::SETLT:  return X86::COND_L;
2554    case ISD::SETLE:  return X86::COND_LE;
2555    case ISD::SETNE:  return X86::COND_NE;
2556    case ISD::SETULT: return X86::COND_B;
2557    case ISD::SETUGT: return X86::COND_A;
2558    case ISD::SETULE: return X86::COND_BE;
2559    case ISD::SETUGE: return X86::COND_AE;
2560    }
2561  }
2562
2563  // First determine if it is required or is profitable to flip the operands.
2564
2565  // If LHS is a foldable load, but RHS is not, flip the condition.
2566  if ((ISD::isNON_EXTLoad(LHS.getNode()) && LHS.hasOneUse()) &&
2567      !(ISD::isNON_EXTLoad(RHS.getNode()) && RHS.hasOneUse())) {
2568    SetCCOpcode = getSetCCSwappedOperands(SetCCOpcode);
2569    std::swap(LHS, RHS);
2570  }
2571
2572  switch (SetCCOpcode) {
2573  default: break;
2574  case ISD::SETOLT:
2575  case ISD::SETOLE:
2576  case ISD::SETUGT:
2577  case ISD::SETUGE:
2578    std::swap(LHS, RHS);
2579    break;
2580  }
2581
2582  // On a floating point condition, the flags are set as follows:
2583  // ZF  PF  CF   op
2584  //  0 | 0 | 0 | X > Y
2585  //  0 | 0 | 1 | X < Y
2586  //  1 | 0 | 0 | X == Y
2587  //  1 | 1 | 1 | unordered
2588  switch (SetCCOpcode) {
2589  default: llvm_unreachable("Condcode should be pre-legalized away");
2590  case ISD::SETUEQ:
2591  case ISD::SETEQ:   return X86::COND_E;
2592  case ISD::SETOLT:              // flipped
2593  case ISD::SETOGT:
2594  case ISD::SETGT:   return X86::COND_A;
2595  case ISD::SETOLE:              // flipped
2596  case ISD::SETOGE:
2597  case ISD::SETGE:   return X86::COND_AE;
2598  case ISD::SETUGT:              // flipped
2599  case ISD::SETULT:
2600  case ISD::SETLT:   return X86::COND_B;
2601  case ISD::SETUGE:              // flipped
2602  case ISD::SETULE:
2603  case ISD::SETLE:   return X86::COND_BE;
2604  case ISD::SETONE:
2605  case ISD::SETNE:   return X86::COND_NE;
2606  case ISD::SETUO:   return X86::COND_P;
2607  case ISD::SETO:    return X86::COND_NP;
2608  case ISD::SETOEQ:
2609  case ISD::SETUNE:  return X86::COND_INVALID;
2610  }
2611}
2612
2613/// hasFPCMov - is there a floating point cmov for the specific X86 condition
2614/// code. Current x86 isa includes the following FP cmov instructions:
2615/// fcmovb, fcomvbe, fcomve, fcmovu, fcmovae, fcmova, fcmovne, fcmovnu.
2616static bool hasFPCMov(unsigned X86CC) {
2617  switch (X86CC) {
2618  default:
2619    return false;
2620  case X86::COND_B:
2621  case X86::COND_BE:
2622  case X86::COND_E:
2623  case X86::COND_P:
2624  case X86::COND_A:
2625  case X86::COND_AE:
2626  case X86::COND_NE:
2627  case X86::COND_NP:
2628    return true;
2629  }
2630}
2631
2632/// isFPImmLegal - Returns true if the target can instruction select the
2633/// specified FP immediate natively. If false, the legalizer will
2634/// materialize the FP immediate as a load from a constant pool.
2635bool X86TargetLowering::isFPImmLegal(const APFloat &Imm, EVT VT) const {
2636  for (unsigned i = 0, e = LegalFPImmediates.size(); i != e; ++i) {
2637    if (Imm.bitwiseIsEqual(LegalFPImmediates[i]))
2638      return true;
2639  }
2640  return false;
2641}
2642
2643/// isUndefOrInRange - Return true if Val is undef or if its value falls within
2644/// the specified range (L, H].
2645static bool isUndefOrInRange(int Val, int Low, int Hi) {
2646  return (Val < 0) || (Val >= Low && Val < Hi);
2647}
2648
2649/// isUndefOrEqual - Val is either less than zero (undef) or equal to the
2650/// specified value.
2651static bool isUndefOrEqual(int Val, int CmpVal) {
2652  if (Val < 0 || Val == CmpVal)
2653    return true;
2654  return false;
2655}
2656
2657/// isPSHUFDMask - Return true if the node specifies a shuffle of elements that
2658/// is suitable for input to PSHUFD or PSHUFW.  That is, it doesn't reference
2659/// the second operand.
2660static bool isPSHUFDMask(const SmallVectorImpl<int> &Mask, EVT VT) {
2661  if (VT == MVT::v4f32 || VT == MVT::v4i32 || VT == MVT::v4i16)
2662    return (Mask[0] < 4 && Mask[1] < 4 && Mask[2] < 4 && Mask[3] < 4);
2663  if (VT == MVT::v2f64 || VT == MVT::v2i64)
2664    return (Mask[0] < 2 && Mask[1] < 2);
2665  return false;
2666}
2667
2668bool X86::isPSHUFDMask(ShuffleVectorSDNode *N) {
2669  SmallVector<int, 8> M;
2670  N->getMask(M);
2671  return ::isPSHUFDMask(M, N->getValueType(0));
2672}
2673
2674/// isPSHUFHWMask - Return true if the node specifies a shuffle of elements that
2675/// is suitable for input to PSHUFHW.
2676static bool isPSHUFHWMask(const SmallVectorImpl<int> &Mask, EVT VT) {
2677  if (VT != MVT::v8i16)
2678    return false;
2679
2680  // Lower quadword copied in order or undef.
2681  for (int i = 0; i != 4; ++i)
2682    if (Mask[i] >= 0 && Mask[i] != i)
2683      return false;
2684
2685  // Upper quadword shuffled.
2686  for (int i = 4; i != 8; ++i)
2687    if (Mask[i] >= 0 && (Mask[i] < 4 || Mask[i] > 7))
2688      return false;
2689
2690  return true;
2691}
2692
2693bool X86::isPSHUFHWMask(ShuffleVectorSDNode *N) {
2694  SmallVector<int, 8> M;
2695  N->getMask(M);
2696  return ::isPSHUFHWMask(M, N->getValueType(0));
2697}
2698
2699/// isPSHUFLWMask - Return true if the node specifies a shuffle of elements that
2700/// is suitable for input to PSHUFLW.
2701static bool isPSHUFLWMask(const SmallVectorImpl<int> &Mask, EVT VT) {
2702  if (VT != MVT::v8i16)
2703    return false;
2704
2705  // Upper quadword copied in order.
2706  for (int i = 4; i != 8; ++i)
2707    if (Mask[i] >= 0 && Mask[i] != i)
2708      return false;
2709
2710  // Lower quadword shuffled.
2711  for (int i = 0; i != 4; ++i)
2712    if (Mask[i] >= 4)
2713      return false;
2714
2715  return true;
2716}
2717
2718bool X86::isPSHUFLWMask(ShuffleVectorSDNode *N) {
2719  SmallVector<int, 8> M;
2720  N->getMask(M);
2721  return ::isPSHUFLWMask(M, N->getValueType(0));
2722}
2723
2724/// isPALIGNRMask - Return true if the node specifies a shuffle of elements that
2725/// is suitable for input to PALIGNR.
2726static bool isPALIGNRMask(const SmallVectorImpl<int> &Mask, EVT VT,
2727                          bool hasSSSE3) {
2728  int i, e = VT.getVectorNumElements();
2729
2730  // Do not handle v2i64 / v2f64 shuffles with palignr.
2731  if (e < 4 || !hasSSSE3)
2732    return false;
2733
2734  for (i = 0; i != e; ++i)
2735    if (Mask[i] >= 0)
2736      break;
2737
2738  // All undef, not a palignr.
2739  if (i == e)
2740    return false;
2741
2742  // Determine if it's ok to perform a palignr with only the LHS, since we
2743  // don't have access to the actual shuffle elements to see if RHS is undef.
2744  bool Unary = Mask[i] < (int)e;
2745  bool NeedsUnary = false;
2746
2747  int s = Mask[i] - i;
2748
2749  // Check the rest of the elements to see if they are consecutive.
2750  for (++i; i != e; ++i) {
2751    int m = Mask[i];
2752    if (m < 0)
2753      continue;
2754
2755    Unary = Unary && (m < (int)e);
2756    NeedsUnary = NeedsUnary || (m < s);
2757
2758    if (NeedsUnary && !Unary)
2759      return false;
2760    if (Unary && m != ((s+i) & (e-1)))
2761      return false;
2762    if (!Unary && m != (s+i))
2763      return false;
2764  }
2765  return true;
2766}
2767
2768bool X86::isPALIGNRMask(ShuffleVectorSDNode *N) {
2769  SmallVector<int, 8> M;
2770  N->getMask(M);
2771  return ::isPALIGNRMask(M, N->getValueType(0), true);
2772}
2773
2774/// isSHUFPMask - Return true if the specified VECTOR_SHUFFLE operand
2775/// specifies a shuffle of elements that is suitable for input to SHUFP*.
2776static bool isSHUFPMask(const SmallVectorImpl<int> &Mask, EVT VT) {
2777  int NumElems = VT.getVectorNumElements();
2778  if (NumElems != 2 && NumElems != 4)
2779    return false;
2780
2781  int Half = NumElems / 2;
2782  for (int i = 0; i < Half; ++i)
2783    if (!isUndefOrInRange(Mask[i], 0, NumElems))
2784      return false;
2785  for (int i = Half; i < NumElems; ++i)
2786    if (!isUndefOrInRange(Mask[i], NumElems, NumElems*2))
2787      return false;
2788
2789  return true;
2790}
2791
2792bool X86::isSHUFPMask(ShuffleVectorSDNode *N) {
2793  SmallVector<int, 8> M;
2794  N->getMask(M);
2795  return ::isSHUFPMask(M, N->getValueType(0));
2796}
2797
2798/// isCommutedSHUFP - Returns true if the shuffle mask is exactly
2799/// the reverse of what x86 shuffles want. x86 shuffles requires the lower
2800/// half elements to come from vector 1 (which would equal the dest.) and
2801/// the upper half to come from vector 2.
2802static bool isCommutedSHUFPMask(const SmallVectorImpl<int> &Mask, EVT VT) {
2803  int NumElems = VT.getVectorNumElements();
2804
2805  if (NumElems != 2 && NumElems != 4)
2806    return false;
2807
2808  int Half = NumElems / 2;
2809  for (int i = 0; i < Half; ++i)
2810    if (!isUndefOrInRange(Mask[i], NumElems, NumElems*2))
2811      return false;
2812  for (int i = Half; i < NumElems; ++i)
2813    if (!isUndefOrInRange(Mask[i], 0, NumElems))
2814      return false;
2815  return true;
2816}
2817
2818static bool isCommutedSHUFP(ShuffleVectorSDNode *N) {
2819  SmallVector<int, 8> M;
2820  N->getMask(M);
2821  return isCommutedSHUFPMask(M, N->getValueType(0));
2822}
2823
2824/// isMOVHLPSMask - Return true if the specified VECTOR_SHUFFLE operand
2825/// specifies a shuffle of elements that is suitable for input to MOVHLPS.
2826bool X86::isMOVHLPSMask(ShuffleVectorSDNode *N) {
2827  if (N->getValueType(0).getVectorNumElements() != 4)
2828    return false;
2829
2830  // Expect bit0 == 6, bit1 == 7, bit2 == 2, bit3 == 3
2831  return isUndefOrEqual(N->getMaskElt(0), 6) &&
2832         isUndefOrEqual(N->getMaskElt(1), 7) &&
2833         isUndefOrEqual(N->getMaskElt(2), 2) &&
2834         isUndefOrEqual(N->getMaskElt(3), 3);
2835}
2836
2837/// isMOVHLPS_v_undef_Mask - Special case of isMOVHLPSMask for canonical form
2838/// of vector_shuffle v, v, <2, 3, 2, 3>, i.e. vector_shuffle v, undef,
2839/// <2, 3, 2, 3>
2840bool X86::isMOVHLPS_v_undef_Mask(ShuffleVectorSDNode *N) {
2841  unsigned NumElems = N->getValueType(0).getVectorNumElements();
2842
2843  if (NumElems != 4)
2844    return false;
2845
2846  return isUndefOrEqual(N->getMaskElt(0), 2) &&
2847  isUndefOrEqual(N->getMaskElt(1), 3) &&
2848  isUndefOrEqual(N->getMaskElt(2), 2) &&
2849  isUndefOrEqual(N->getMaskElt(3), 3);
2850}
2851
2852/// isMOVLPMask - Return true if the specified VECTOR_SHUFFLE operand
2853/// specifies a shuffle of elements that is suitable for input to MOVLP{S|D}.
2854bool X86::isMOVLPMask(ShuffleVectorSDNode *N) {
2855  unsigned NumElems = N->getValueType(0).getVectorNumElements();
2856
2857  if (NumElems != 2 && NumElems != 4)
2858    return false;
2859
2860  for (unsigned i = 0; i < NumElems/2; ++i)
2861    if (!isUndefOrEqual(N->getMaskElt(i), i + NumElems))
2862      return false;
2863
2864  for (unsigned i = NumElems/2; i < NumElems; ++i)
2865    if (!isUndefOrEqual(N->getMaskElt(i), i))
2866      return false;
2867
2868  return true;
2869}
2870
2871/// isMOVLHPSMask - Return true if the specified VECTOR_SHUFFLE operand
2872/// specifies a shuffle of elements that is suitable for input to MOVLHPS.
2873bool X86::isMOVLHPSMask(ShuffleVectorSDNode *N) {
2874  unsigned NumElems = N->getValueType(0).getVectorNumElements();
2875
2876  if (NumElems != 2 && NumElems != 4)
2877    return false;
2878
2879  for (unsigned i = 0; i < NumElems/2; ++i)
2880    if (!isUndefOrEqual(N->getMaskElt(i), i))
2881      return false;
2882
2883  for (unsigned i = 0; i < NumElems/2; ++i)
2884    if (!isUndefOrEqual(N->getMaskElt(i + NumElems/2), i + NumElems))
2885      return false;
2886
2887  return true;
2888}
2889
2890/// isUNPCKLMask - Return true if the specified VECTOR_SHUFFLE operand
2891/// specifies a shuffle of elements that is suitable for input to UNPCKL.
2892static bool isUNPCKLMask(const SmallVectorImpl<int> &Mask, EVT VT,
2893                         bool V2IsSplat = false) {
2894  int NumElts = VT.getVectorNumElements();
2895  if (NumElts != 2 && NumElts != 4 && NumElts != 8 && NumElts != 16)
2896    return false;
2897
2898  for (int i = 0, j = 0; i != NumElts; i += 2, ++j) {
2899    int BitI  = Mask[i];
2900    int BitI1 = Mask[i+1];
2901    if (!isUndefOrEqual(BitI, j))
2902      return false;
2903    if (V2IsSplat) {
2904      if (!isUndefOrEqual(BitI1, NumElts))
2905        return false;
2906    } else {
2907      if (!isUndefOrEqual(BitI1, j + NumElts))
2908        return false;
2909    }
2910  }
2911  return true;
2912}
2913
2914bool X86::isUNPCKLMask(ShuffleVectorSDNode *N, bool V2IsSplat) {
2915  SmallVector<int, 8> M;
2916  N->getMask(M);
2917  return ::isUNPCKLMask(M, N->getValueType(0), V2IsSplat);
2918}
2919
2920/// isUNPCKHMask - Return true if the specified VECTOR_SHUFFLE operand
2921/// specifies a shuffle of elements that is suitable for input to UNPCKH.
2922static bool isUNPCKHMask(const SmallVectorImpl<int> &Mask, EVT VT,
2923                         bool V2IsSplat = false) {
2924  int NumElts = VT.getVectorNumElements();
2925  if (NumElts != 2 && NumElts != 4 && NumElts != 8 && NumElts != 16)
2926    return false;
2927
2928  for (int i = 0, j = 0; i != NumElts; i += 2, ++j) {
2929    int BitI  = Mask[i];
2930    int BitI1 = Mask[i+1];
2931    if (!isUndefOrEqual(BitI, j + NumElts/2))
2932      return false;
2933    if (V2IsSplat) {
2934      if (isUndefOrEqual(BitI1, NumElts))
2935        return false;
2936    } else {
2937      if (!isUndefOrEqual(BitI1, j + NumElts/2 + NumElts))
2938        return false;
2939    }
2940  }
2941  return true;
2942}
2943
2944bool X86::isUNPCKHMask(ShuffleVectorSDNode *N, bool V2IsSplat) {
2945  SmallVector<int, 8> M;
2946  N->getMask(M);
2947  return ::isUNPCKHMask(M, N->getValueType(0), V2IsSplat);
2948}
2949
2950/// isUNPCKL_v_undef_Mask - Special case of isUNPCKLMask for canonical form
2951/// of vector_shuffle v, v, <0, 4, 1, 5>, i.e. vector_shuffle v, undef,
2952/// <0, 0, 1, 1>
2953static bool isUNPCKL_v_undef_Mask(const SmallVectorImpl<int> &Mask, EVT VT) {
2954  int NumElems = VT.getVectorNumElements();
2955  if (NumElems != 2 && NumElems != 4 && NumElems != 8 && NumElems != 16)
2956    return false;
2957
2958  for (int i = 0, j = 0; i != NumElems; i += 2, ++j) {
2959    int BitI  = Mask[i];
2960    int BitI1 = Mask[i+1];
2961    if (!isUndefOrEqual(BitI, j))
2962      return false;
2963    if (!isUndefOrEqual(BitI1, j))
2964      return false;
2965  }
2966  return true;
2967}
2968
2969bool X86::isUNPCKL_v_undef_Mask(ShuffleVectorSDNode *N) {
2970  SmallVector<int, 8> M;
2971  N->getMask(M);
2972  return ::isUNPCKL_v_undef_Mask(M, N->getValueType(0));
2973}
2974
2975/// isUNPCKH_v_undef_Mask - Special case of isUNPCKHMask for canonical form
2976/// of vector_shuffle v, v, <2, 6, 3, 7>, i.e. vector_shuffle v, undef,
2977/// <2, 2, 3, 3>
2978static bool isUNPCKH_v_undef_Mask(const SmallVectorImpl<int> &Mask, EVT VT) {
2979  int NumElems = VT.getVectorNumElements();
2980  if (NumElems != 2 && NumElems != 4 && NumElems != 8 && NumElems != 16)
2981    return false;
2982
2983  for (int i = 0, j = NumElems / 2; i != NumElems; i += 2, ++j) {
2984    int BitI  = Mask[i];
2985    int BitI1 = Mask[i+1];
2986    if (!isUndefOrEqual(BitI, j))
2987      return false;
2988    if (!isUndefOrEqual(BitI1, j))
2989      return false;
2990  }
2991  return true;
2992}
2993
2994bool X86::isUNPCKH_v_undef_Mask(ShuffleVectorSDNode *N) {
2995  SmallVector<int, 8> M;
2996  N->getMask(M);
2997  return ::isUNPCKH_v_undef_Mask(M, N->getValueType(0));
2998}
2999
3000/// isMOVLMask - Return true if the specified VECTOR_SHUFFLE operand
3001/// specifies a shuffle of elements that is suitable for input to MOVSS,
3002/// MOVSD, and MOVD, i.e. setting the lowest element.
3003static bool isMOVLMask(const SmallVectorImpl<int> &Mask, EVT VT) {
3004  if (VT.getVectorElementType().getSizeInBits() < 32)
3005    return false;
3006
3007  int NumElts = VT.getVectorNumElements();
3008
3009  if (!isUndefOrEqual(Mask[0], NumElts))
3010    return false;
3011
3012  for (int i = 1; i < NumElts; ++i)
3013    if (!isUndefOrEqual(Mask[i], i))
3014      return false;
3015
3016  return true;
3017}
3018
3019bool X86::isMOVLMask(ShuffleVectorSDNode *N) {
3020  SmallVector<int, 8> M;
3021  N->getMask(M);
3022  return ::isMOVLMask(M, N->getValueType(0));
3023}
3024
3025/// isCommutedMOVL - Returns true if the shuffle mask is except the reverse
3026/// of what x86 movss want. X86 movs requires the lowest  element to be lowest
3027/// element of vector 2 and the other elements to come from vector 1 in order.
3028static bool isCommutedMOVLMask(const SmallVectorImpl<int> &Mask, EVT VT,
3029                               bool V2IsSplat = false, bool V2IsUndef = false) {
3030  int NumOps = VT.getVectorNumElements();
3031  if (NumOps != 2 && NumOps != 4 && NumOps != 8 && NumOps != 16)
3032    return false;
3033
3034  if (!isUndefOrEqual(Mask[0], 0))
3035    return false;
3036
3037  for (int i = 1; i < NumOps; ++i)
3038    if (!(isUndefOrEqual(Mask[i], i+NumOps) ||
3039          (V2IsUndef && isUndefOrInRange(Mask[i], NumOps, NumOps*2)) ||
3040          (V2IsSplat && isUndefOrEqual(Mask[i], NumOps))))
3041      return false;
3042
3043  return true;
3044}
3045
3046static bool isCommutedMOVL(ShuffleVectorSDNode *N, bool V2IsSplat = false,
3047                           bool V2IsUndef = false) {
3048  SmallVector<int, 8> M;
3049  N->getMask(M);
3050  return isCommutedMOVLMask(M, N->getValueType(0), V2IsSplat, V2IsUndef);
3051}
3052
3053/// isMOVSHDUPMask - Return true if the specified VECTOR_SHUFFLE operand
3054/// specifies a shuffle of elements that is suitable for input to MOVSHDUP.
3055bool X86::isMOVSHDUPMask(ShuffleVectorSDNode *N) {
3056  if (N->getValueType(0).getVectorNumElements() != 4)
3057    return false;
3058
3059  // Expect 1, 1, 3, 3
3060  for (unsigned i = 0; i < 2; ++i) {
3061    int Elt = N->getMaskElt(i);
3062    if (Elt >= 0 && Elt != 1)
3063      return false;
3064  }
3065
3066  bool HasHi = false;
3067  for (unsigned i = 2; i < 4; ++i) {
3068    int Elt = N->getMaskElt(i);
3069    if (Elt >= 0 && Elt != 3)
3070      return false;
3071    if (Elt == 3)
3072      HasHi = true;
3073  }
3074  // Don't use movshdup if it can be done with a shufps.
3075  // FIXME: verify that matching u, u, 3, 3 is what we want.
3076  return HasHi;
3077}
3078
3079/// isMOVSLDUPMask - Return true if the specified VECTOR_SHUFFLE operand
3080/// specifies a shuffle of elements that is suitable for input to MOVSLDUP.
3081bool X86::isMOVSLDUPMask(ShuffleVectorSDNode *N) {
3082  if (N->getValueType(0).getVectorNumElements() != 4)
3083    return false;
3084
3085  // Expect 0, 0, 2, 2
3086  for (unsigned i = 0; i < 2; ++i)
3087    if (N->getMaskElt(i) > 0)
3088      return false;
3089
3090  bool HasHi = false;
3091  for (unsigned i = 2; i < 4; ++i) {
3092    int Elt = N->getMaskElt(i);
3093    if (Elt >= 0 && Elt != 2)
3094      return false;
3095    if (Elt == 2)
3096      HasHi = true;
3097  }
3098  // Don't use movsldup if it can be done with a shufps.
3099  return HasHi;
3100}
3101
3102/// isMOVDDUPMask - Return true if the specified VECTOR_SHUFFLE operand
3103/// specifies a shuffle of elements that is suitable for input to MOVDDUP.
3104bool X86::isMOVDDUPMask(ShuffleVectorSDNode *N) {
3105  int e = N->getValueType(0).getVectorNumElements() / 2;
3106
3107  for (int i = 0; i < e; ++i)
3108    if (!isUndefOrEqual(N->getMaskElt(i), i))
3109      return false;
3110  for (int i = 0; i < e; ++i)
3111    if (!isUndefOrEqual(N->getMaskElt(e+i), i))
3112      return false;
3113  return true;
3114}
3115
3116/// getShuffleSHUFImmediate - Return the appropriate immediate to shuffle
3117/// the specified VECTOR_SHUFFLE mask with PSHUF* and SHUFP* instructions.
3118unsigned X86::getShuffleSHUFImmediate(SDNode *N) {
3119  ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(N);
3120  int NumOperands = SVOp->getValueType(0).getVectorNumElements();
3121
3122  unsigned Shift = (NumOperands == 4) ? 2 : 1;
3123  unsigned Mask = 0;
3124  for (int i = 0; i < NumOperands; ++i) {
3125    int Val = SVOp->getMaskElt(NumOperands-i-1);
3126    if (Val < 0) Val = 0;
3127    if (Val >= NumOperands) Val -= NumOperands;
3128    Mask |= Val;
3129    if (i != NumOperands - 1)
3130      Mask <<= Shift;
3131  }
3132  return Mask;
3133}
3134
3135/// getShufflePSHUFHWImmediate - Return the appropriate immediate to shuffle
3136/// the specified VECTOR_SHUFFLE mask with the PSHUFHW instruction.
3137unsigned X86::getShufflePSHUFHWImmediate(SDNode *N) {
3138  ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(N);
3139  unsigned Mask = 0;
3140  // 8 nodes, but we only care about the last 4.
3141  for (unsigned i = 7; i >= 4; --i) {
3142    int Val = SVOp->getMaskElt(i);
3143    if (Val >= 0)
3144      Mask |= (Val - 4);
3145    if (i != 4)
3146      Mask <<= 2;
3147  }
3148  return Mask;
3149}
3150
3151/// getShufflePSHUFLWImmediate - Return the appropriate immediate to shuffle
3152/// the specified VECTOR_SHUFFLE mask with the PSHUFLW instruction.
3153unsigned X86::getShufflePSHUFLWImmediate(SDNode *N) {
3154  ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(N);
3155  unsigned Mask = 0;
3156  // 8 nodes, but we only care about the first 4.
3157  for (int i = 3; i >= 0; --i) {
3158    int Val = SVOp->getMaskElt(i);
3159    if (Val >= 0)
3160      Mask |= Val;
3161    if (i != 0)
3162      Mask <<= 2;
3163  }
3164  return Mask;
3165}
3166
3167/// getShufflePALIGNRImmediate - Return the appropriate immediate to shuffle
3168/// the specified VECTOR_SHUFFLE mask with the PALIGNR instruction.
3169unsigned X86::getShufflePALIGNRImmediate(SDNode *N) {
3170  ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(N);
3171  EVT VVT = N->getValueType(0);
3172  unsigned EltSize = VVT.getVectorElementType().getSizeInBits() >> 3;
3173  int Val = 0;
3174
3175  unsigned i, e;
3176  for (i = 0, e = VVT.getVectorNumElements(); i != e; ++i) {
3177    Val = SVOp->getMaskElt(i);
3178    if (Val >= 0)
3179      break;
3180  }
3181  return (Val - i) * EltSize;
3182}
3183
3184/// isZeroNode - Returns true if Elt is a constant zero or a floating point
3185/// constant +0.0.
3186bool X86::isZeroNode(SDValue Elt) {
3187  return ((isa<ConstantSDNode>(Elt) &&
3188           cast<ConstantSDNode>(Elt)->isNullValue()) ||
3189          (isa<ConstantFPSDNode>(Elt) &&
3190           cast<ConstantFPSDNode>(Elt)->getValueAPF().isPosZero()));
3191}
3192
3193/// CommuteVectorShuffle - Swap vector_shuffle operands as well as values in
3194/// their permute mask.
3195static SDValue CommuteVectorShuffle(ShuffleVectorSDNode *SVOp,
3196                                    SelectionDAG &DAG) {
3197  EVT VT = SVOp->getValueType(0);
3198  unsigned NumElems = VT.getVectorNumElements();
3199  SmallVector<int, 8> MaskVec;
3200
3201  for (unsigned i = 0; i != NumElems; ++i) {
3202    int idx = SVOp->getMaskElt(i);
3203    if (idx < 0)
3204      MaskVec.push_back(idx);
3205    else if (idx < (int)NumElems)
3206      MaskVec.push_back(idx + NumElems);
3207    else
3208      MaskVec.push_back(idx - NumElems);
3209  }
3210  return DAG.getVectorShuffle(VT, SVOp->getDebugLoc(), SVOp->getOperand(1),
3211                              SVOp->getOperand(0), &MaskVec[0]);
3212}
3213
3214/// CommuteVectorShuffleMask - Change values in a shuffle permute mask assuming
3215/// the two vector operands have swapped position.
3216static void CommuteVectorShuffleMask(SmallVectorImpl<int> &Mask, EVT VT) {
3217  unsigned NumElems = VT.getVectorNumElements();
3218  for (unsigned i = 0; i != NumElems; ++i) {
3219    int idx = Mask[i];
3220    if (idx < 0)
3221      continue;
3222    else if (idx < (int)NumElems)
3223      Mask[i] = idx + NumElems;
3224    else
3225      Mask[i] = idx - NumElems;
3226  }
3227}
3228
3229/// ShouldXformToMOVHLPS - Return true if the node should be transformed to
3230/// match movhlps. The lower half elements should come from upper half of
3231/// V1 (and in order), and the upper half elements should come from the upper
3232/// half of V2 (and in order).
3233static bool ShouldXformToMOVHLPS(ShuffleVectorSDNode *Op) {
3234  if (Op->getValueType(0).getVectorNumElements() != 4)
3235    return false;
3236  for (unsigned i = 0, e = 2; i != e; ++i)
3237    if (!isUndefOrEqual(Op->getMaskElt(i), i+2))
3238      return false;
3239  for (unsigned i = 2; i != 4; ++i)
3240    if (!isUndefOrEqual(Op->getMaskElt(i), i+4))
3241      return false;
3242  return true;
3243}
3244
3245/// isScalarLoadToVector - Returns true if the node is a scalar load that
3246/// is promoted to a vector. It also returns the LoadSDNode by reference if
3247/// required.
3248static bool isScalarLoadToVector(SDNode *N, LoadSDNode **LD = NULL) {
3249  if (N->getOpcode() != ISD::SCALAR_TO_VECTOR)
3250    return false;
3251  N = N->getOperand(0).getNode();
3252  if (!ISD::isNON_EXTLoad(N))
3253    return false;
3254  if (LD)
3255    *LD = cast<LoadSDNode>(N);
3256  return true;
3257}
3258
3259/// ShouldXformToMOVLP{S|D} - Return true if the node should be transformed to
3260/// match movlp{s|d}. The lower half elements should come from lower half of
3261/// V1 (and in order), and the upper half elements should come from the upper
3262/// half of V2 (and in order). And since V1 will become the source of the
3263/// MOVLP, it must be either a vector load or a scalar load to vector.
3264static bool ShouldXformToMOVLP(SDNode *V1, SDNode *V2,
3265                               ShuffleVectorSDNode *Op) {
3266  if (!ISD::isNON_EXTLoad(V1) && !isScalarLoadToVector(V1))
3267    return false;
3268  // Is V2 is a vector load, don't do this transformation. We will try to use
3269  // load folding shufps op.
3270  if (ISD::isNON_EXTLoad(V2))
3271    return false;
3272
3273  unsigned NumElems = Op->getValueType(0).getVectorNumElements();
3274
3275  if (NumElems != 2 && NumElems != 4)
3276    return false;
3277  for (unsigned i = 0, e = NumElems/2; i != e; ++i)
3278    if (!isUndefOrEqual(Op->getMaskElt(i), i))
3279      return false;
3280  for (unsigned i = NumElems/2; i != NumElems; ++i)
3281    if (!isUndefOrEqual(Op->getMaskElt(i), i+NumElems))
3282      return false;
3283  return true;
3284}
3285
3286/// isSplatVector - Returns true if N is a BUILD_VECTOR node whose elements are
3287/// all the same.
3288static bool isSplatVector(SDNode *N) {
3289  if (N->getOpcode() != ISD::BUILD_VECTOR)
3290    return false;
3291
3292  SDValue SplatValue = N->getOperand(0);
3293  for (unsigned i = 1, e = N->getNumOperands(); i != e; ++i)
3294    if (N->getOperand(i) != SplatValue)
3295      return false;
3296  return true;
3297}
3298
3299/// isZeroShuffle - Returns true if N is a VECTOR_SHUFFLE that can be resolved
3300/// to an zero vector.
3301/// FIXME: move to dag combiner / method on ShuffleVectorSDNode
3302static bool isZeroShuffle(ShuffleVectorSDNode *N) {
3303  SDValue V1 = N->getOperand(0);
3304  SDValue V2 = N->getOperand(1);
3305  unsigned NumElems = N->getValueType(0).getVectorNumElements();
3306  for (unsigned i = 0; i != NumElems; ++i) {
3307    int Idx = N->getMaskElt(i);
3308    if (Idx >= (int)NumElems) {
3309      unsigned Opc = V2.getOpcode();
3310      if (Opc == ISD::UNDEF || ISD::isBuildVectorAllZeros(V2.getNode()))
3311        continue;
3312      if (Opc != ISD::BUILD_VECTOR ||
3313          !X86::isZeroNode(V2.getOperand(Idx-NumElems)))
3314        return false;
3315    } else if (Idx >= 0) {
3316      unsigned Opc = V1.getOpcode();
3317      if (Opc == ISD::UNDEF || ISD::isBuildVectorAllZeros(V1.getNode()))
3318        continue;
3319      if (Opc != ISD::BUILD_VECTOR ||
3320          !X86::isZeroNode(V1.getOperand(Idx)))
3321        return false;
3322    }
3323  }
3324  return true;
3325}
3326
3327/// getZeroVector - Returns a vector of specified type with all zero elements.
3328///
3329static SDValue getZeroVector(EVT VT, bool HasSSE2, SelectionDAG &DAG,
3330                             DebugLoc dl) {
3331  assert(VT.isVector() && "Expected a vector type");
3332
3333  // Always build zero vectors as <4 x i32> or <2 x i32> bitcasted to their dest
3334  // type.  This ensures they get CSE'd.
3335  SDValue Vec;
3336  if (VT.getSizeInBits() == 64) { // MMX
3337    SDValue Cst = DAG.getTargetConstant(0, MVT::i32);
3338    Vec = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v2i32, Cst, Cst);
3339  } else if (HasSSE2) {  // SSE2
3340    SDValue Cst = DAG.getTargetConstant(0, MVT::i32);
3341    Vec = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v4i32, Cst, Cst, Cst, Cst);
3342  } else { // SSE1
3343    SDValue Cst = DAG.getTargetConstantFP(+0.0, MVT::f32);
3344    Vec = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v4f32, Cst, Cst, Cst, Cst);
3345  }
3346  return DAG.getNode(ISD::BIT_CONVERT, dl, VT, Vec);
3347}
3348
3349/// getOnesVector - Returns a vector of specified type with all bits set.
3350///
3351static SDValue getOnesVector(EVT VT, SelectionDAG &DAG, DebugLoc dl) {
3352  assert(VT.isVector() && "Expected a vector type");
3353
3354  // Always build ones vectors as <4 x i32> or <2 x i32> bitcasted to their dest
3355  // type.  This ensures they get CSE'd.
3356  SDValue Cst = DAG.getTargetConstant(~0U, MVT::i32);
3357  SDValue Vec;
3358  if (VT.getSizeInBits() == 64)  // MMX
3359    Vec = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v2i32, Cst, Cst);
3360  else                                              // SSE
3361    Vec = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v4i32, Cst, Cst, Cst, Cst);
3362  return DAG.getNode(ISD::BIT_CONVERT, dl, VT, Vec);
3363}
3364
3365
3366/// NormalizeMask - V2 is a splat, modify the mask (if needed) so all elements
3367/// that point to V2 points to its first element.
3368static SDValue NormalizeMask(ShuffleVectorSDNode *SVOp, SelectionDAG &DAG) {
3369  EVT VT = SVOp->getValueType(0);
3370  unsigned NumElems = VT.getVectorNumElements();
3371
3372  bool Changed = false;
3373  SmallVector<int, 8> MaskVec;
3374  SVOp->getMask(MaskVec);
3375
3376  for (unsigned i = 0; i != NumElems; ++i) {
3377    if (MaskVec[i] > (int)NumElems) {
3378      MaskVec[i] = NumElems;
3379      Changed = true;
3380    }
3381  }
3382  if (Changed)
3383    return DAG.getVectorShuffle(VT, SVOp->getDebugLoc(), SVOp->getOperand(0),
3384                                SVOp->getOperand(1), &MaskVec[0]);
3385  return SDValue(SVOp, 0);
3386}
3387
3388/// getMOVLMask - Returns a vector_shuffle mask for an movs{s|d}, movd
3389/// operation of specified width.
3390static SDValue getMOVL(SelectionDAG &DAG, DebugLoc dl, EVT VT, SDValue V1,
3391                       SDValue V2) {
3392  unsigned NumElems = VT.getVectorNumElements();
3393  SmallVector<int, 8> Mask;
3394  Mask.push_back(NumElems);
3395  for (unsigned i = 1; i != NumElems; ++i)
3396    Mask.push_back(i);
3397  return DAG.getVectorShuffle(VT, dl, V1, V2, &Mask[0]);
3398}
3399
3400/// getUnpackl - Returns a vector_shuffle node for an unpackl operation.
3401static SDValue getUnpackl(SelectionDAG &DAG, DebugLoc dl, EVT VT, SDValue V1,
3402                          SDValue V2) {
3403  unsigned NumElems = VT.getVectorNumElements();
3404  SmallVector<int, 8> Mask;
3405  for (unsigned i = 0, e = NumElems/2; i != e; ++i) {
3406    Mask.push_back(i);
3407    Mask.push_back(i + NumElems);
3408  }
3409  return DAG.getVectorShuffle(VT, dl, V1, V2, &Mask[0]);
3410}
3411
3412/// getUnpackhMask - Returns a vector_shuffle node for an unpackh operation.
3413static SDValue getUnpackh(SelectionDAG &DAG, DebugLoc dl, EVT VT, SDValue V1,
3414                          SDValue V2) {
3415  unsigned NumElems = VT.getVectorNumElements();
3416  unsigned Half = NumElems/2;
3417  SmallVector<int, 8> Mask;
3418  for (unsigned i = 0; i != Half; ++i) {
3419    Mask.push_back(i + Half);
3420    Mask.push_back(i + NumElems + Half);
3421  }
3422  return DAG.getVectorShuffle(VT, dl, V1, V2, &Mask[0]);
3423}
3424
3425/// PromoteSplat - Promote a splat of v4f32, v8i16 or v16i8 to v4i32.
3426static SDValue PromoteSplat(ShuffleVectorSDNode *SV, SelectionDAG &DAG,
3427                            bool HasSSE2) {
3428  if (SV->getValueType(0).getVectorNumElements() <= 4)
3429    return SDValue(SV, 0);
3430
3431  EVT PVT = MVT::v4f32;
3432  EVT VT = SV->getValueType(0);
3433  DebugLoc dl = SV->getDebugLoc();
3434  SDValue V1 = SV->getOperand(0);
3435  int NumElems = VT.getVectorNumElements();
3436  int EltNo = SV->getSplatIndex();
3437
3438  // unpack elements to the correct location
3439  while (NumElems > 4) {
3440    if (EltNo < NumElems/2) {
3441      V1 = getUnpackl(DAG, dl, VT, V1, V1);
3442    } else {
3443      V1 = getUnpackh(DAG, dl, VT, V1, V1);
3444      EltNo -= NumElems/2;
3445    }
3446    NumElems >>= 1;
3447  }
3448
3449  // Perform the splat.
3450  int SplatMask[4] = { EltNo, EltNo, EltNo, EltNo };
3451  V1 = DAG.getNode(ISD::BIT_CONVERT, dl, PVT, V1);
3452  V1 = DAG.getVectorShuffle(PVT, dl, V1, DAG.getUNDEF(PVT), &SplatMask[0]);
3453  return DAG.getNode(ISD::BIT_CONVERT, dl, VT, V1);
3454}
3455
3456/// getShuffleVectorZeroOrUndef - Return a vector_shuffle of the specified
3457/// vector of zero or undef vector.  This produces a shuffle where the low
3458/// element of V2 is swizzled into the zero/undef vector, landing at element
3459/// Idx.  This produces a shuffle mask like 4,1,2,3 (idx=0) or  0,1,2,4 (idx=3).
3460static SDValue getShuffleVectorZeroOrUndef(SDValue V2, unsigned Idx,
3461                                             bool isZero, bool HasSSE2,
3462                                             SelectionDAG &DAG) {
3463  EVT VT = V2.getValueType();
3464  SDValue V1 = isZero
3465    ? getZeroVector(VT, HasSSE2, DAG, V2.getDebugLoc()) : DAG.getUNDEF(VT);
3466  unsigned NumElems = VT.getVectorNumElements();
3467  SmallVector<int, 16> MaskVec;
3468  for (unsigned i = 0; i != NumElems; ++i)
3469    // If this is the insertion idx, put the low elt of V2 here.
3470    MaskVec.push_back(i == Idx ? NumElems : i);
3471  return DAG.getVectorShuffle(VT, V2.getDebugLoc(), V1, V2, &MaskVec[0]);
3472}
3473
3474/// getNumOfConsecutiveZeros - Return the number of elements in a result of
3475/// a shuffle that is zero.
3476static
3477unsigned getNumOfConsecutiveZeros(ShuffleVectorSDNode *SVOp, int NumElems,
3478                                  bool Low, SelectionDAG &DAG) {
3479  unsigned NumZeros = 0;
3480  for (int i = 0; i < NumElems; ++i) {
3481    unsigned Index = Low ? i : NumElems-i-1;
3482    int Idx = SVOp->getMaskElt(Index);
3483    if (Idx < 0) {
3484      ++NumZeros;
3485      continue;
3486    }
3487    SDValue Elt = DAG.getShuffleScalarElt(SVOp, Index);
3488    if (Elt.getNode() && X86::isZeroNode(Elt))
3489      ++NumZeros;
3490    else
3491      break;
3492  }
3493  return NumZeros;
3494}
3495
3496/// isVectorShift - Returns true if the shuffle can be implemented as a
3497/// logical left or right shift of a vector.
3498/// FIXME: split into pslldqi, psrldqi, palignr variants.
3499static bool isVectorShift(ShuffleVectorSDNode *SVOp, SelectionDAG &DAG,
3500                          bool &isLeft, SDValue &ShVal, unsigned &ShAmt) {
3501  unsigned NumElems = SVOp->getValueType(0).getVectorNumElements();
3502
3503  isLeft = true;
3504  unsigned NumZeros = getNumOfConsecutiveZeros(SVOp, NumElems, true, DAG);
3505  if (!NumZeros) {
3506    isLeft = false;
3507    NumZeros = getNumOfConsecutiveZeros(SVOp, NumElems, false, DAG);
3508    if (!NumZeros)
3509      return false;
3510  }
3511  bool SeenV1 = false;
3512  bool SeenV2 = false;
3513  for (unsigned i = NumZeros; i < NumElems; ++i) {
3514    unsigned Val = isLeft ? (i - NumZeros) : i;
3515    int Idx_ = SVOp->getMaskElt(isLeft ? i : (i - NumZeros));
3516    if (Idx_ < 0)
3517      continue;
3518    unsigned Idx = (unsigned) Idx_;
3519    if (Idx < NumElems)
3520      SeenV1 = true;
3521    else {
3522      Idx -= NumElems;
3523      SeenV2 = true;
3524    }
3525    if (Idx != Val)
3526      return false;
3527  }
3528  if (SeenV1 && SeenV2)
3529    return false;
3530
3531  ShVal = SeenV1 ? SVOp->getOperand(0) : SVOp->getOperand(1);
3532  ShAmt = NumZeros;
3533  return true;
3534}
3535
3536
3537/// LowerBuildVectorv16i8 - Custom lower build_vector of v16i8.
3538///
3539static SDValue LowerBuildVectorv16i8(SDValue Op, unsigned NonZeros,
3540                                       unsigned NumNonZero, unsigned NumZero,
3541                                       SelectionDAG &DAG,
3542                                       const TargetLowering &TLI) {
3543  if (NumNonZero > 8)
3544    return SDValue();
3545
3546  DebugLoc dl = Op.getDebugLoc();
3547  SDValue V(0, 0);
3548  bool First = true;
3549  for (unsigned i = 0; i < 16; ++i) {
3550    bool ThisIsNonZero = (NonZeros & (1 << i)) != 0;
3551    if (ThisIsNonZero && First) {
3552      if (NumZero)
3553        V = getZeroVector(MVT::v8i16, true, DAG, dl);
3554      else
3555        V = DAG.getUNDEF(MVT::v8i16);
3556      First = false;
3557    }
3558
3559    if ((i & 1) != 0) {
3560      SDValue ThisElt(0, 0), LastElt(0, 0);
3561      bool LastIsNonZero = (NonZeros & (1 << (i-1))) != 0;
3562      if (LastIsNonZero) {
3563        LastElt = DAG.getNode(ISD::ZERO_EXTEND, dl,
3564                              MVT::i16, Op.getOperand(i-1));
3565      }
3566      if (ThisIsNonZero) {
3567        ThisElt = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i16, Op.getOperand(i));
3568        ThisElt = DAG.getNode(ISD::SHL, dl, MVT::i16,
3569                              ThisElt, DAG.getConstant(8, MVT::i8));
3570        if (LastIsNonZero)
3571          ThisElt = DAG.getNode(ISD::OR, dl, MVT::i16, ThisElt, LastElt);
3572      } else
3573        ThisElt = LastElt;
3574
3575      if (ThisElt.getNode())
3576        V = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, MVT::v8i16, V, ThisElt,
3577                        DAG.getIntPtrConstant(i/2));
3578    }
3579  }
3580
3581  return DAG.getNode(ISD::BIT_CONVERT, dl, MVT::v16i8, V);
3582}
3583
3584/// LowerBuildVectorv8i16 - Custom lower build_vector of v8i16.
3585///
3586static SDValue LowerBuildVectorv8i16(SDValue Op, unsigned NonZeros,
3587                                     unsigned NumNonZero, unsigned NumZero,
3588                                     SelectionDAG &DAG,
3589                                     const TargetLowering &TLI) {
3590  if (NumNonZero > 4)
3591    return SDValue();
3592
3593  DebugLoc dl = Op.getDebugLoc();
3594  SDValue V(0, 0);
3595  bool First = true;
3596  for (unsigned i = 0; i < 8; ++i) {
3597    bool isNonZero = (NonZeros & (1 << i)) != 0;
3598    if (isNonZero) {
3599      if (First) {
3600        if (NumZero)
3601          V = getZeroVector(MVT::v8i16, true, DAG, dl);
3602        else
3603          V = DAG.getUNDEF(MVT::v8i16);
3604        First = false;
3605      }
3606      V = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl,
3607                      MVT::v8i16, V, Op.getOperand(i),
3608                      DAG.getIntPtrConstant(i));
3609    }
3610  }
3611
3612  return V;
3613}
3614
3615/// getVShift - Return a vector logical shift node.
3616///
3617static SDValue getVShift(bool isLeft, EVT VT, SDValue SrcOp,
3618                         unsigned NumBits, SelectionDAG &DAG,
3619                         const TargetLowering &TLI, DebugLoc dl) {
3620  bool isMMX = VT.getSizeInBits() == 64;
3621  EVT ShVT = isMMX ? MVT::v1i64 : MVT::v2i64;
3622  unsigned Opc = isLeft ? X86ISD::VSHL : X86ISD::VSRL;
3623  SrcOp = DAG.getNode(ISD::BIT_CONVERT, dl, ShVT, SrcOp);
3624  return DAG.getNode(ISD::BIT_CONVERT, dl, VT,
3625                     DAG.getNode(Opc, dl, ShVT, SrcOp,
3626                             DAG.getConstant(NumBits, TLI.getShiftAmountTy())));
3627}
3628
3629SDValue
3630X86TargetLowering::LowerAsSplatVectorLoad(SDValue SrcOp, EVT VT, DebugLoc dl,
3631                                          SelectionDAG &DAG) const {
3632
3633  // Check if the scalar load can be widened into a vector load. And if
3634  // the address is "base + cst" see if the cst can be "absorbed" into
3635  // the shuffle mask.
3636  if (LoadSDNode *LD = dyn_cast<LoadSDNode>(SrcOp)) {
3637    SDValue Ptr = LD->getBasePtr();
3638    if (!ISD::isNormalLoad(LD) || LD->isVolatile())
3639      return SDValue();
3640    EVT PVT = LD->getValueType(0);
3641    if (PVT != MVT::i32 && PVT != MVT::f32)
3642      return SDValue();
3643
3644    int FI = -1;
3645    int64_t Offset = 0;
3646    if (FrameIndexSDNode *FINode = dyn_cast<FrameIndexSDNode>(Ptr)) {
3647      FI = FINode->getIndex();
3648      Offset = 0;
3649    } else if (Ptr.getOpcode() == ISD::ADD &&
3650               isa<ConstantSDNode>(Ptr.getOperand(1)) &&
3651               isa<FrameIndexSDNode>(Ptr.getOperand(0))) {
3652      FI = cast<FrameIndexSDNode>(Ptr.getOperand(0))->getIndex();
3653      Offset = Ptr.getConstantOperandVal(1);
3654      Ptr = Ptr.getOperand(0);
3655    } else {
3656      return SDValue();
3657    }
3658
3659    SDValue Chain = LD->getChain();
3660    // Make sure the stack object alignment is at least 16.
3661    MachineFrameInfo *MFI = DAG.getMachineFunction().getFrameInfo();
3662    if (DAG.InferPtrAlignment(Ptr) < 16) {
3663      if (MFI->isFixedObjectIndex(FI)) {
3664        // Can't change the alignment. FIXME: It's possible to compute
3665        // the exact stack offset and reference FI + adjust offset instead.
3666        // If someone *really* cares about this. That's the way to implement it.
3667        return SDValue();
3668      } else {
3669        MFI->setObjectAlignment(FI, 16);
3670      }
3671    }
3672
3673    // (Offset % 16) must be multiple of 4. Then address is then
3674    // Ptr + (Offset & ~15).
3675    if (Offset < 0)
3676      return SDValue();
3677    if ((Offset % 16) & 3)
3678      return SDValue();
3679    int64_t StartOffset = Offset & ~15;
3680    if (StartOffset)
3681      Ptr = DAG.getNode(ISD::ADD, Ptr.getDebugLoc(), Ptr.getValueType(),
3682                        Ptr,DAG.getConstant(StartOffset, Ptr.getValueType()));
3683
3684    int EltNo = (Offset - StartOffset) >> 2;
3685    int Mask[4] = { EltNo, EltNo, EltNo, EltNo };
3686    EVT VT = (PVT == MVT::i32) ? MVT::v4i32 : MVT::v4f32;
3687    SDValue V1 = DAG.getLoad(VT, dl, Chain, Ptr,LD->getSrcValue(),0,
3688                             false, false, 0);
3689    // Canonicalize it to a v4i32 shuffle.
3690    V1 = DAG.getNode(ISD::BIT_CONVERT, dl, MVT::v4i32, V1);
3691    return DAG.getNode(ISD::BIT_CONVERT, dl, VT,
3692                       DAG.getVectorShuffle(MVT::v4i32, dl, V1,
3693                                            DAG.getUNDEF(MVT::v4i32), &Mask[0]));
3694  }
3695
3696  return SDValue();
3697}
3698
3699/// EltsFromConsecutiveLoads - Given the initializing elements 'Elts' of a
3700/// vector of type 'VT', see if the elements can be replaced by a single large
3701/// load which has the same value as a build_vector whose operands are 'elts'.
3702///
3703/// Example: <load i32 *a, load i32 *a+4, undef, undef> -> zextload a
3704///
3705/// FIXME: we'd also like to handle the case where the last elements are zero
3706/// rather than undef via VZEXT_LOAD, but we do not detect that case today.
3707/// There's even a handy isZeroNode for that purpose.
3708static SDValue EltsFromConsecutiveLoads(EVT VT, SmallVectorImpl<SDValue> &Elts,
3709                                        DebugLoc &dl, SelectionDAG &DAG) {
3710  EVT EltVT = VT.getVectorElementType();
3711  unsigned NumElems = Elts.size();
3712
3713  LoadSDNode *LDBase = NULL;
3714  unsigned LastLoadedElt = -1U;
3715
3716  // For each element in the initializer, see if we've found a load or an undef.
3717  // If we don't find an initial load element, or later load elements are
3718  // non-consecutive, bail out.
3719  for (unsigned i = 0; i < NumElems; ++i) {
3720    SDValue Elt = Elts[i];
3721
3722    if (!Elt.getNode() ||
3723        (Elt.getOpcode() != ISD::UNDEF && !ISD::isNON_EXTLoad(Elt.getNode())))
3724      return SDValue();
3725    if (!LDBase) {
3726      if (Elt.getNode()->getOpcode() == ISD::UNDEF)
3727        return SDValue();
3728      LDBase = cast<LoadSDNode>(Elt.getNode());
3729      LastLoadedElt = i;
3730      continue;
3731    }
3732    if (Elt.getOpcode() == ISD::UNDEF)
3733      continue;
3734
3735    LoadSDNode *LD = cast<LoadSDNode>(Elt);
3736    if (!DAG.isConsecutiveLoad(LD, LDBase, EltVT.getSizeInBits()/8, i))
3737      return SDValue();
3738    LastLoadedElt = i;
3739  }
3740
3741  // If we have found an entire vector of loads and undefs, then return a large
3742  // load of the entire vector width starting at the base pointer.  If we found
3743  // consecutive loads for the low half, generate a vzext_load node.
3744  if (LastLoadedElt == NumElems - 1) {
3745    if (DAG.InferPtrAlignment(LDBase->getBasePtr()) >= 16)
3746      return DAG.getLoad(VT, dl, LDBase->getChain(), LDBase->getBasePtr(),
3747                         LDBase->getSrcValue(), LDBase->getSrcValueOffset(),
3748                         LDBase->isVolatile(), LDBase->isNonTemporal(), 0);
3749    return DAG.getLoad(VT, dl, LDBase->getChain(), LDBase->getBasePtr(),
3750                       LDBase->getSrcValue(), LDBase->getSrcValueOffset(),
3751                       LDBase->isVolatile(), LDBase->isNonTemporal(),
3752                       LDBase->getAlignment());
3753  } else if (NumElems == 4 && LastLoadedElt == 1) {
3754    SDVTList Tys = DAG.getVTList(MVT::v2i64, MVT::Other);
3755    SDValue Ops[] = { LDBase->getChain(), LDBase->getBasePtr() };
3756    SDValue ResNode = DAG.getNode(X86ISD::VZEXT_LOAD, dl, Tys, Ops, 2);
3757    return DAG.getNode(ISD::BIT_CONVERT, dl, VT, ResNode);
3758  }
3759  return SDValue();
3760}
3761
3762SDValue
3763X86TargetLowering::LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG) const {
3764  DebugLoc dl = Op.getDebugLoc();
3765  // All zero's are handled with pxor, all one's are handled with pcmpeqd.
3766  if (ISD::isBuildVectorAllZeros(Op.getNode())
3767      || ISD::isBuildVectorAllOnes(Op.getNode())) {
3768    // Canonicalize this to either <4 x i32> or <2 x i32> (SSE vs MMX) to
3769    // 1) ensure the zero vectors are CSE'd, and 2) ensure that i64 scalars are
3770    // eliminated on x86-32 hosts.
3771    if (Op.getValueType() == MVT::v4i32 || Op.getValueType() == MVT::v2i32)
3772      return Op;
3773
3774    if (ISD::isBuildVectorAllOnes(Op.getNode()))
3775      return getOnesVector(Op.getValueType(), DAG, dl);
3776    return getZeroVector(Op.getValueType(), Subtarget->hasSSE2(), DAG, dl);
3777  }
3778
3779  EVT VT = Op.getValueType();
3780  EVT ExtVT = VT.getVectorElementType();
3781  unsigned EVTBits = ExtVT.getSizeInBits();
3782
3783  unsigned NumElems = Op.getNumOperands();
3784  unsigned NumZero  = 0;
3785  unsigned NumNonZero = 0;
3786  unsigned NonZeros = 0;
3787  bool IsAllConstants = true;
3788  SmallSet<SDValue, 8> Values;
3789  for (unsigned i = 0; i < NumElems; ++i) {
3790    SDValue Elt = Op.getOperand(i);
3791    if (Elt.getOpcode() == ISD::UNDEF)
3792      continue;
3793    Values.insert(Elt);
3794    if (Elt.getOpcode() != ISD::Constant &&
3795        Elt.getOpcode() != ISD::ConstantFP)
3796      IsAllConstants = false;
3797    if (X86::isZeroNode(Elt))
3798      NumZero++;
3799    else {
3800      NonZeros |= (1 << i);
3801      NumNonZero++;
3802    }
3803  }
3804
3805  if (NumNonZero == 0) {
3806    // All undef vector. Return an UNDEF.  All zero vectors were handled above.
3807    return DAG.getUNDEF(VT);
3808  }
3809
3810  // Special case for single non-zero, non-undef, element.
3811  if (NumNonZero == 1) {
3812    unsigned Idx = CountTrailingZeros_32(NonZeros);
3813    SDValue Item = Op.getOperand(Idx);
3814
3815    // If this is an insertion of an i64 value on x86-32, and if the top bits of
3816    // the value are obviously zero, truncate the value to i32 and do the
3817    // insertion that way.  Only do this if the value is non-constant or if the
3818    // value is a constant being inserted into element 0.  It is cheaper to do
3819    // a constant pool load than it is to do a movd + shuffle.
3820    if (ExtVT == MVT::i64 && !Subtarget->is64Bit() &&
3821        (!IsAllConstants || Idx == 0)) {
3822      if (DAG.MaskedValueIsZero(Item, APInt::getBitsSet(64, 32, 64))) {
3823        // Handle MMX and SSE both.
3824        EVT VecVT = VT == MVT::v2i64 ? MVT::v4i32 : MVT::v2i32;
3825        unsigned VecElts = VT == MVT::v2i64 ? 4 : 2;
3826
3827        // Truncate the value (which may itself be a constant) to i32, and
3828        // convert it to a vector with movd (S2V+shuffle to zero extend).
3829        Item = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, Item);
3830        Item = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VecVT, Item);
3831        Item = getShuffleVectorZeroOrUndef(Item, 0, true,
3832                                           Subtarget->hasSSE2(), DAG);
3833
3834        // Now we have our 32-bit value zero extended in the low element of
3835        // a vector.  If Idx != 0, swizzle it into place.
3836        if (Idx != 0) {
3837          SmallVector<int, 4> Mask;
3838          Mask.push_back(Idx);
3839          for (unsigned i = 1; i != VecElts; ++i)
3840            Mask.push_back(i);
3841          Item = DAG.getVectorShuffle(VecVT, dl, Item,
3842                                      DAG.getUNDEF(Item.getValueType()),
3843                                      &Mask[0]);
3844        }
3845        return DAG.getNode(ISD::BIT_CONVERT, dl, Op.getValueType(), Item);
3846      }
3847    }
3848
3849    // If we have a constant or non-constant insertion into the low element of
3850    // a vector, we can do this with SCALAR_TO_VECTOR + shuffle of zero into
3851    // the rest of the elements.  This will be matched as movd/movq/movss/movsd
3852    // depending on what the source datatype is.
3853    if (Idx == 0) {
3854      if (NumZero == 0) {
3855        return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Item);
3856      } else if (ExtVT == MVT::i32 || ExtVT == MVT::f32 || ExtVT == MVT::f64 ||
3857          (ExtVT == MVT::i64 && Subtarget->is64Bit())) {
3858        Item = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Item);
3859        // Turn it into a MOVL (i.e. movss, movsd, or movd) to a zero vector.
3860        return getShuffleVectorZeroOrUndef(Item, 0, true, Subtarget->hasSSE2(),
3861                                           DAG);
3862      } else if (ExtVT == MVT::i16 || ExtVT == MVT::i8) {
3863        Item = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i32, Item);
3864        EVT MiddleVT = VT.getSizeInBits() == 64 ? MVT::v2i32 : MVT::v4i32;
3865        Item = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MiddleVT, Item);
3866        Item = getShuffleVectorZeroOrUndef(Item, 0, true,
3867                                           Subtarget->hasSSE2(), DAG);
3868        return DAG.getNode(ISD::BIT_CONVERT, dl, VT, Item);
3869      }
3870    }
3871
3872    // Is it a vector logical left shift?
3873    if (NumElems == 2 && Idx == 1 &&
3874        X86::isZeroNode(Op.getOperand(0)) &&
3875        !X86::isZeroNode(Op.getOperand(1))) {
3876      unsigned NumBits = VT.getSizeInBits();
3877      return getVShift(true, VT,
3878                       DAG.getNode(ISD::SCALAR_TO_VECTOR, dl,
3879                                   VT, Op.getOperand(1)),
3880                       NumBits/2, DAG, *this, dl);
3881    }
3882
3883    if (IsAllConstants) // Otherwise, it's better to do a constpool load.
3884      return SDValue();
3885
3886    // Otherwise, if this is a vector with i32 or f32 elements, and the element
3887    // is a non-constant being inserted into an element other than the low one,
3888    // we can't use a constant pool load.  Instead, use SCALAR_TO_VECTOR (aka
3889    // movd/movss) to move this into the low element, then shuffle it into
3890    // place.
3891    if (EVTBits == 32) {
3892      Item = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Item);
3893
3894      // Turn it into a shuffle of zero and zero-extended scalar to vector.
3895      Item = getShuffleVectorZeroOrUndef(Item, 0, NumZero > 0,
3896                                         Subtarget->hasSSE2(), DAG);
3897      SmallVector<int, 8> MaskVec;
3898      for (unsigned i = 0; i < NumElems; i++)
3899        MaskVec.push_back(i == Idx ? 0 : 1);
3900      return DAG.getVectorShuffle(VT, dl, Item, DAG.getUNDEF(VT), &MaskVec[0]);
3901    }
3902  }
3903
3904  // Splat is obviously ok. Let legalizer expand it to a shuffle.
3905  if (Values.size() == 1) {
3906    if (EVTBits == 32) {
3907      // Instead of a shuffle like this:
3908      // shuffle (scalar_to_vector (load (ptr + 4))), undef, <0, 0, 0, 0>
3909      // Check if it's possible to issue this instead.
3910      // shuffle (vload ptr)), undef, <1, 1, 1, 1>
3911      unsigned Idx = CountTrailingZeros_32(NonZeros);
3912      SDValue Item = Op.getOperand(Idx);
3913      if (Op.getNode()->isOnlyUserOf(Item.getNode()))
3914        return LowerAsSplatVectorLoad(Item, VT, dl, DAG);
3915    }
3916    return SDValue();
3917  }
3918
3919  // A vector full of immediates; various special cases are already
3920  // handled, so this is best done with a single constant-pool load.
3921  if (IsAllConstants)
3922    return SDValue();
3923
3924  // Let legalizer expand 2-wide build_vectors.
3925  if (EVTBits == 64) {
3926    if (NumNonZero == 1) {
3927      // One half is zero or undef.
3928      unsigned Idx = CountTrailingZeros_32(NonZeros);
3929      SDValue V2 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT,
3930                                 Op.getOperand(Idx));
3931      return getShuffleVectorZeroOrUndef(V2, Idx, true,
3932                                         Subtarget->hasSSE2(), DAG);
3933    }
3934    return SDValue();
3935  }
3936
3937  // If element VT is < 32 bits, convert it to inserts into a zero vector.
3938  if (EVTBits == 8 && NumElems == 16) {
3939    SDValue V = LowerBuildVectorv16i8(Op, NonZeros,NumNonZero,NumZero, DAG,
3940                                        *this);
3941    if (V.getNode()) return V;
3942  }
3943
3944  if (EVTBits == 16 && NumElems == 8) {
3945    SDValue V = LowerBuildVectorv8i16(Op, NonZeros,NumNonZero,NumZero, DAG,
3946                                        *this);
3947    if (V.getNode()) return V;
3948  }
3949
3950  // If element VT is == 32 bits, turn it into a number of shuffles.
3951  SmallVector<SDValue, 8> V;
3952  V.resize(NumElems);
3953  if (NumElems == 4 && NumZero > 0) {
3954    for (unsigned i = 0; i < 4; ++i) {
3955      bool isZero = !(NonZeros & (1 << i));
3956      if (isZero)
3957        V[i] = getZeroVector(VT, Subtarget->hasSSE2(), DAG, dl);
3958      else
3959        V[i] = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Op.getOperand(i));
3960    }
3961
3962    for (unsigned i = 0; i < 2; ++i) {
3963      switch ((NonZeros & (0x3 << i*2)) >> (i*2)) {
3964        default: break;
3965        case 0:
3966          V[i] = V[i*2];  // Must be a zero vector.
3967          break;
3968        case 1:
3969          V[i] = getMOVL(DAG, dl, VT, V[i*2+1], V[i*2]);
3970          break;
3971        case 2:
3972          V[i] = getMOVL(DAG, dl, VT, V[i*2], V[i*2+1]);
3973          break;
3974        case 3:
3975          V[i] = getUnpackl(DAG, dl, VT, V[i*2], V[i*2+1]);
3976          break;
3977      }
3978    }
3979
3980    SmallVector<int, 8> MaskVec;
3981    bool Reverse = (NonZeros & 0x3) == 2;
3982    for (unsigned i = 0; i < 2; ++i)
3983      MaskVec.push_back(Reverse ? 1-i : i);
3984    Reverse = ((NonZeros & (0x3 << 2)) >> 2) == 2;
3985    for (unsigned i = 0; i < 2; ++i)
3986      MaskVec.push_back(Reverse ? 1-i+NumElems : i+NumElems);
3987    return DAG.getVectorShuffle(VT, dl, V[0], V[1], &MaskVec[0]);
3988  }
3989
3990  if (Values.size() > 1 && VT.getSizeInBits() == 128) {
3991    // Check for a build vector of consecutive loads.
3992    for (unsigned i = 0; i < NumElems; ++i)
3993      V[i] = Op.getOperand(i);
3994
3995    // Check for elements which are consecutive loads.
3996    SDValue LD = EltsFromConsecutiveLoads(VT, V, dl, DAG);
3997    if (LD.getNode())
3998      return LD;
3999
4000    // For SSE 4.1, use inserts into undef.
4001    if (getSubtarget()->hasSSE41()) {
4002      V[0] = DAG.getUNDEF(VT);
4003      for (unsigned i = 0; i < NumElems; ++i)
4004        if (Op.getOperand(i).getOpcode() != ISD::UNDEF)
4005          V[0] = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VT, V[0],
4006                             Op.getOperand(i), DAG.getIntPtrConstant(i));
4007      return V[0];
4008    }
4009
4010    // Otherwise, expand into a number of unpckl*
4011    // e.g. for v4f32
4012    //   Step 1: unpcklps 0, 2 ==> X: <?, ?, 2, 0>
4013    //         : unpcklps 1, 3 ==> Y: <?, ?, 3, 1>
4014    //   Step 2: unpcklps X, Y ==>    <3, 2, 1, 0>
4015    for (unsigned i = 0; i < NumElems; ++i)
4016      V[i] = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Op.getOperand(i));
4017    NumElems >>= 1;
4018    while (NumElems != 0) {
4019      for (unsigned i = 0; i < NumElems; ++i)
4020        V[i] = getUnpackl(DAG, dl, VT, V[i], V[i + NumElems]);
4021      NumElems >>= 1;
4022    }
4023    return V[0];
4024  }
4025  return SDValue();
4026}
4027
4028SDValue
4029X86TargetLowering::LowerCONCAT_VECTORS(SDValue Op, SelectionDAG &DAG) const {
4030  // We support concatenate two MMX registers and place them in a MMX
4031  // register.  This is better than doing a stack convert.
4032  DebugLoc dl = Op.getDebugLoc();
4033  EVT ResVT = Op.getValueType();
4034  assert(Op.getNumOperands() == 2);
4035  assert(ResVT == MVT::v2i64 || ResVT == MVT::v4i32 ||
4036         ResVT == MVT::v8i16 || ResVT == MVT::v16i8);
4037  int Mask[2];
4038  SDValue InVec = DAG.getNode(ISD::BIT_CONVERT,dl, MVT::v1i64, Op.getOperand(0));
4039  SDValue VecOp = DAG.getNode(X86ISD::MOVQ2DQ, dl, MVT::v2i64, InVec);
4040  InVec = Op.getOperand(1);
4041  if (InVec.getOpcode() == ISD::SCALAR_TO_VECTOR) {
4042    unsigned NumElts = ResVT.getVectorNumElements();
4043    VecOp = DAG.getNode(ISD::BIT_CONVERT, dl, ResVT, VecOp);
4044    VecOp = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, ResVT, VecOp,
4045                       InVec.getOperand(0), DAG.getIntPtrConstant(NumElts/2+1));
4046  } else {
4047    InVec = DAG.getNode(ISD::BIT_CONVERT, dl, MVT::v1i64, InVec);
4048    SDValue VecOp2 = DAG.getNode(X86ISD::MOVQ2DQ, dl, MVT::v2i64, InVec);
4049    Mask[0] = 0; Mask[1] = 2;
4050    VecOp = DAG.getVectorShuffle(MVT::v2i64, dl, VecOp, VecOp2, Mask);
4051  }
4052  return DAG.getNode(ISD::BIT_CONVERT, dl, ResVT, VecOp);
4053}
4054
4055// v8i16 shuffles - Prefer shuffles in the following order:
4056// 1. [all]   pshuflw, pshufhw, optional move
4057// 2. [ssse3] 1 x pshufb
4058// 3. [ssse3] 2 x pshufb + 1 x por
4059// 4. [all]   mov + pshuflw + pshufhw + N x (pextrw + pinsrw)
4060static
4061SDValue LowerVECTOR_SHUFFLEv8i16(ShuffleVectorSDNode *SVOp,
4062                                 SelectionDAG &DAG,
4063                                 const X86TargetLowering &TLI) {
4064  SDValue V1 = SVOp->getOperand(0);
4065  SDValue V2 = SVOp->getOperand(1);
4066  DebugLoc dl = SVOp->getDebugLoc();
4067  SmallVector<int, 8> MaskVals;
4068
4069  // Determine if more than 1 of the words in each of the low and high quadwords
4070  // of the result come from the same quadword of one of the two inputs.  Undef
4071  // mask values count as coming from any quadword, for better codegen.
4072  SmallVector<unsigned, 4> LoQuad(4);
4073  SmallVector<unsigned, 4> HiQuad(4);
4074  BitVector InputQuads(4);
4075  for (unsigned i = 0; i < 8; ++i) {
4076    SmallVectorImpl<unsigned> &Quad = i < 4 ? LoQuad : HiQuad;
4077    int EltIdx = SVOp->getMaskElt(i);
4078    MaskVals.push_back(EltIdx);
4079    if (EltIdx < 0) {
4080      ++Quad[0];
4081      ++Quad[1];
4082      ++Quad[2];
4083      ++Quad[3];
4084      continue;
4085    }
4086    ++Quad[EltIdx / 4];
4087    InputQuads.set(EltIdx / 4);
4088  }
4089
4090  int BestLoQuad = -1;
4091  unsigned MaxQuad = 1;
4092  for (unsigned i = 0; i < 4; ++i) {
4093    if (LoQuad[i] > MaxQuad) {
4094      BestLoQuad = i;
4095      MaxQuad = LoQuad[i];
4096    }
4097  }
4098
4099  int BestHiQuad = -1;
4100  MaxQuad = 1;
4101  for (unsigned i = 0; i < 4; ++i) {
4102    if (HiQuad[i] > MaxQuad) {
4103      BestHiQuad = i;
4104      MaxQuad = HiQuad[i];
4105    }
4106  }
4107
4108  // For SSSE3, If all 8 words of the result come from only 1 quadword of each
4109  // of the two input vectors, shuffle them into one input vector so only a
4110  // single pshufb instruction is necessary. If There are more than 2 input
4111  // quads, disable the next transformation since it does not help SSSE3.
4112  bool V1Used = InputQuads[0] || InputQuads[1];
4113  bool V2Used = InputQuads[2] || InputQuads[3];
4114  if (TLI.getSubtarget()->hasSSSE3()) {
4115    if (InputQuads.count() == 2 && V1Used && V2Used) {
4116      BestLoQuad = InputQuads.find_first();
4117      BestHiQuad = InputQuads.find_next(BestLoQuad);
4118    }
4119    if (InputQuads.count() > 2) {
4120      BestLoQuad = -1;
4121      BestHiQuad = -1;
4122    }
4123  }
4124
4125  // If BestLoQuad or BestHiQuad are set, shuffle the quads together and update
4126  // the shuffle mask.  If a quad is scored as -1, that means that it contains
4127  // words from all 4 input quadwords.
4128  SDValue NewV;
4129  if (BestLoQuad >= 0 || BestHiQuad >= 0) {
4130    SmallVector<int, 8> MaskV;
4131    MaskV.push_back(BestLoQuad < 0 ? 0 : BestLoQuad);
4132    MaskV.push_back(BestHiQuad < 0 ? 1 : BestHiQuad);
4133    NewV = DAG.getVectorShuffle(MVT::v2i64, dl,
4134                  DAG.getNode(ISD::BIT_CONVERT, dl, MVT::v2i64, V1),
4135                  DAG.getNode(ISD::BIT_CONVERT, dl, MVT::v2i64, V2), &MaskV[0]);
4136    NewV = DAG.getNode(ISD::BIT_CONVERT, dl, MVT::v8i16, NewV);
4137
4138    // Rewrite the MaskVals and assign NewV to V1 if NewV now contains all the
4139    // source words for the shuffle, to aid later transformations.
4140    bool AllWordsInNewV = true;
4141    bool InOrder[2] = { true, true };
4142    for (unsigned i = 0; i != 8; ++i) {
4143      int idx = MaskVals[i];
4144      if (idx != (int)i)
4145        InOrder[i/4] = false;
4146      if (idx < 0 || (idx/4) == BestLoQuad || (idx/4) == BestHiQuad)
4147        continue;
4148      AllWordsInNewV = false;
4149      break;
4150    }
4151
4152    bool pshuflw = AllWordsInNewV, pshufhw = AllWordsInNewV;
4153    if (AllWordsInNewV) {
4154      for (int i = 0; i != 8; ++i) {
4155        int idx = MaskVals[i];
4156        if (idx < 0)
4157          continue;
4158        idx = MaskVals[i] = (idx / 4) == BestLoQuad ? (idx & 3) : (idx & 3) + 4;
4159        if ((idx != i) && idx < 4)
4160          pshufhw = false;
4161        if ((idx != i) && idx > 3)
4162          pshuflw = false;
4163      }
4164      V1 = NewV;
4165      V2Used = false;
4166      BestLoQuad = 0;
4167      BestHiQuad = 1;
4168    }
4169
4170    // If we've eliminated the use of V2, and the new mask is a pshuflw or
4171    // pshufhw, that's as cheap as it gets.  Return the new shuffle.
4172    if ((pshufhw && InOrder[0]) || (pshuflw && InOrder[1])) {
4173      return DAG.getVectorShuffle(MVT::v8i16, dl, NewV,
4174                                  DAG.getUNDEF(MVT::v8i16), &MaskVals[0]);
4175    }
4176  }
4177
4178  // If we have SSSE3, and all words of the result are from 1 input vector,
4179  // case 2 is generated, otherwise case 3 is generated.  If no SSSE3
4180  // is present, fall back to case 4.
4181  if (TLI.getSubtarget()->hasSSSE3()) {
4182    SmallVector<SDValue,16> pshufbMask;
4183
4184    // If we have elements from both input vectors, set the high bit of the
4185    // shuffle mask element to zero out elements that come from V2 in the V1
4186    // mask, and elements that come from V1 in the V2 mask, so that the two
4187    // results can be OR'd together.
4188    bool TwoInputs = V1Used && V2Used;
4189    for (unsigned i = 0; i != 8; ++i) {
4190      int EltIdx = MaskVals[i] * 2;
4191      if (TwoInputs && (EltIdx >= 16)) {
4192        pshufbMask.push_back(DAG.getConstant(0x80, MVT::i8));
4193        pshufbMask.push_back(DAG.getConstant(0x80, MVT::i8));
4194        continue;
4195      }
4196      pshufbMask.push_back(DAG.getConstant(EltIdx,   MVT::i8));
4197      pshufbMask.push_back(DAG.getConstant(EltIdx+1, MVT::i8));
4198    }
4199    V1 = DAG.getNode(ISD::BIT_CONVERT, dl, MVT::v16i8, V1);
4200    V1 = DAG.getNode(X86ISD::PSHUFB, dl, MVT::v16i8, V1,
4201                     DAG.getNode(ISD::BUILD_VECTOR, dl,
4202                                 MVT::v16i8, &pshufbMask[0], 16));
4203    if (!TwoInputs)
4204      return DAG.getNode(ISD::BIT_CONVERT, dl, MVT::v8i16, V1);
4205
4206    // Calculate the shuffle mask for the second input, shuffle it, and
4207    // OR it with the first shuffled input.
4208    pshufbMask.clear();
4209    for (unsigned i = 0; i != 8; ++i) {
4210      int EltIdx = MaskVals[i] * 2;
4211      if (EltIdx < 16) {
4212        pshufbMask.push_back(DAG.getConstant(0x80, MVT::i8));
4213        pshufbMask.push_back(DAG.getConstant(0x80, MVT::i8));
4214        continue;
4215      }
4216      pshufbMask.push_back(DAG.getConstant(EltIdx - 16, MVT::i8));
4217      pshufbMask.push_back(DAG.getConstant(EltIdx - 15, MVT::i8));
4218    }
4219    V2 = DAG.getNode(ISD::BIT_CONVERT, dl, MVT::v16i8, V2);
4220    V2 = DAG.getNode(X86ISD::PSHUFB, dl, MVT::v16i8, V2,
4221                     DAG.getNode(ISD::BUILD_VECTOR, dl,
4222                                 MVT::v16i8, &pshufbMask[0], 16));
4223    V1 = DAG.getNode(ISD::OR, dl, MVT::v16i8, V1, V2);
4224    return DAG.getNode(ISD::BIT_CONVERT, dl, MVT::v8i16, V1);
4225  }
4226
4227  // If BestLoQuad >= 0, generate a pshuflw to put the low elements in order,
4228  // and update MaskVals with new element order.
4229  BitVector InOrder(8);
4230  if (BestLoQuad >= 0) {
4231    SmallVector<int, 8> MaskV;
4232    for (int i = 0; i != 4; ++i) {
4233      int idx = MaskVals[i];
4234      if (idx < 0) {
4235        MaskV.push_back(-1);
4236        InOrder.set(i);
4237      } else if ((idx / 4) == BestLoQuad) {
4238        MaskV.push_back(idx & 3);
4239        InOrder.set(i);
4240      } else {
4241        MaskV.push_back(-1);
4242      }
4243    }
4244    for (unsigned i = 4; i != 8; ++i)
4245      MaskV.push_back(i);
4246    NewV = DAG.getVectorShuffle(MVT::v8i16, dl, NewV, DAG.getUNDEF(MVT::v8i16),
4247                                &MaskV[0]);
4248  }
4249
4250  // If BestHi >= 0, generate a pshufhw to put the high elements in order,
4251  // and update MaskVals with the new element order.
4252  if (BestHiQuad >= 0) {
4253    SmallVector<int, 8> MaskV;
4254    for (unsigned i = 0; i != 4; ++i)
4255      MaskV.push_back(i);
4256    for (unsigned i = 4; i != 8; ++i) {
4257      int idx = MaskVals[i];
4258      if (idx < 0) {
4259        MaskV.push_back(-1);
4260        InOrder.set(i);
4261      } else if ((idx / 4) == BestHiQuad) {
4262        MaskV.push_back((idx & 3) + 4);
4263        InOrder.set(i);
4264      } else {
4265        MaskV.push_back(-1);
4266      }
4267    }
4268    NewV = DAG.getVectorShuffle(MVT::v8i16, dl, NewV, DAG.getUNDEF(MVT::v8i16),
4269                                &MaskV[0]);
4270  }
4271
4272  // In case BestHi & BestLo were both -1, which means each quadword has a word
4273  // from each of the four input quadwords, calculate the InOrder bitvector now
4274  // before falling through to the insert/extract cleanup.
4275  if (BestLoQuad == -1 && BestHiQuad == -1) {
4276    NewV = V1;
4277    for (int i = 0; i != 8; ++i)
4278      if (MaskVals[i] < 0 || MaskVals[i] == i)
4279        InOrder.set(i);
4280  }
4281
4282  // The other elements are put in the right place using pextrw and pinsrw.
4283  for (unsigned i = 0; i != 8; ++i) {
4284    if (InOrder[i])
4285      continue;
4286    int EltIdx = MaskVals[i];
4287    if (EltIdx < 0)
4288      continue;
4289    SDValue ExtOp = (EltIdx < 8)
4290    ? DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i16, V1,
4291                  DAG.getIntPtrConstant(EltIdx))
4292    : DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i16, V2,
4293                  DAG.getIntPtrConstant(EltIdx - 8));
4294    NewV = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, MVT::v8i16, NewV, ExtOp,
4295                       DAG.getIntPtrConstant(i));
4296  }
4297  return NewV;
4298}
4299
4300// v16i8 shuffles - Prefer shuffles in the following order:
4301// 1. [ssse3] 1 x pshufb
4302// 2. [ssse3] 2 x pshufb + 1 x por
4303// 3. [all]   v8i16 shuffle + N x pextrw + rotate + pinsrw
4304static
4305SDValue LowerVECTOR_SHUFFLEv16i8(ShuffleVectorSDNode *SVOp,
4306                                 SelectionDAG &DAG,
4307                                 const X86TargetLowering &TLI) {
4308  SDValue V1 = SVOp->getOperand(0);
4309  SDValue V2 = SVOp->getOperand(1);
4310  DebugLoc dl = SVOp->getDebugLoc();
4311  SmallVector<int, 16> MaskVals;
4312  SVOp->getMask(MaskVals);
4313
4314  // If we have SSSE3, case 1 is generated when all result bytes come from
4315  // one of  the inputs.  Otherwise, case 2 is generated.  If no SSSE3 is
4316  // present, fall back to case 3.
4317  // FIXME: kill V2Only once shuffles are canonizalized by getNode.
4318  bool V1Only = true;
4319  bool V2Only = true;
4320  for (unsigned i = 0; i < 16; ++i) {
4321    int EltIdx = MaskVals[i];
4322    if (EltIdx < 0)
4323      continue;
4324    if (EltIdx < 16)
4325      V2Only = false;
4326    else
4327      V1Only = false;
4328  }
4329
4330  // If SSSE3, use 1 pshufb instruction per vector with elements in the result.
4331  if (TLI.getSubtarget()->hasSSSE3()) {
4332    SmallVector<SDValue,16> pshufbMask;
4333
4334    // If all result elements are from one input vector, then only translate
4335    // undef mask values to 0x80 (zero out result) in the pshufb mask.
4336    //
4337    // Otherwise, we have elements from both input vectors, and must zero out
4338    // elements that come from V2 in the first mask, and V1 in the second mask
4339    // so that we can OR them together.
4340    bool TwoInputs = !(V1Only || V2Only);
4341    for (unsigned i = 0; i != 16; ++i) {
4342      int EltIdx = MaskVals[i];
4343      if (EltIdx < 0 || (TwoInputs && EltIdx >= 16)) {
4344        pshufbMask.push_back(DAG.getConstant(0x80, MVT::i8));
4345        continue;
4346      }
4347      pshufbMask.push_back(DAG.getConstant(EltIdx, MVT::i8));
4348    }
4349    // If all the elements are from V2, assign it to V1 and return after
4350    // building the first pshufb.
4351    if (V2Only)
4352      V1 = V2;
4353    V1 = DAG.getNode(X86ISD::PSHUFB, dl, MVT::v16i8, V1,
4354                     DAG.getNode(ISD::BUILD_VECTOR, dl,
4355                                 MVT::v16i8, &pshufbMask[0], 16));
4356    if (!TwoInputs)
4357      return V1;
4358
4359    // Calculate the shuffle mask for the second input, shuffle it, and
4360    // OR it with the first shuffled input.
4361    pshufbMask.clear();
4362    for (unsigned i = 0; i != 16; ++i) {
4363      int EltIdx = MaskVals[i];
4364      if (EltIdx < 16) {
4365        pshufbMask.push_back(DAG.getConstant(0x80, MVT::i8));
4366        continue;
4367      }
4368      pshufbMask.push_back(DAG.getConstant(EltIdx - 16, MVT::i8));
4369    }
4370    V2 = DAG.getNode(X86ISD::PSHUFB, dl, MVT::v16i8, V2,
4371                     DAG.getNode(ISD::BUILD_VECTOR, dl,
4372                                 MVT::v16i8, &pshufbMask[0], 16));
4373    return DAG.getNode(ISD::OR, dl, MVT::v16i8, V1, V2);
4374  }
4375
4376  // No SSSE3 - Calculate in place words and then fix all out of place words
4377  // With 0-16 extracts & inserts.  Worst case is 16 bytes out of order from
4378  // the 16 different words that comprise the two doublequadword input vectors.
4379  V1 = DAG.getNode(ISD::BIT_CONVERT, dl, MVT::v8i16, V1);
4380  V2 = DAG.getNode(ISD::BIT_CONVERT, dl, MVT::v8i16, V2);
4381  SDValue NewV = V2Only ? V2 : V1;
4382  for (int i = 0; i != 8; ++i) {
4383    int Elt0 = MaskVals[i*2];
4384    int Elt1 = MaskVals[i*2+1];
4385
4386    // This word of the result is all undef, skip it.
4387    if (Elt0 < 0 && Elt1 < 0)
4388      continue;
4389
4390    // This word of the result is already in the correct place, skip it.
4391    if (V1Only && (Elt0 == i*2) && (Elt1 == i*2+1))
4392      continue;
4393    if (V2Only && (Elt0 == i*2+16) && (Elt1 == i*2+17))
4394      continue;
4395
4396    SDValue Elt0Src = Elt0 < 16 ? V1 : V2;
4397    SDValue Elt1Src = Elt1 < 16 ? V1 : V2;
4398    SDValue InsElt;
4399
4400    // If Elt0 and Elt1 are defined, are consecutive, and can be load
4401    // using a single extract together, load it and store it.
4402    if ((Elt0 >= 0) && ((Elt0 + 1) == Elt1) && ((Elt0 & 1) == 0)) {
4403      InsElt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i16, Elt1Src,
4404                           DAG.getIntPtrConstant(Elt1 / 2));
4405      NewV = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, MVT::v8i16, NewV, InsElt,
4406                        DAG.getIntPtrConstant(i));
4407      continue;
4408    }
4409
4410    // If Elt1 is defined, extract it from the appropriate source.  If the
4411    // source byte is not also odd, shift the extracted word left 8 bits
4412    // otherwise clear the bottom 8 bits if we need to do an or.
4413    if (Elt1 >= 0) {
4414      InsElt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i16, Elt1Src,
4415                           DAG.getIntPtrConstant(Elt1 / 2));
4416      if ((Elt1 & 1) == 0)
4417        InsElt = DAG.getNode(ISD::SHL, dl, MVT::i16, InsElt,
4418                             DAG.getConstant(8, TLI.getShiftAmountTy()));
4419      else if (Elt0 >= 0)
4420        InsElt = DAG.getNode(ISD::AND, dl, MVT::i16, InsElt,
4421                             DAG.getConstant(0xFF00, MVT::i16));
4422    }
4423    // If Elt0 is defined, extract it from the appropriate source.  If the
4424    // source byte is not also even, shift the extracted word right 8 bits. If
4425    // Elt1 was also defined, OR the extracted values together before
4426    // inserting them in the result.
4427    if (Elt0 >= 0) {
4428      SDValue InsElt0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i16,
4429                                    Elt0Src, DAG.getIntPtrConstant(Elt0 / 2));
4430      if ((Elt0 & 1) != 0)
4431        InsElt0 = DAG.getNode(ISD::SRL, dl, MVT::i16, InsElt0,
4432                              DAG.getConstant(8, TLI.getShiftAmountTy()));
4433      else if (Elt1 >= 0)
4434        InsElt0 = DAG.getNode(ISD::AND, dl, MVT::i16, InsElt0,
4435                             DAG.getConstant(0x00FF, MVT::i16));
4436      InsElt = Elt1 >= 0 ? DAG.getNode(ISD::OR, dl, MVT::i16, InsElt, InsElt0)
4437                         : InsElt0;
4438    }
4439    NewV = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, MVT::v8i16, NewV, InsElt,
4440                       DAG.getIntPtrConstant(i));
4441  }
4442  return DAG.getNode(ISD::BIT_CONVERT, dl, MVT::v16i8, NewV);
4443}
4444
4445/// RewriteAsNarrowerShuffle - Try rewriting v8i16 and v16i8 shuffles as 4 wide
4446/// ones, or rewriting v4i32 / v2f32 as 2 wide ones if possible. This can be
4447/// done when every pair / quad of shuffle mask elements point to elements in
4448/// the right sequence. e.g.
4449/// vector_shuffle <>, <>, < 3, 4, | 10, 11, | 0, 1, | 14, 15>
4450static
4451SDValue RewriteAsNarrowerShuffle(ShuffleVectorSDNode *SVOp,
4452                                 SelectionDAG &DAG,
4453                                 const TargetLowering &TLI, DebugLoc dl) {
4454  EVT VT = SVOp->getValueType(0);
4455  SDValue V1 = SVOp->getOperand(0);
4456  SDValue V2 = SVOp->getOperand(1);
4457  unsigned NumElems = VT.getVectorNumElements();
4458  unsigned NewWidth = (NumElems == 4) ? 2 : 4;
4459  EVT MaskVT = MVT::getIntVectorWithNumElements(NewWidth);
4460  EVT MaskEltVT = MaskVT.getVectorElementType();
4461  EVT NewVT = MaskVT;
4462  switch (VT.getSimpleVT().SimpleTy) {
4463  default: assert(false && "Unexpected!");
4464  case MVT::v4f32: NewVT = MVT::v2f64; break;
4465  case MVT::v4i32: NewVT = MVT::v2i64; break;
4466  case MVT::v8i16: NewVT = MVT::v4i32; break;
4467  case MVT::v16i8: NewVT = MVT::v4i32; break;
4468  }
4469
4470  if (NewWidth == 2) {
4471    if (VT.isInteger())
4472      NewVT = MVT::v2i64;
4473    else
4474      NewVT = MVT::v2f64;
4475  }
4476  int Scale = NumElems / NewWidth;
4477  SmallVector<int, 8> MaskVec;
4478  for (unsigned i = 0; i < NumElems; i += Scale) {
4479    int StartIdx = -1;
4480    for (int j = 0; j < Scale; ++j) {
4481      int EltIdx = SVOp->getMaskElt(i+j);
4482      if (EltIdx < 0)
4483        continue;
4484      if (StartIdx == -1)
4485        StartIdx = EltIdx - (EltIdx % Scale);
4486      if (EltIdx != StartIdx + j)
4487        return SDValue();
4488    }
4489    if (StartIdx == -1)
4490      MaskVec.push_back(-1);
4491    else
4492      MaskVec.push_back(StartIdx / Scale);
4493  }
4494
4495  V1 = DAG.getNode(ISD::BIT_CONVERT, dl, NewVT, V1);
4496  V2 = DAG.getNode(ISD::BIT_CONVERT, dl, NewVT, V2);
4497  return DAG.getVectorShuffle(NewVT, dl, V1, V2, &MaskVec[0]);
4498}
4499
4500/// getVZextMovL - Return a zero-extending vector move low node.
4501///
4502static SDValue getVZextMovL(EVT VT, EVT OpVT,
4503                            SDValue SrcOp, SelectionDAG &DAG,
4504                            const X86Subtarget *Subtarget, DebugLoc dl) {
4505  if (VT == MVT::v2f64 || VT == MVT::v4f32) {
4506    LoadSDNode *LD = NULL;
4507    if (!isScalarLoadToVector(SrcOp.getNode(), &LD))
4508      LD = dyn_cast<LoadSDNode>(SrcOp);
4509    if (!LD) {
4510      // movssrr and movsdrr do not clear top bits. Try to use movd, movq
4511      // instead.
4512      MVT ExtVT = (OpVT == MVT::v2f64) ? MVT::i64 : MVT::i32;
4513      if ((ExtVT.SimpleTy != MVT::i64 || Subtarget->is64Bit()) &&
4514          SrcOp.getOpcode() == ISD::SCALAR_TO_VECTOR &&
4515          SrcOp.getOperand(0).getOpcode() == ISD::BIT_CONVERT &&
4516          SrcOp.getOperand(0).getOperand(0).getValueType() == ExtVT) {
4517        // PR2108
4518        OpVT = (OpVT == MVT::v2f64) ? MVT::v2i64 : MVT::v4i32;
4519        return DAG.getNode(ISD::BIT_CONVERT, dl, VT,
4520                           DAG.getNode(X86ISD::VZEXT_MOVL, dl, OpVT,
4521                                       DAG.getNode(ISD::SCALAR_TO_VECTOR, dl,
4522                                                   OpVT,
4523                                                   SrcOp.getOperand(0)
4524                                                          .getOperand(0))));
4525      }
4526    }
4527  }
4528
4529  return DAG.getNode(ISD::BIT_CONVERT, dl, VT,
4530                     DAG.getNode(X86ISD::VZEXT_MOVL, dl, OpVT,
4531                                 DAG.getNode(ISD::BIT_CONVERT, dl,
4532                                             OpVT, SrcOp)));
4533}
4534
4535/// LowerVECTOR_SHUFFLE_4wide - Handle all 4 wide cases with a number of
4536/// shuffles.
4537static SDValue
4538LowerVECTOR_SHUFFLE_4wide(ShuffleVectorSDNode *SVOp, SelectionDAG &DAG) {
4539  SDValue V1 = SVOp->getOperand(0);
4540  SDValue V2 = SVOp->getOperand(1);
4541  DebugLoc dl = SVOp->getDebugLoc();
4542  EVT VT = SVOp->getValueType(0);
4543
4544  SmallVector<std::pair<int, int>, 8> Locs;
4545  Locs.resize(4);
4546  SmallVector<int, 8> Mask1(4U, -1);
4547  SmallVector<int, 8> PermMask;
4548  SVOp->getMask(PermMask);
4549
4550  unsigned NumHi = 0;
4551  unsigned NumLo = 0;
4552  for (unsigned i = 0; i != 4; ++i) {
4553    int Idx = PermMask[i];
4554    if (Idx < 0) {
4555      Locs[i] = std::make_pair(-1, -1);
4556    } else {
4557      assert(Idx < 8 && "Invalid VECTOR_SHUFFLE index!");
4558      if (Idx < 4) {
4559        Locs[i] = std::make_pair(0, NumLo);
4560        Mask1[NumLo] = Idx;
4561        NumLo++;
4562      } else {
4563        Locs[i] = std::make_pair(1, NumHi);
4564        if (2+NumHi < 4)
4565          Mask1[2+NumHi] = Idx;
4566        NumHi++;
4567      }
4568    }
4569  }
4570
4571  if (NumLo <= 2 && NumHi <= 2) {
4572    // If no more than two elements come from either vector. This can be
4573    // implemented with two shuffles. First shuffle gather the elements.
4574    // The second shuffle, which takes the first shuffle as both of its
4575    // vector operands, put the elements into the right order.
4576    V1 = DAG.getVectorShuffle(VT, dl, V1, V2, &Mask1[0]);
4577
4578    SmallVector<int, 8> Mask2(4U, -1);
4579
4580    for (unsigned i = 0; i != 4; ++i) {
4581      if (Locs[i].first == -1)
4582        continue;
4583      else {
4584        unsigned Idx = (i < 2) ? 0 : 4;
4585        Idx += Locs[i].first * 2 + Locs[i].second;
4586        Mask2[i] = Idx;
4587      }
4588    }
4589
4590    return DAG.getVectorShuffle(VT, dl, V1, V1, &Mask2[0]);
4591  } else if (NumLo == 3 || NumHi == 3) {
4592    // Otherwise, we must have three elements from one vector, call it X, and
4593    // one element from the other, call it Y.  First, use a shufps to build an
4594    // intermediate vector with the one element from Y and the element from X
4595    // that will be in the same half in the final destination (the indexes don't
4596    // matter). Then, use a shufps to build the final vector, taking the half
4597    // containing the element from Y from the intermediate, and the other half
4598    // from X.
4599    if (NumHi == 3) {
4600      // Normalize it so the 3 elements come from V1.
4601      CommuteVectorShuffleMask(PermMask, VT);
4602      std::swap(V1, V2);
4603    }
4604
4605    // Find the element from V2.
4606    unsigned HiIndex;
4607    for (HiIndex = 0; HiIndex < 3; ++HiIndex) {
4608      int Val = PermMask[HiIndex];
4609      if (Val < 0)
4610        continue;
4611      if (Val >= 4)
4612        break;
4613    }
4614
4615    Mask1[0] = PermMask[HiIndex];
4616    Mask1[1] = -1;
4617    Mask1[2] = PermMask[HiIndex^1];
4618    Mask1[3] = -1;
4619    V2 = DAG.getVectorShuffle(VT, dl, V1, V2, &Mask1[0]);
4620
4621    if (HiIndex >= 2) {
4622      Mask1[0] = PermMask[0];
4623      Mask1[1] = PermMask[1];
4624      Mask1[2] = HiIndex & 1 ? 6 : 4;
4625      Mask1[3] = HiIndex & 1 ? 4 : 6;
4626      return DAG.getVectorShuffle(VT, dl, V1, V2, &Mask1[0]);
4627    } else {
4628      Mask1[0] = HiIndex & 1 ? 2 : 0;
4629      Mask1[1] = HiIndex & 1 ? 0 : 2;
4630      Mask1[2] = PermMask[2];
4631      Mask1[3] = PermMask[3];
4632      if (Mask1[2] >= 0)
4633        Mask1[2] += 4;
4634      if (Mask1[3] >= 0)
4635        Mask1[3] += 4;
4636      return DAG.getVectorShuffle(VT, dl, V2, V1, &Mask1[0]);
4637    }
4638  }
4639
4640  // Break it into (shuffle shuffle_hi, shuffle_lo).
4641  Locs.clear();
4642  SmallVector<int,8> LoMask(4U, -1);
4643  SmallVector<int,8> HiMask(4U, -1);
4644
4645  SmallVector<int,8> *MaskPtr = &LoMask;
4646  unsigned MaskIdx = 0;
4647  unsigned LoIdx = 0;
4648  unsigned HiIdx = 2;
4649  for (unsigned i = 0; i != 4; ++i) {
4650    if (i == 2) {
4651      MaskPtr = &HiMask;
4652      MaskIdx = 1;
4653      LoIdx = 0;
4654      HiIdx = 2;
4655    }
4656    int Idx = PermMask[i];
4657    if (Idx < 0) {
4658      Locs[i] = std::make_pair(-1, -1);
4659    } else if (Idx < 4) {
4660      Locs[i] = std::make_pair(MaskIdx, LoIdx);
4661      (*MaskPtr)[LoIdx] = Idx;
4662      LoIdx++;
4663    } else {
4664      Locs[i] = std::make_pair(MaskIdx, HiIdx);
4665      (*MaskPtr)[HiIdx] = Idx;
4666      HiIdx++;
4667    }
4668  }
4669
4670  SDValue LoShuffle = DAG.getVectorShuffle(VT, dl, V1, V2, &LoMask[0]);
4671  SDValue HiShuffle = DAG.getVectorShuffle(VT, dl, V1, V2, &HiMask[0]);
4672  SmallVector<int, 8> MaskOps;
4673  for (unsigned i = 0; i != 4; ++i) {
4674    if (Locs[i].first == -1) {
4675      MaskOps.push_back(-1);
4676    } else {
4677      unsigned Idx = Locs[i].first * 4 + Locs[i].second;
4678      MaskOps.push_back(Idx);
4679    }
4680  }
4681  return DAG.getVectorShuffle(VT, dl, LoShuffle, HiShuffle, &MaskOps[0]);
4682}
4683
4684SDValue
4685X86TargetLowering::LowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG) const {
4686  ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(Op);
4687  SDValue V1 = Op.getOperand(0);
4688  SDValue V2 = Op.getOperand(1);
4689  EVT VT = Op.getValueType();
4690  DebugLoc dl = Op.getDebugLoc();
4691  unsigned NumElems = VT.getVectorNumElements();
4692  bool isMMX = VT.getSizeInBits() == 64;
4693  bool V1IsUndef = V1.getOpcode() == ISD::UNDEF;
4694  bool V2IsUndef = V2.getOpcode() == ISD::UNDEF;
4695  bool V1IsSplat = false;
4696  bool V2IsSplat = false;
4697
4698  if (isZeroShuffle(SVOp))
4699    return getZeroVector(VT, Subtarget->hasSSE2(), DAG, dl);
4700
4701  // Promote splats to v4f32.
4702  if (SVOp->isSplat()) {
4703    if (isMMX || NumElems < 4)
4704      return Op;
4705    return PromoteSplat(SVOp, DAG, Subtarget->hasSSE2());
4706  }
4707
4708  // If the shuffle can be profitably rewritten as a narrower shuffle, then
4709  // do it!
4710  if (VT == MVT::v8i16 || VT == MVT::v16i8) {
4711    SDValue NewOp = RewriteAsNarrowerShuffle(SVOp, DAG, *this, dl);
4712    if (NewOp.getNode())
4713      return DAG.getNode(ISD::BIT_CONVERT, dl, VT,
4714                         LowerVECTOR_SHUFFLE(NewOp, DAG));
4715  } else if ((VT == MVT::v4i32 || (VT == MVT::v4f32 && Subtarget->hasSSE2()))) {
4716    // FIXME: Figure out a cleaner way to do this.
4717    // Try to make use of movq to zero out the top part.
4718    if (ISD::isBuildVectorAllZeros(V2.getNode())) {
4719      SDValue NewOp = RewriteAsNarrowerShuffle(SVOp, DAG, *this, dl);
4720      if (NewOp.getNode()) {
4721        if (isCommutedMOVL(cast<ShuffleVectorSDNode>(NewOp), true, false))
4722          return getVZextMovL(VT, NewOp.getValueType(), NewOp.getOperand(0),
4723                              DAG, Subtarget, dl);
4724      }
4725    } else if (ISD::isBuildVectorAllZeros(V1.getNode())) {
4726      SDValue NewOp = RewriteAsNarrowerShuffle(SVOp, DAG, *this, dl);
4727      if (NewOp.getNode() && X86::isMOVLMask(cast<ShuffleVectorSDNode>(NewOp)))
4728        return getVZextMovL(VT, NewOp.getValueType(), NewOp.getOperand(1),
4729                            DAG, Subtarget, dl);
4730    }
4731  }
4732
4733  if (X86::isPSHUFDMask(SVOp))
4734    return Op;
4735
4736  // Check if this can be converted into a logical shift.
4737  bool isLeft = false;
4738  unsigned ShAmt = 0;
4739  SDValue ShVal;
4740  bool isShift = getSubtarget()->hasSSE2() &&
4741    isVectorShift(SVOp, DAG, isLeft, ShVal, ShAmt);
4742  if (isShift && ShVal.hasOneUse()) {
4743    // If the shifted value has multiple uses, it may be cheaper to use
4744    // v_set0 + movlhps or movhlps, etc.
4745    EVT EltVT = VT.getVectorElementType();
4746    ShAmt *= EltVT.getSizeInBits();
4747    return getVShift(isLeft, VT, ShVal, ShAmt, DAG, *this, dl);
4748  }
4749
4750  if (X86::isMOVLMask(SVOp)) {
4751    if (V1IsUndef)
4752      return V2;
4753    if (ISD::isBuildVectorAllZeros(V1.getNode()))
4754      return getVZextMovL(VT, VT, V2, DAG, Subtarget, dl);
4755    if (!isMMX)
4756      return Op;
4757  }
4758
4759  // FIXME: fold these into legal mask.
4760  if (!isMMX && (X86::isMOVSHDUPMask(SVOp) ||
4761                 X86::isMOVSLDUPMask(SVOp) ||
4762                 X86::isMOVHLPSMask(SVOp) ||
4763                 X86::isMOVLHPSMask(SVOp) ||
4764                 X86::isMOVLPMask(SVOp)))
4765    return Op;
4766
4767  if (ShouldXformToMOVHLPS(SVOp) ||
4768      ShouldXformToMOVLP(V1.getNode(), V2.getNode(), SVOp))
4769    return CommuteVectorShuffle(SVOp, DAG);
4770
4771  if (isShift) {
4772    // No better options. Use a vshl / vsrl.
4773    EVT EltVT = VT.getVectorElementType();
4774    ShAmt *= EltVT.getSizeInBits();
4775    return getVShift(isLeft, VT, ShVal, ShAmt, DAG, *this, dl);
4776  }
4777
4778  bool Commuted = false;
4779  // FIXME: This should also accept a bitcast of a splat?  Be careful, not
4780  // 1,1,1,1 -> v8i16 though.
4781  V1IsSplat = isSplatVector(V1.getNode());
4782  V2IsSplat = isSplatVector(V2.getNode());
4783
4784  // Canonicalize the splat or undef, if present, to be on the RHS.
4785  if ((V1IsSplat || V1IsUndef) && !(V2IsSplat || V2IsUndef)) {
4786    Op = CommuteVectorShuffle(SVOp, DAG);
4787    SVOp = cast<ShuffleVectorSDNode>(Op);
4788    V1 = SVOp->getOperand(0);
4789    V2 = SVOp->getOperand(1);
4790    std::swap(V1IsSplat, V2IsSplat);
4791    std::swap(V1IsUndef, V2IsUndef);
4792    Commuted = true;
4793  }
4794
4795  if (isCommutedMOVL(SVOp, V2IsSplat, V2IsUndef)) {
4796    // Shuffling low element of v1 into undef, just return v1.
4797    if (V2IsUndef)
4798      return V1;
4799    // If V2 is a splat, the mask may be malformed such as <4,3,3,3>, which
4800    // the instruction selector will not match, so get a canonical MOVL with
4801    // swapped operands to undo the commute.
4802    return getMOVL(DAG, dl, VT, V2, V1);
4803  }
4804
4805  if (X86::isUNPCKL_v_undef_Mask(SVOp) ||
4806      X86::isUNPCKH_v_undef_Mask(SVOp) ||
4807      X86::isUNPCKLMask(SVOp) ||
4808      X86::isUNPCKHMask(SVOp))
4809    return Op;
4810
4811  if (V2IsSplat) {
4812    // Normalize mask so all entries that point to V2 points to its first
4813    // element then try to match unpck{h|l} again. If match, return a
4814    // new vector_shuffle with the corrected mask.
4815    SDValue NewMask = NormalizeMask(SVOp, DAG);
4816    ShuffleVectorSDNode *NSVOp = cast<ShuffleVectorSDNode>(NewMask);
4817    if (NSVOp != SVOp) {
4818      if (X86::isUNPCKLMask(NSVOp, true)) {
4819        return NewMask;
4820      } else if (X86::isUNPCKHMask(NSVOp, true)) {
4821        return NewMask;
4822      }
4823    }
4824  }
4825
4826  if (Commuted) {
4827    // Commute is back and try unpck* again.
4828    // FIXME: this seems wrong.
4829    SDValue NewOp = CommuteVectorShuffle(SVOp, DAG);
4830    ShuffleVectorSDNode *NewSVOp = cast<ShuffleVectorSDNode>(NewOp);
4831    if (X86::isUNPCKL_v_undef_Mask(NewSVOp) ||
4832        X86::isUNPCKH_v_undef_Mask(NewSVOp) ||
4833        X86::isUNPCKLMask(NewSVOp) ||
4834        X86::isUNPCKHMask(NewSVOp))
4835      return NewOp;
4836  }
4837
4838  // FIXME: for mmx, bitcast v2i32 to v4i16 for shuffle.
4839
4840  // Normalize the node to match x86 shuffle ops if needed
4841  if (!isMMX && V2.getOpcode() != ISD::UNDEF && isCommutedSHUFP(SVOp))
4842    return CommuteVectorShuffle(SVOp, DAG);
4843
4844  // Check for legal shuffle and return?
4845  SmallVector<int, 16> PermMask;
4846  SVOp->getMask(PermMask);
4847  if (isShuffleMaskLegal(PermMask, VT))
4848    return Op;
4849
4850  // Handle v8i16 specifically since SSE can do byte extraction and insertion.
4851  if (VT == MVT::v8i16) {
4852    SDValue NewOp = LowerVECTOR_SHUFFLEv8i16(SVOp, DAG, *this);
4853    if (NewOp.getNode())
4854      return NewOp;
4855  }
4856
4857  if (VT == MVT::v16i8) {
4858    SDValue NewOp = LowerVECTOR_SHUFFLEv16i8(SVOp, DAG, *this);
4859    if (NewOp.getNode())
4860      return NewOp;
4861  }
4862
4863  // Handle all 4 wide cases with a number of shuffles except for MMX.
4864  if (NumElems == 4 && !isMMX)
4865    return LowerVECTOR_SHUFFLE_4wide(SVOp, DAG);
4866
4867  return SDValue();
4868}
4869
4870SDValue
4871X86TargetLowering::LowerEXTRACT_VECTOR_ELT_SSE4(SDValue Op,
4872                                                SelectionDAG &DAG) const {
4873  EVT VT = Op.getValueType();
4874  DebugLoc dl = Op.getDebugLoc();
4875  if (VT.getSizeInBits() == 8) {
4876    SDValue Extract = DAG.getNode(X86ISD::PEXTRB, dl, MVT::i32,
4877                                    Op.getOperand(0), Op.getOperand(1));
4878    SDValue Assert  = DAG.getNode(ISD::AssertZext, dl, MVT::i32, Extract,
4879                                    DAG.getValueType(VT));
4880    return DAG.getNode(ISD::TRUNCATE, dl, VT, Assert);
4881  } else if (VT.getSizeInBits() == 16) {
4882    unsigned Idx = cast<ConstantSDNode>(Op.getOperand(1))->getZExtValue();
4883    // If Idx is 0, it's cheaper to do a move instead of a pextrw.
4884    if (Idx == 0)
4885      return DAG.getNode(ISD::TRUNCATE, dl, MVT::i16,
4886                         DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i32,
4887                                     DAG.getNode(ISD::BIT_CONVERT, dl,
4888                                                 MVT::v4i32,
4889                                                 Op.getOperand(0)),
4890                                     Op.getOperand(1)));
4891    SDValue Extract = DAG.getNode(X86ISD::PEXTRW, dl, MVT::i32,
4892                                    Op.getOperand(0), Op.getOperand(1));
4893    SDValue Assert  = DAG.getNode(ISD::AssertZext, dl, MVT::i32, Extract,
4894                                    DAG.getValueType(VT));
4895    return DAG.getNode(ISD::TRUNCATE, dl, VT, Assert);
4896  } else if (VT == MVT::f32) {
4897    // EXTRACTPS outputs to a GPR32 register which will require a movd to copy
4898    // the result back to FR32 register. It's only worth matching if the
4899    // result has a single use which is a store or a bitcast to i32.  And in
4900    // the case of a store, it's not worth it if the index is a constant 0,
4901    // because a MOVSSmr can be used instead, which is smaller and faster.
4902    if (!Op.hasOneUse())
4903      return SDValue();
4904    SDNode *User = *Op.getNode()->use_begin();
4905    if ((User->getOpcode() != ISD::STORE ||
4906         (isa<ConstantSDNode>(Op.getOperand(1)) &&
4907          cast<ConstantSDNode>(Op.getOperand(1))->isNullValue())) &&
4908        (User->getOpcode() != ISD::BIT_CONVERT ||
4909         User->getValueType(0) != MVT::i32))
4910      return SDValue();
4911    SDValue Extract = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i32,
4912                                  DAG.getNode(ISD::BIT_CONVERT, dl, MVT::v4i32,
4913                                              Op.getOperand(0)),
4914                                              Op.getOperand(1));
4915    return DAG.getNode(ISD::BIT_CONVERT, dl, MVT::f32, Extract);
4916  } else if (VT == MVT::i32) {
4917    // ExtractPS works with constant index.
4918    if (isa<ConstantSDNode>(Op.getOperand(1)))
4919      return Op;
4920  }
4921  return SDValue();
4922}
4923
4924
4925SDValue
4926X86TargetLowering::LowerEXTRACT_VECTOR_ELT(SDValue Op,
4927                                           SelectionDAG &DAG) const {
4928  if (!isa<ConstantSDNode>(Op.getOperand(1)))
4929    return SDValue();
4930
4931  if (Subtarget->hasSSE41()) {
4932    SDValue Res = LowerEXTRACT_VECTOR_ELT_SSE4(Op, DAG);
4933    if (Res.getNode())
4934      return Res;
4935  }
4936
4937  EVT VT = Op.getValueType();
4938  DebugLoc dl = Op.getDebugLoc();
4939  // TODO: handle v16i8.
4940  if (VT.getSizeInBits() == 16) {
4941    SDValue Vec = Op.getOperand(0);
4942    unsigned Idx = cast<ConstantSDNode>(Op.getOperand(1))->getZExtValue();
4943    if (Idx == 0)
4944      return DAG.getNode(ISD::TRUNCATE, dl, MVT::i16,
4945                         DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i32,
4946                                     DAG.getNode(ISD::BIT_CONVERT, dl,
4947                                                 MVT::v4i32, Vec),
4948                                     Op.getOperand(1)));
4949    // Transform it so it match pextrw which produces a 32-bit result.
4950    EVT EltVT = MVT::i32;
4951    SDValue Extract = DAG.getNode(X86ISD::PEXTRW, dl, EltVT,
4952                                    Op.getOperand(0), Op.getOperand(1));
4953    SDValue Assert  = DAG.getNode(ISD::AssertZext, dl, EltVT, Extract,
4954                                    DAG.getValueType(VT));
4955    return DAG.getNode(ISD::TRUNCATE, dl, VT, Assert);
4956  } else if (VT.getSizeInBits() == 32) {
4957    unsigned Idx = cast<ConstantSDNode>(Op.getOperand(1))->getZExtValue();
4958    if (Idx == 0)
4959      return Op;
4960
4961    // SHUFPS the element to the lowest double word, then movss.
4962    int Mask[4] = { Idx, -1, -1, -1 };
4963    EVT VVT = Op.getOperand(0).getValueType();
4964    SDValue Vec = DAG.getVectorShuffle(VVT, dl, Op.getOperand(0),
4965                                       DAG.getUNDEF(VVT), Mask);
4966    return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, Vec,
4967                       DAG.getIntPtrConstant(0));
4968  } else if (VT.getSizeInBits() == 64) {
4969    // FIXME: .td only matches this for <2 x f64>, not <2 x i64> on 32b
4970    // FIXME: seems like this should be unnecessary if mov{h,l}pd were taught
4971    //        to match extract_elt for f64.
4972    unsigned Idx = cast<ConstantSDNode>(Op.getOperand(1))->getZExtValue();
4973    if (Idx == 0)
4974      return Op;
4975
4976    // UNPCKHPD the element to the lowest double word, then movsd.
4977    // Note if the lower 64 bits of the result of the UNPCKHPD is then stored
4978    // to a f64mem, the whole operation is folded into a single MOVHPDmr.
4979    int Mask[2] = { 1, -1 };
4980    EVT VVT = Op.getOperand(0).getValueType();
4981    SDValue Vec = DAG.getVectorShuffle(VVT, dl, Op.getOperand(0),
4982                                       DAG.getUNDEF(VVT), Mask);
4983    return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, Vec,
4984                       DAG.getIntPtrConstant(0));
4985  }
4986
4987  return SDValue();
4988}
4989
4990SDValue
4991X86TargetLowering::LowerINSERT_VECTOR_ELT_SSE4(SDValue Op,
4992                                               SelectionDAG &DAG) const {
4993  EVT VT = Op.getValueType();
4994  EVT EltVT = VT.getVectorElementType();
4995  DebugLoc dl = Op.getDebugLoc();
4996
4997  SDValue N0 = Op.getOperand(0);
4998  SDValue N1 = Op.getOperand(1);
4999  SDValue N2 = Op.getOperand(2);
5000
5001  if ((EltVT.getSizeInBits() == 8 || EltVT.getSizeInBits() == 16) &&
5002      isa<ConstantSDNode>(N2)) {
5003    unsigned Opc;
5004    if (VT == MVT::v8i16)
5005      Opc = X86ISD::PINSRW;
5006    else if (VT == MVT::v4i16)
5007      Opc = X86ISD::MMX_PINSRW;
5008    else if (VT == MVT::v16i8)
5009      Opc = X86ISD::PINSRB;
5010    else
5011      Opc = X86ISD::PINSRB;
5012
5013    // Transform it so it match pinsr{b,w} which expects a GR32 as its second
5014    // argument.
5015    if (N1.getValueType() != MVT::i32)
5016      N1 = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i32, N1);
5017    if (N2.getValueType() != MVT::i32)
5018      N2 = DAG.getIntPtrConstant(cast<ConstantSDNode>(N2)->getZExtValue());
5019    return DAG.getNode(Opc, dl, VT, N0, N1, N2);
5020  } else if (EltVT == MVT::f32 && isa<ConstantSDNode>(N2)) {
5021    // Bits [7:6] of the constant are the source select.  This will always be
5022    //  zero here.  The DAG Combiner may combine an extract_elt index into these
5023    //  bits.  For example (insert (extract, 3), 2) could be matched by putting
5024    //  the '3' into bits [7:6] of X86ISD::INSERTPS.
5025    // Bits [5:4] of the constant are the destination select.  This is the
5026    //  value of the incoming immediate.
5027    // Bits [3:0] of the constant are the zero mask.  The DAG Combiner may
5028    //   combine either bitwise AND or insert of float 0.0 to set these bits.
5029    N2 = DAG.getIntPtrConstant(cast<ConstantSDNode>(N2)->getZExtValue() << 4);
5030    // Create this as a scalar to vector..
5031    N1 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v4f32, N1);
5032    return DAG.getNode(X86ISD::INSERTPS, dl, VT, N0, N1, N2);
5033  } else if (EltVT == MVT::i32 && isa<ConstantSDNode>(N2)) {
5034    // PINSR* works with constant index.
5035    return Op;
5036  }
5037  return SDValue();
5038}
5039
5040SDValue
5041X86TargetLowering::LowerINSERT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG) const {
5042  EVT VT = Op.getValueType();
5043  EVT EltVT = VT.getVectorElementType();
5044
5045  if (Subtarget->hasSSE41())
5046    return LowerINSERT_VECTOR_ELT_SSE4(Op, DAG);
5047
5048  if (EltVT == MVT::i8)
5049    return SDValue();
5050
5051  DebugLoc dl = Op.getDebugLoc();
5052  SDValue N0 = Op.getOperand(0);
5053  SDValue N1 = Op.getOperand(1);
5054  SDValue N2 = Op.getOperand(2);
5055
5056  if (EltVT.getSizeInBits() == 16 && isa<ConstantSDNode>(N2)) {
5057    // Transform it so it match pinsrw which expects a 16-bit value in a GR32
5058    // as its second argument.
5059    if (N1.getValueType() != MVT::i32)
5060      N1 = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i32, N1);
5061    if (N2.getValueType() != MVT::i32)
5062      N2 = DAG.getIntPtrConstant(cast<ConstantSDNode>(N2)->getZExtValue());
5063    return DAG.getNode(VT == MVT::v8i16 ? X86ISD::PINSRW : X86ISD::MMX_PINSRW,
5064                       dl, VT, N0, N1, N2);
5065  }
5066  return SDValue();
5067}
5068
5069SDValue
5070X86TargetLowering::LowerSCALAR_TO_VECTOR(SDValue Op, SelectionDAG &DAG) const {
5071  DebugLoc dl = Op.getDebugLoc();
5072  if (Op.getValueType() == MVT::v2f32)
5073    return DAG.getNode(ISD::BIT_CONVERT, dl, MVT::v2f32,
5074                       DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v2i32,
5075                                   DAG.getNode(ISD::BIT_CONVERT, dl, MVT::i32,
5076                                               Op.getOperand(0))));
5077
5078  if (Op.getValueType() == MVT::v1i64 && Op.getOperand(0).getValueType() == MVT::i64)
5079    return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v1i64, Op.getOperand(0));
5080
5081  SDValue AnyExt = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i32, Op.getOperand(0));
5082  EVT VT = MVT::v2i32;
5083  switch (Op.getValueType().getSimpleVT().SimpleTy) {
5084  default: break;
5085  case MVT::v16i8:
5086  case MVT::v8i16:
5087    VT = MVT::v4i32;
5088    break;
5089  }
5090  return DAG.getNode(ISD::BIT_CONVERT, dl, Op.getValueType(),
5091                     DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, AnyExt));
5092}
5093
5094// ConstantPool, JumpTable, GlobalAddress, and ExternalSymbol are lowered as
5095// their target countpart wrapped in the X86ISD::Wrapper node. Suppose N is
5096// one of the above mentioned nodes. It has to be wrapped because otherwise
5097// Select(N) returns N. So the raw TargetGlobalAddress nodes, etc. can only
5098// be used to form addressing mode. These wrapped nodes will be selected
5099// into MOV32ri.
5100SDValue
5101X86TargetLowering::LowerConstantPool(SDValue Op, SelectionDAG &DAG) const {
5102  ConstantPoolSDNode *CP = cast<ConstantPoolSDNode>(Op);
5103
5104  // In PIC mode (unless we're in RIPRel PIC mode) we add an offset to the
5105  // global base reg.
5106  unsigned char OpFlag = 0;
5107  unsigned WrapperKind = X86ISD::Wrapper;
5108  CodeModel::Model M = getTargetMachine().getCodeModel();
5109
5110  if (Subtarget->isPICStyleRIPRel() &&
5111      (M == CodeModel::Small || M == CodeModel::Kernel))
5112    WrapperKind = X86ISD::WrapperRIP;
5113  else if (Subtarget->isPICStyleGOT())
5114    OpFlag = X86II::MO_GOTOFF;
5115  else if (Subtarget->isPICStyleStubPIC())
5116    OpFlag = X86II::MO_PIC_BASE_OFFSET;
5117
5118  SDValue Result = DAG.getTargetConstantPool(CP->getConstVal(), getPointerTy(),
5119                                             CP->getAlignment(),
5120                                             CP->getOffset(), OpFlag);
5121  DebugLoc DL = CP->getDebugLoc();
5122  Result = DAG.getNode(WrapperKind, DL, getPointerTy(), Result);
5123  // With PIC, the address is actually $g + Offset.
5124  if (OpFlag) {
5125    Result = DAG.getNode(ISD::ADD, DL, getPointerTy(),
5126                         DAG.getNode(X86ISD::GlobalBaseReg,
5127                                     DebugLoc(), getPointerTy()),
5128                         Result);
5129  }
5130
5131  return Result;
5132}
5133
5134SDValue X86TargetLowering::LowerJumpTable(SDValue Op, SelectionDAG &DAG) const {
5135  JumpTableSDNode *JT = cast<JumpTableSDNode>(Op);
5136
5137  // In PIC mode (unless we're in RIPRel PIC mode) we add an offset to the
5138  // global base reg.
5139  unsigned char OpFlag = 0;
5140  unsigned WrapperKind = X86ISD::Wrapper;
5141  CodeModel::Model M = getTargetMachine().getCodeModel();
5142
5143  if (Subtarget->isPICStyleRIPRel() &&
5144      (M == CodeModel::Small || M == CodeModel::Kernel))
5145    WrapperKind = X86ISD::WrapperRIP;
5146  else if (Subtarget->isPICStyleGOT())
5147    OpFlag = X86II::MO_GOTOFF;
5148  else if (Subtarget->isPICStyleStubPIC())
5149    OpFlag = X86II::MO_PIC_BASE_OFFSET;
5150
5151  SDValue Result = DAG.getTargetJumpTable(JT->getIndex(), getPointerTy(),
5152                                          OpFlag);
5153  DebugLoc DL = JT->getDebugLoc();
5154  Result = DAG.getNode(WrapperKind, DL, getPointerTy(), Result);
5155
5156  // With PIC, the address is actually $g + Offset.
5157  if (OpFlag) {
5158    Result = DAG.getNode(ISD::ADD, DL, getPointerTy(),
5159                         DAG.getNode(X86ISD::GlobalBaseReg,
5160                                     DebugLoc(), getPointerTy()),
5161                         Result);
5162  }
5163
5164  return Result;
5165}
5166
5167SDValue
5168X86TargetLowering::LowerExternalSymbol(SDValue Op, SelectionDAG &DAG) const {
5169  const char *Sym = cast<ExternalSymbolSDNode>(Op)->getSymbol();
5170
5171  // In PIC mode (unless we're in RIPRel PIC mode) we add an offset to the
5172  // global base reg.
5173  unsigned char OpFlag = 0;
5174  unsigned WrapperKind = X86ISD::Wrapper;
5175  CodeModel::Model M = getTargetMachine().getCodeModel();
5176
5177  if (Subtarget->isPICStyleRIPRel() &&
5178      (M == CodeModel::Small || M == CodeModel::Kernel))
5179    WrapperKind = X86ISD::WrapperRIP;
5180  else if (Subtarget->isPICStyleGOT())
5181    OpFlag = X86II::MO_GOTOFF;
5182  else if (Subtarget->isPICStyleStubPIC())
5183    OpFlag = X86II::MO_PIC_BASE_OFFSET;
5184
5185  SDValue Result = DAG.getTargetExternalSymbol(Sym, getPointerTy(), OpFlag);
5186
5187  DebugLoc DL = Op.getDebugLoc();
5188  Result = DAG.getNode(WrapperKind, DL, getPointerTy(), Result);
5189
5190
5191  // With PIC, the address is actually $g + Offset.
5192  if (getTargetMachine().getRelocationModel() == Reloc::PIC_ &&
5193      !Subtarget->is64Bit()) {
5194    Result = DAG.getNode(ISD::ADD, DL, getPointerTy(),
5195                         DAG.getNode(X86ISD::GlobalBaseReg,
5196                                     DebugLoc(), getPointerTy()),
5197                         Result);
5198  }
5199
5200  return Result;
5201}
5202
5203SDValue
5204X86TargetLowering::LowerBlockAddress(SDValue Op, SelectionDAG &DAG) const {
5205  // Create the TargetBlockAddressAddress node.
5206  unsigned char OpFlags =
5207    Subtarget->ClassifyBlockAddressReference();
5208  CodeModel::Model M = getTargetMachine().getCodeModel();
5209  const BlockAddress *BA = cast<BlockAddressSDNode>(Op)->getBlockAddress();
5210  DebugLoc dl = Op.getDebugLoc();
5211  SDValue Result = DAG.getBlockAddress(BA, getPointerTy(),
5212                                       /*isTarget=*/true, OpFlags);
5213
5214  if (Subtarget->isPICStyleRIPRel() &&
5215      (M == CodeModel::Small || M == CodeModel::Kernel))
5216    Result = DAG.getNode(X86ISD::WrapperRIP, dl, getPointerTy(), Result);
5217  else
5218    Result = DAG.getNode(X86ISD::Wrapper, dl, getPointerTy(), Result);
5219
5220  // With PIC, the address is actually $g + Offset.
5221  if (isGlobalRelativeToPICBase(OpFlags)) {
5222    Result = DAG.getNode(ISD::ADD, dl, getPointerTy(),
5223                         DAG.getNode(X86ISD::GlobalBaseReg, dl, getPointerTy()),
5224                         Result);
5225  }
5226
5227  return Result;
5228}
5229
5230SDValue
5231X86TargetLowering::LowerGlobalAddress(const GlobalValue *GV, DebugLoc dl,
5232                                      int64_t Offset,
5233                                      SelectionDAG &DAG) const {
5234  // Create the TargetGlobalAddress node, folding in the constant
5235  // offset if it is legal.
5236  unsigned char OpFlags =
5237    Subtarget->ClassifyGlobalReference(GV, getTargetMachine());
5238  CodeModel::Model M = getTargetMachine().getCodeModel();
5239  SDValue Result;
5240  if (OpFlags == X86II::MO_NO_FLAG &&
5241      X86::isOffsetSuitableForCodeModel(Offset, M)) {
5242    // A direct static reference to a global.
5243    Result = DAG.getTargetGlobalAddress(GV, getPointerTy(), Offset);
5244    Offset = 0;
5245  } else {
5246    Result = DAG.getTargetGlobalAddress(GV, getPointerTy(), 0, OpFlags);
5247  }
5248
5249  if (Subtarget->isPICStyleRIPRel() &&
5250      (M == CodeModel::Small || M == CodeModel::Kernel))
5251    Result = DAG.getNode(X86ISD::WrapperRIP, dl, getPointerTy(), Result);
5252  else
5253    Result = DAG.getNode(X86ISD::Wrapper, dl, getPointerTy(), Result);
5254
5255  // With PIC, the address is actually $g + Offset.
5256  if (isGlobalRelativeToPICBase(OpFlags)) {
5257    Result = DAG.getNode(ISD::ADD, dl, getPointerTy(),
5258                         DAG.getNode(X86ISD::GlobalBaseReg, dl, getPointerTy()),
5259                         Result);
5260  }
5261
5262  // For globals that require a load from a stub to get the address, emit the
5263  // load.
5264  if (isGlobalStubReference(OpFlags))
5265    Result = DAG.getLoad(getPointerTy(), dl, DAG.getEntryNode(), Result,
5266                         PseudoSourceValue::getGOT(), 0, false, false, 0);
5267
5268  // If there was a non-zero offset that we didn't fold, create an explicit
5269  // addition for it.
5270  if (Offset != 0)
5271    Result = DAG.getNode(ISD::ADD, dl, getPointerTy(), Result,
5272                         DAG.getConstant(Offset, getPointerTy()));
5273
5274  return Result;
5275}
5276
5277SDValue
5278X86TargetLowering::LowerGlobalAddress(SDValue Op, SelectionDAG &DAG) const {
5279  const GlobalValue *GV = cast<GlobalAddressSDNode>(Op)->getGlobal();
5280  int64_t Offset = cast<GlobalAddressSDNode>(Op)->getOffset();
5281  return LowerGlobalAddress(GV, Op.getDebugLoc(), Offset, DAG);
5282}
5283
5284static SDValue
5285GetTLSADDR(SelectionDAG &DAG, SDValue Chain, GlobalAddressSDNode *GA,
5286           SDValue *InFlag, const EVT PtrVT, unsigned ReturnReg,
5287           unsigned char OperandFlags) {
5288  MachineFrameInfo *MFI = DAG.getMachineFunction().getFrameInfo();
5289  SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Flag);
5290  DebugLoc dl = GA->getDebugLoc();
5291  SDValue TGA = DAG.getTargetGlobalAddress(GA->getGlobal(),
5292                                           GA->getValueType(0),
5293                                           GA->getOffset(),
5294                                           OperandFlags);
5295  if (InFlag) {
5296    SDValue Ops[] = { Chain,  TGA, *InFlag };
5297    Chain = DAG.getNode(X86ISD::TLSADDR, dl, NodeTys, Ops, 3);
5298  } else {
5299    SDValue Ops[]  = { Chain, TGA };
5300    Chain = DAG.getNode(X86ISD::TLSADDR, dl, NodeTys, Ops, 2);
5301  }
5302
5303  // TLSADDR will be codegen'ed as call. Inform MFI that function has calls.
5304  MFI->setAdjustsStack(true);
5305
5306  SDValue Flag = Chain.getValue(1);
5307  return DAG.getCopyFromReg(Chain, dl, ReturnReg, PtrVT, Flag);
5308}
5309
5310// Lower ISD::GlobalTLSAddress using the "general dynamic" model, 32 bit
5311static SDValue
5312LowerToTLSGeneralDynamicModel32(GlobalAddressSDNode *GA, SelectionDAG &DAG,
5313                                const EVT PtrVT) {
5314  SDValue InFlag;
5315  DebugLoc dl = GA->getDebugLoc();  // ? function entry point might be better
5316  SDValue Chain = DAG.getCopyToReg(DAG.getEntryNode(), dl, X86::EBX,
5317                                     DAG.getNode(X86ISD::GlobalBaseReg,
5318                                                 DebugLoc(), PtrVT), InFlag);
5319  InFlag = Chain.getValue(1);
5320
5321  return GetTLSADDR(DAG, Chain, GA, &InFlag, PtrVT, X86::EAX, X86II::MO_TLSGD);
5322}
5323
5324// Lower ISD::GlobalTLSAddress using the "general dynamic" model, 64 bit
5325static SDValue
5326LowerToTLSGeneralDynamicModel64(GlobalAddressSDNode *GA, SelectionDAG &DAG,
5327                                const EVT PtrVT) {
5328  return GetTLSADDR(DAG, DAG.getEntryNode(), GA, NULL, PtrVT,
5329                    X86::RAX, X86II::MO_TLSGD);
5330}
5331
5332// Lower ISD::GlobalTLSAddress using the "initial exec" (for no-pic) or
5333// "local exec" model.
5334static SDValue LowerToTLSExecModel(GlobalAddressSDNode *GA, SelectionDAG &DAG,
5335                                   const EVT PtrVT, TLSModel::Model model,
5336                                   bool is64Bit) {
5337  DebugLoc dl = GA->getDebugLoc();
5338  // Get the Thread Pointer
5339  SDValue Base = DAG.getNode(X86ISD::SegmentBaseAddress,
5340                             DebugLoc(), PtrVT,
5341                             DAG.getRegister(is64Bit? X86::FS : X86::GS,
5342                                             MVT::i32));
5343
5344  SDValue ThreadPointer = DAG.getLoad(PtrVT, dl, DAG.getEntryNode(), Base,
5345                                      NULL, 0, false, false, 0);
5346
5347  unsigned char OperandFlags = 0;
5348  // Most TLS accesses are not RIP relative, even on x86-64.  One exception is
5349  // initialexec.
5350  unsigned WrapperKind = X86ISD::Wrapper;
5351  if (model == TLSModel::LocalExec) {
5352    OperandFlags = is64Bit ? X86II::MO_TPOFF : X86II::MO_NTPOFF;
5353  } else if (is64Bit) {
5354    assert(model == TLSModel::InitialExec);
5355    OperandFlags = X86II::MO_GOTTPOFF;
5356    WrapperKind = X86ISD::WrapperRIP;
5357  } else {
5358    assert(model == TLSModel::InitialExec);
5359    OperandFlags = X86II::MO_INDNTPOFF;
5360  }
5361
5362  // emit "addl x@ntpoff,%eax" (local exec) or "addl x@indntpoff,%eax" (initial
5363  // exec)
5364  SDValue TGA = DAG.getTargetGlobalAddress(GA->getGlobal(), GA->getValueType(0),
5365                                           GA->getOffset(), OperandFlags);
5366  SDValue Offset = DAG.getNode(WrapperKind, dl, PtrVT, TGA);
5367
5368  if (model == TLSModel::InitialExec)
5369    Offset = DAG.getLoad(PtrVT, dl, DAG.getEntryNode(), Offset,
5370                         PseudoSourceValue::getGOT(), 0, false, false, 0);
5371
5372  // The address of the thread local variable is the add of the thread
5373  // pointer with the offset of the variable.
5374  return DAG.getNode(ISD::ADD, dl, PtrVT, ThreadPointer, Offset);
5375}
5376
5377SDValue
5378X86TargetLowering::LowerGlobalTLSAddress(SDValue Op, SelectionDAG &DAG) const {
5379
5380  GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(Op);
5381  const GlobalValue *GV = GA->getGlobal();
5382
5383  if (Subtarget->isTargetELF()) {
5384    // TODO: implement the "local dynamic" model
5385    // TODO: implement the "initial exec"model for pic executables
5386
5387    // If GV is an alias then use the aliasee for determining
5388    // thread-localness.
5389    if (const GlobalAlias *GA = dyn_cast<GlobalAlias>(GV))
5390      GV = GA->resolveAliasedGlobal(false);
5391
5392    TLSModel::Model model
5393      = getTLSModel(GV, getTargetMachine().getRelocationModel());
5394
5395    switch (model) {
5396      case TLSModel::GeneralDynamic:
5397      case TLSModel::LocalDynamic: // not implemented
5398        if (Subtarget->is64Bit())
5399          return LowerToTLSGeneralDynamicModel64(GA, DAG, getPointerTy());
5400        return LowerToTLSGeneralDynamicModel32(GA, DAG, getPointerTy());
5401
5402      case TLSModel::InitialExec:
5403      case TLSModel::LocalExec:
5404        return LowerToTLSExecModel(GA, DAG, getPointerTy(), model,
5405                                   Subtarget->is64Bit());
5406    }
5407  } else if (Subtarget->isTargetDarwin()) {
5408    // Darwin only has one model of TLS.  Lower to that.
5409    unsigned char OpFlag = 0;
5410    unsigned WrapperKind = Subtarget->isPICStyleRIPRel() ?
5411                           X86ISD::WrapperRIP : X86ISD::Wrapper;
5412
5413    // In PIC mode (unless we're in RIPRel PIC mode) we add an offset to the
5414    // global base reg.
5415    bool PIC32 = (getTargetMachine().getRelocationModel() == Reloc::PIC_) &&
5416                  !Subtarget->is64Bit();
5417    if (PIC32)
5418      OpFlag = X86II::MO_TLVP_PIC_BASE;
5419    else
5420      OpFlag = X86II::MO_TLVP;
5421
5422    SDValue Result = DAG.getTargetGlobalAddress(GA->getGlobal(),
5423                                                getPointerTy(),
5424                                                GA->getOffset(), OpFlag);
5425
5426    DebugLoc DL = Op.getDebugLoc();
5427    SDValue Offset = DAG.getNode(WrapperKind, DL, getPointerTy(), Result);
5428
5429    // With PIC32, the address is actually $g + Offset.
5430    if (PIC32)
5431      Offset = DAG.getNode(ISD::ADD, DL, getPointerTy(),
5432                           DAG.getNode(X86ISD::GlobalBaseReg,
5433                                       DebugLoc(), getPointerTy()),
5434                           Offset);
5435
5436    // Lowering the machine isd will make sure everything is in the right
5437    // location.
5438    SDValue Args[] = { Offset };
5439    SDValue Chain = DAG.getNode(X86ISD::TLSCALL, DL, MVT::Other, Args, 1);
5440
5441    // TLSCALL will be codegen'ed as call. Inform MFI that function has calls.
5442    MachineFrameInfo *MFI = DAG.getMachineFunction().getFrameInfo();
5443    MFI->setAdjustsStack(true);
5444
5445    // And our return value (tls address) is in the standard call return value
5446    // location.
5447    unsigned Reg = Subtarget->is64Bit() ? X86::RAX : X86::EAX;
5448    return DAG.getCopyFromReg(Chain, DL, Reg, getPointerTy());
5449  }
5450
5451  assert(false &&
5452         "TLS not implemented for this target.");
5453
5454  llvm_unreachable("Unreachable");
5455  return SDValue();
5456}
5457
5458
5459/// LowerShift - Lower SRA_PARTS and friends, which return two i32 values and
5460/// take a 2 x i32 value to shift plus a shift amount.
5461SDValue X86TargetLowering::LowerShift(SDValue Op, SelectionDAG &DAG) const {
5462  assert(Op.getNumOperands() == 3 && "Not a double-shift!");
5463  EVT VT = Op.getValueType();
5464  unsigned VTBits = VT.getSizeInBits();
5465  DebugLoc dl = Op.getDebugLoc();
5466  bool isSRA = Op.getOpcode() == ISD::SRA_PARTS;
5467  SDValue ShOpLo = Op.getOperand(0);
5468  SDValue ShOpHi = Op.getOperand(1);
5469  SDValue ShAmt  = Op.getOperand(2);
5470  SDValue Tmp1 = isSRA ? DAG.getNode(ISD::SRA, dl, VT, ShOpHi,
5471                                     DAG.getConstant(VTBits - 1, MVT::i8))
5472                       : DAG.getConstant(0, VT);
5473
5474  SDValue Tmp2, Tmp3;
5475  if (Op.getOpcode() == ISD::SHL_PARTS) {
5476    Tmp2 = DAG.getNode(X86ISD::SHLD, dl, VT, ShOpHi, ShOpLo, ShAmt);
5477    Tmp3 = DAG.getNode(ISD::SHL, dl, VT, ShOpLo, ShAmt);
5478  } else {
5479    Tmp2 = DAG.getNode(X86ISD::SHRD, dl, VT, ShOpLo, ShOpHi, ShAmt);
5480    Tmp3 = DAG.getNode(isSRA ? ISD::SRA : ISD::SRL, dl, VT, ShOpHi, ShAmt);
5481  }
5482
5483  SDValue AndNode = DAG.getNode(ISD::AND, dl, MVT::i8, ShAmt,
5484                                DAG.getConstant(VTBits, MVT::i8));
5485  SDValue Cond = DAG.getNode(X86ISD::CMP, dl, MVT::i32,
5486                             AndNode, DAG.getConstant(0, MVT::i8));
5487
5488  SDValue Hi, Lo;
5489  SDValue CC = DAG.getConstant(X86::COND_NE, MVT::i8);
5490  SDValue Ops0[4] = { Tmp2, Tmp3, CC, Cond };
5491  SDValue Ops1[4] = { Tmp3, Tmp1, CC, Cond };
5492
5493  if (Op.getOpcode() == ISD::SHL_PARTS) {
5494    Hi = DAG.getNode(X86ISD::CMOV, dl, VT, Ops0, 4);
5495    Lo = DAG.getNode(X86ISD::CMOV, dl, VT, Ops1, 4);
5496  } else {
5497    Lo = DAG.getNode(X86ISD::CMOV, dl, VT, Ops0, 4);
5498    Hi = DAG.getNode(X86ISD::CMOV, dl, VT, Ops1, 4);
5499  }
5500
5501  SDValue Ops[2] = { Lo, Hi };
5502  return DAG.getMergeValues(Ops, 2, dl);
5503}
5504
5505SDValue X86TargetLowering::LowerSINT_TO_FP(SDValue Op,
5506                                           SelectionDAG &DAG) const {
5507  EVT SrcVT = Op.getOperand(0).getValueType();
5508
5509  if (SrcVT.isVector()) {
5510    if (SrcVT == MVT::v2i32 && Op.getValueType() == MVT::v2f64) {
5511      return Op;
5512    }
5513    return SDValue();
5514  }
5515
5516  assert(SrcVT.getSimpleVT() <= MVT::i64 && SrcVT.getSimpleVT() >= MVT::i16 &&
5517         "Unknown SINT_TO_FP to lower!");
5518
5519  // These are really Legal; return the operand so the caller accepts it as
5520  // Legal.
5521  if (SrcVT == MVT::i32 && isScalarFPTypeInSSEReg(Op.getValueType()))
5522    return Op;
5523  if (SrcVT == MVT::i64 && isScalarFPTypeInSSEReg(Op.getValueType()) &&
5524      Subtarget->is64Bit()) {
5525    return Op;
5526  }
5527
5528  DebugLoc dl = Op.getDebugLoc();
5529  unsigned Size = SrcVT.getSizeInBits()/8;
5530  MachineFunction &MF = DAG.getMachineFunction();
5531  int SSFI = MF.getFrameInfo()->CreateStackObject(Size, Size, false);
5532  SDValue StackSlot = DAG.getFrameIndex(SSFI, getPointerTy());
5533  SDValue Chain = DAG.getStore(DAG.getEntryNode(), dl, Op.getOperand(0),
5534                               StackSlot,
5535                               PseudoSourceValue::getFixedStack(SSFI), 0,
5536                               false, false, 0);
5537  return BuildFILD(Op, SrcVT, Chain, StackSlot, DAG);
5538}
5539
5540SDValue X86TargetLowering::BuildFILD(SDValue Op, EVT SrcVT, SDValue Chain,
5541                                     SDValue StackSlot,
5542                                     SelectionDAG &DAG) const {
5543  // Build the FILD
5544  DebugLoc dl = Op.getDebugLoc();
5545  SDVTList Tys;
5546  bool useSSE = isScalarFPTypeInSSEReg(Op.getValueType());
5547  if (useSSE)
5548    Tys = DAG.getVTList(MVT::f64, MVT::Other, MVT::Flag);
5549  else
5550    Tys = DAG.getVTList(Op.getValueType(), MVT::Other);
5551  SDValue Ops[] = { Chain, StackSlot, DAG.getValueType(SrcVT) };
5552  SDValue Result = DAG.getNode(useSSE ? X86ISD::FILD_FLAG : X86ISD::FILD, dl,
5553                               Tys, Ops, array_lengthof(Ops));
5554
5555  if (useSSE) {
5556    Chain = Result.getValue(1);
5557    SDValue InFlag = Result.getValue(2);
5558
5559    // FIXME: Currently the FST is flagged to the FILD_FLAG. This
5560    // shouldn't be necessary except that RFP cannot be live across
5561    // multiple blocks. When stackifier is fixed, they can be uncoupled.
5562    MachineFunction &MF = DAG.getMachineFunction();
5563    int SSFI = MF.getFrameInfo()->CreateStackObject(8, 8, false);
5564    SDValue StackSlot = DAG.getFrameIndex(SSFI, getPointerTy());
5565    Tys = DAG.getVTList(MVT::Other);
5566    SDValue Ops[] = {
5567      Chain, Result, StackSlot, DAG.getValueType(Op.getValueType()), InFlag
5568    };
5569    Chain = DAG.getNode(X86ISD::FST, dl, Tys, Ops, array_lengthof(Ops));
5570    Result = DAG.getLoad(Op.getValueType(), dl, Chain, StackSlot,
5571                         PseudoSourceValue::getFixedStack(SSFI), 0,
5572                         false, false, 0);
5573  }
5574
5575  return Result;
5576}
5577
5578// LowerUINT_TO_FP_i64 - 64-bit unsigned integer to double expansion.
5579SDValue X86TargetLowering::LowerUINT_TO_FP_i64(SDValue Op,
5580                                               SelectionDAG &DAG) const {
5581  // This algorithm is not obvious. Here it is in C code, more or less:
5582  /*
5583    double uint64_to_double( uint32_t hi, uint32_t lo ) {
5584      static const __m128i exp = { 0x4330000045300000ULL, 0 };
5585      static const __m128d bias = { 0x1.0p84, 0x1.0p52 };
5586
5587      // Copy ints to xmm registers.
5588      __m128i xh = _mm_cvtsi32_si128( hi );
5589      __m128i xl = _mm_cvtsi32_si128( lo );
5590
5591      // Combine into low half of a single xmm register.
5592      __m128i x = _mm_unpacklo_epi32( xh, xl );
5593      __m128d d;
5594      double sd;
5595
5596      // Merge in appropriate exponents to give the integer bits the right
5597      // magnitude.
5598      x = _mm_unpacklo_epi32( x, exp );
5599
5600      // Subtract away the biases to deal with the IEEE-754 double precision
5601      // implicit 1.
5602      d = _mm_sub_pd( (__m128d) x, bias );
5603
5604      // All conversions up to here are exact. The correctly rounded result is
5605      // calculated using the current rounding mode using the following
5606      // horizontal add.
5607      d = _mm_add_sd( d, _mm_unpackhi_pd( d, d ) );
5608      _mm_store_sd( &sd, d );   // Because we are returning doubles in XMM, this
5609                                // store doesn't really need to be here (except
5610                                // maybe to zero the other double)
5611      return sd;
5612    }
5613  */
5614
5615  DebugLoc dl = Op.getDebugLoc();
5616  LLVMContext *Context = DAG.getContext();
5617
5618  // Build some magic constants.
5619  std::vector<Constant*> CV0;
5620  CV0.push_back(ConstantInt::get(*Context, APInt(32, 0x45300000)));
5621  CV0.push_back(ConstantInt::get(*Context, APInt(32, 0x43300000)));
5622  CV0.push_back(ConstantInt::get(*Context, APInt(32, 0)));
5623  CV0.push_back(ConstantInt::get(*Context, APInt(32, 0)));
5624  Constant *C0 = ConstantVector::get(CV0);
5625  SDValue CPIdx0 = DAG.getConstantPool(C0, getPointerTy(), 16);
5626
5627  std::vector<Constant*> CV1;
5628  CV1.push_back(
5629    ConstantFP::get(*Context, APFloat(APInt(64, 0x4530000000000000ULL))));
5630  CV1.push_back(
5631    ConstantFP::get(*Context, APFloat(APInt(64, 0x4330000000000000ULL))));
5632  Constant *C1 = ConstantVector::get(CV1);
5633  SDValue CPIdx1 = DAG.getConstantPool(C1, getPointerTy(), 16);
5634
5635  SDValue XR1 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v4i32,
5636                            DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32,
5637                                        Op.getOperand(0),
5638                                        DAG.getIntPtrConstant(1)));
5639  SDValue XR2 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v4i32,
5640                            DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32,
5641                                        Op.getOperand(0),
5642                                        DAG.getIntPtrConstant(0)));
5643  SDValue Unpck1 = getUnpackl(DAG, dl, MVT::v4i32, XR1, XR2);
5644  SDValue CLod0 = DAG.getLoad(MVT::v4i32, dl, DAG.getEntryNode(), CPIdx0,
5645                              PseudoSourceValue::getConstantPool(), 0,
5646                              false, false, 16);
5647  SDValue Unpck2 = getUnpackl(DAG, dl, MVT::v4i32, Unpck1, CLod0);
5648  SDValue XR2F = DAG.getNode(ISD::BIT_CONVERT, dl, MVT::v2f64, Unpck2);
5649  SDValue CLod1 = DAG.getLoad(MVT::v2f64, dl, CLod0.getValue(1), CPIdx1,
5650                              PseudoSourceValue::getConstantPool(), 0,
5651                              false, false, 16);
5652  SDValue Sub = DAG.getNode(ISD::FSUB, dl, MVT::v2f64, XR2F, CLod1);
5653
5654  // Add the halves; easiest way is to swap them into another reg first.
5655  int ShufMask[2] = { 1, -1 };
5656  SDValue Shuf = DAG.getVectorShuffle(MVT::v2f64, dl, Sub,
5657                                      DAG.getUNDEF(MVT::v2f64), ShufMask);
5658  SDValue Add = DAG.getNode(ISD::FADD, dl, MVT::v2f64, Shuf, Sub);
5659  return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f64, Add,
5660                     DAG.getIntPtrConstant(0));
5661}
5662
5663// LowerUINT_TO_FP_i32 - 32-bit unsigned integer to float expansion.
5664SDValue X86TargetLowering::LowerUINT_TO_FP_i32(SDValue Op,
5665                                               SelectionDAG &DAG) const {
5666  DebugLoc dl = Op.getDebugLoc();
5667  // FP constant to bias correct the final result.
5668  SDValue Bias = DAG.getConstantFP(BitsToDouble(0x4330000000000000ULL),
5669                                   MVT::f64);
5670
5671  // Load the 32-bit value into an XMM register.
5672  SDValue Load = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v4i32,
5673                             DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32,
5674                                         Op.getOperand(0),
5675                                         DAG.getIntPtrConstant(0)));
5676
5677  Load = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f64,
5678                     DAG.getNode(ISD::BIT_CONVERT, dl, MVT::v2f64, Load),
5679                     DAG.getIntPtrConstant(0));
5680
5681  // Or the load with the bias.
5682  SDValue Or = DAG.getNode(ISD::OR, dl, MVT::v2i64,
5683                           DAG.getNode(ISD::BIT_CONVERT, dl, MVT::v2i64,
5684                                       DAG.getNode(ISD::SCALAR_TO_VECTOR, dl,
5685                                                   MVT::v2f64, Load)),
5686                           DAG.getNode(ISD::BIT_CONVERT, dl, MVT::v2i64,
5687                                       DAG.getNode(ISD::SCALAR_TO_VECTOR, dl,
5688                                                   MVT::v2f64, Bias)));
5689  Or = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f64,
5690                   DAG.getNode(ISD::BIT_CONVERT, dl, MVT::v2f64, Or),
5691                   DAG.getIntPtrConstant(0));
5692
5693  // Subtract the bias.
5694  SDValue Sub = DAG.getNode(ISD::FSUB, dl, MVT::f64, Or, Bias);
5695
5696  // Handle final rounding.
5697  EVT DestVT = Op.getValueType();
5698
5699  if (DestVT.bitsLT(MVT::f64)) {
5700    return DAG.getNode(ISD::FP_ROUND, dl, DestVT, Sub,
5701                       DAG.getIntPtrConstant(0));
5702  } else if (DestVT.bitsGT(MVT::f64)) {
5703    return DAG.getNode(ISD::FP_EXTEND, dl, DestVT, Sub);
5704  }
5705
5706  // Handle final rounding.
5707  return Sub;
5708}
5709
5710SDValue X86TargetLowering::LowerUINT_TO_FP(SDValue Op,
5711                                           SelectionDAG &DAG) const {
5712  SDValue N0 = Op.getOperand(0);
5713  DebugLoc dl = Op.getDebugLoc();
5714
5715  // Since UINT_TO_FP is legal (it's marked custom), dag combiner won't
5716  // optimize it to a SINT_TO_FP when the sign bit is known zero. Perform
5717  // the optimization here.
5718  if (DAG.SignBitIsZero(N0))
5719    return DAG.getNode(ISD::SINT_TO_FP, dl, Op.getValueType(), N0);
5720
5721  EVT SrcVT = N0.getValueType();
5722  EVT DstVT = Op.getValueType();
5723  if (SrcVT == MVT::i64 && DstVT == MVT::f64 && X86ScalarSSEf64)
5724    return LowerUINT_TO_FP_i64(Op, DAG);
5725  else if (SrcVT == MVT::i32 && X86ScalarSSEf64)
5726    return LowerUINT_TO_FP_i32(Op, DAG);
5727
5728  // Make a 64-bit buffer, and use it to build an FILD.
5729  SDValue StackSlot = DAG.CreateStackTemporary(MVT::i64);
5730  if (SrcVT == MVT::i32) {
5731    SDValue WordOff = DAG.getConstant(4, getPointerTy());
5732    SDValue OffsetSlot = DAG.getNode(ISD::ADD, dl,
5733                                     getPointerTy(), StackSlot, WordOff);
5734    SDValue Store1 = DAG.getStore(DAG.getEntryNode(), dl, Op.getOperand(0),
5735                                  StackSlot, NULL, 0, false, false, 0);
5736    SDValue Store2 = DAG.getStore(Store1, dl, DAG.getConstant(0, MVT::i32),
5737                                  OffsetSlot, NULL, 0, false, false, 0);
5738    SDValue Fild = BuildFILD(Op, MVT::i64, Store2, StackSlot, DAG);
5739    return Fild;
5740  }
5741
5742  assert(SrcVT == MVT::i64 && "Unexpected type in UINT_TO_FP");
5743  SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Op.getOperand(0),
5744                                StackSlot, NULL, 0, false, false, 0);
5745  // For i64 source, we need to add the appropriate power of 2 if the input
5746  // was negative.  This is the same as the optimization in
5747  // DAGTypeLegalizer::ExpandIntOp_UNIT_TO_FP, and for it to be safe here,
5748  // we must be careful to do the computation in x87 extended precision, not
5749  // in SSE. (The generic code can't know it's OK to do this, or how to.)
5750  SDVTList Tys = DAG.getVTList(MVT::f80, MVT::Other);
5751  SDValue Ops[] = { Store, StackSlot, DAG.getValueType(MVT::i64) };
5752  SDValue Fild = DAG.getNode(X86ISD::FILD, dl, Tys, Ops, 3);
5753
5754  APInt FF(32, 0x5F800000ULL);
5755
5756  // Check whether the sign bit is set.
5757  SDValue SignSet = DAG.getSetCC(dl, getSetCCResultType(MVT::i64),
5758                                 Op.getOperand(0), DAG.getConstant(0, MVT::i64),
5759                                 ISD::SETLT);
5760
5761  // Build a 64 bit pair (0, FF) in the constant pool, with FF in the lo bits.
5762  SDValue FudgePtr = DAG.getConstantPool(
5763                             ConstantInt::get(*DAG.getContext(), FF.zext(64)),
5764                                         getPointerTy());
5765
5766  // Get a pointer to FF if the sign bit was set, or to 0 otherwise.
5767  SDValue Zero = DAG.getIntPtrConstant(0);
5768  SDValue Four = DAG.getIntPtrConstant(4);
5769  SDValue Offset = DAG.getNode(ISD::SELECT, dl, Zero.getValueType(), SignSet,
5770                               Zero, Four);
5771  FudgePtr = DAG.getNode(ISD::ADD, dl, getPointerTy(), FudgePtr, Offset);
5772
5773  // Load the value out, extending it from f32 to f80.
5774  // FIXME: Avoid the extend by constructing the right constant pool?
5775  SDValue Fudge = DAG.getExtLoad(ISD::EXTLOAD, dl, MVT::f80, DAG.getEntryNode(),
5776                                 FudgePtr, PseudoSourceValue::getConstantPool(),
5777                                 0, MVT::f32, false, false, 4);
5778  // Extend everything to 80 bits to force it to be done on x87.
5779  SDValue Add = DAG.getNode(ISD::FADD, dl, MVT::f80, Fild, Fudge);
5780  return DAG.getNode(ISD::FP_ROUND, dl, DstVT, Add, DAG.getIntPtrConstant(0));
5781}
5782
5783std::pair<SDValue,SDValue> X86TargetLowering::
5784FP_TO_INTHelper(SDValue Op, SelectionDAG &DAG, bool IsSigned) const {
5785  DebugLoc dl = Op.getDebugLoc();
5786
5787  EVT DstTy = Op.getValueType();
5788
5789  if (!IsSigned) {
5790    assert(DstTy == MVT::i32 && "Unexpected FP_TO_UINT");
5791    DstTy = MVT::i64;
5792  }
5793
5794  assert(DstTy.getSimpleVT() <= MVT::i64 &&
5795         DstTy.getSimpleVT() >= MVT::i16 &&
5796         "Unknown FP_TO_SINT to lower!");
5797
5798  // These are really Legal.
5799  if (DstTy == MVT::i32 &&
5800      isScalarFPTypeInSSEReg(Op.getOperand(0).getValueType()))
5801    return std::make_pair(SDValue(), SDValue());
5802  if (Subtarget->is64Bit() &&
5803      DstTy == MVT::i64 &&
5804      isScalarFPTypeInSSEReg(Op.getOperand(0).getValueType()))
5805    return std::make_pair(SDValue(), SDValue());
5806
5807  // We lower FP->sint64 into FISTP64, followed by a load, all to a temporary
5808  // stack slot.
5809  MachineFunction &MF = DAG.getMachineFunction();
5810  unsigned MemSize = DstTy.getSizeInBits()/8;
5811  int SSFI = MF.getFrameInfo()->CreateStackObject(MemSize, MemSize, false);
5812  SDValue StackSlot = DAG.getFrameIndex(SSFI, getPointerTy());
5813
5814  unsigned Opc;
5815  switch (DstTy.getSimpleVT().SimpleTy) {
5816  default: llvm_unreachable("Invalid FP_TO_SINT to lower!");
5817  case MVT::i16: Opc = X86ISD::FP_TO_INT16_IN_MEM; break;
5818  case MVT::i32: Opc = X86ISD::FP_TO_INT32_IN_MEM; break;
5819  case MVT::i64: Opc = X86ISD::FP_TO_INT64_IN_MEM; break;
5820  }
5821
5822  SDValue Chain = DAG.getEntryNode();
5823  SDValue Value = Op.getOperand(0);
5824  if (isScalarFPTypeInSSEReg(Op.getOperand(0).getValueType())) {
5825    assert(DstTy == MVT::i64 && "Invalid FP_TO_SINT to lower!");
5826    Chain = DAG.getStore(Chain, dl, Value, StackSlot,
5827                         PseudoSourceValue::getFixedStack(SSFI), 0,
5828                         false, false, 0);
5829    SDVTList Tys = DAG.getVTList(Op.getOperand(0).getValueType(), MVT::Other);
5830    SDValue Ops[] = {
5831      Chain, StackSlot, DAG.getValueType(Op.getOperand(0).getValueType())
5832    };
5833    Value = DAG.getNode(X86ISD::FLD, dl, Tys, Ops, 3);
5834    Chain = Value.getValue(1);
5835    SSFI = MF.getFrameInfo()->CreateStackObject(MemSize, MemSize, false);
5836    StackSlot = DAG.getFrameIndex(SSFI, getPointerTy());
5837  }
5838
5839  // Build the FP_TO_INT*_IN_MEM
5840  SDValue Ops[] = { Chain, Value, StackSlot };
5841  SDValue FIST = DAG.getNode(Opc, dl, MVT::Other, Ops, 3);
5842
5843  return std::make_pair(FIST, StackSlot);
5844}
5845
5846SDValue X86TargetLowering::LowerFP_TO_SINT(SDValue Op,
5847                                           SelectionDAG &DAG) const {
5848  if (Op.getValueType().isVector()) {
5849    if (Op.getValueType() == MVT::v2i32 &&
5850        Op.getOperand(0).getValueType() == MVT::v2f64) {
5851      return Op;
5852    }
5853    return SDValue();
5854  }
5855
5856  std::pair<SDValue,SDValue> Vals = FP_TO_INTHelper(Op, DAG, true);
5857  SDValue FIST = Vals.first, StackSlot = Vals.second;
5858  // If FP_TO_INTHelper failed, the node is actually supposed to be Legal.
5859  if (FIST.getNode() == 0) return Op;
5860
5861  // Load the result.
5862  return DAG.getLoad(Op.getValueType(), Op.getDebugLoc(),
5863                     FIST, StackSlot, NULL, 0, false, false, 0);
5864}
5865
5866SDValue X86TargetLowering::LowerFP_TO_UINT(SDValue Op,
5867                                           SelectionDAG &DAG) const {
5868  std::pair<SDValue,SDValue> Vals = FP_TO_INTHelper(Op, DAG, false);
5869  SDValue FIST = Vals.first, StackSlot = Vals.second;
5870  assert(FIST.getNode() && "Unexpected failure");
5871
5872  // Load the result.
5873  return DAG.getLoad(Op.getValueType(), Op.getDebugLoc(),
5874                     FIST, StackSlot, NULL, 0, false, false, 0);
5875}
5876
5877SDValue X86TargetLowering::LowerFABS(SDValue Op,
5878                                     SelectionDAG &DAG) const {
5879  LLVMContext *Context = DAG.getContext();
5880  DebugLoc dl = Op.getDebugLoc();
5881  EVT VT = Op.getValueType();
5882  EVT EltVT = VT;
5883  if (VT.isVector())
5884    EltVT = VT.getVectorElementType();
5885  std::vector<Constant*> CV;
5886  if (EltVT == MVT::f64) {
5887    Constant *C = ConstantFP::get(*Context, APFloat(APInt(64, ~(1ULL << 63))));
5888    CV.push_back(C);
5889    CV.push_back(C);
5890  } else {
5891    Constant *C = ConstantFP::get(*Context, APFloat(APInt(32, ~(1U << 31))));
5892    CV.push_back(C);
5893    CV.push_back(C);
5894    CV.push_back(C);
5895    CV.push_back(C);
5896  }
5897  Constant *C = ConstantVector::get(CV);
5898  SDValue CPIdx = DAG.getConstantPool(C, getPointerTy(), 16);
5899  SDValue Mask = DAG.getLoad(VT, dl, DAG.getEntryNode(), CPIdx,
5900                             PseudoSourceValue::getConstantPool(), 0,
5901                             false, false, 16);
5902  return DAG.getNode(X86ISD::FAND, dl, VT, Op.getOperand(0), Mask);
5903}
5904
5905SDValue X86TargetLowering::LowerFNEG(SDValue Op, SelectionDAG &DAG) const {
5906  LLVMContext *Context = DAG.getContext();
5907  DebugLoc dl = Op.getDebugLoc();
5908  EVT VT = Op.getValueType();
5909  EVT EltVT = VT;
5910  if (VT.isVector())
5911    EltVT = VT.getVectorElementType();
5912  std::vector<Constant*> CV;
5913  if (EltVT == MVT::f64) {
5914    Constant *C = ConstantFP::get(*Context, APFloat(APInt(64, 1ULL << 63)));
5915    CV.push_back(C);
5916    CV.push_back(C);
5917  } else {
5918    Constant *C = ConstantFP::get(*Context, APFloat(APInt(32, 1U << 31)));
5919    CV.push_back(C);
5920    CV.push_back(C);
5921    CV.push_back(C);
5922    CV.push_back(C);
5923  }
5924  Constant *C = ConstantVector::get(CV);
5925  SDValue CPIdx = DAG.getConstantPool(C, getPointerTy(), 16);
5926  SDValue Mask = DAG.getLoad(VT, dl, DAG.getEntryNode(), CPIdx,
5927                             PseudoSourceValue::getConstantPool(), 0,
5928                             false, false, 16);
5929  if (VT.isVector()) {
5930    return DAG.getNode(ISD::BIT_CONVERT, dl, VT,
5931                       DAG.getNode(ISD::XOR, dl, MVT::v2i64,
5932                    DAG.getNode(ISD::BIT_CONVERT, dl, MVT::v2i64,
5933                                Op.getOperand(0)),
5934                    DAG.getNode(ISD::BIT_CONVERT, dl, MVT::v2i64, Mask)));
5935  } else {
5936    return DAG.getNode(X86ISD::FXOR, dl, VT, Op.getOperand(0), Mask);
5937  }
5938}
5939
5940SDValue X86TargetLowering::LowerFCOPYSIGN(SDValue Op, SelectionDAG &DAG) const {
5941  LLVMContext *Context = DAG.getContext();
5942  SDValue Op0 = Op.getOperand(0);
5943  SDValue Op1 = Op.getOperand(1);
5944  DebugLoc dl = Op.getDebugLoc();
5945  EVT VT = Op.getValueType();
5946  EVT SrcVT = Op1.getValueType();
5947
5948  // If second operand is smaller, extend it first.
5949  if (SrcVT.bitsLT(VT)) {
5950    Op1 = DAG.getNode(ISD::FP_EXTEND, dl, VT, Op1);
5951    SrcVT = VT;
5952  }
5953  // And if it is bigger, shrink it first.
5954  if (SrcVT.bitsGT(VT)) {
5955    Op1 = DAG.getNode(ISD::FP_ROUND, dl, VT, Op1, DAG.getIntPtrConstant(1));
5956    SrcVT = VT;
5957  }
5958
5959  // At this point the operands and the result should have the same
5960  // type, and that won't be f80 since that is not custom lowered.
5961
5962  // First get the sign bit of second operand.
5963  std::vector<Constant*> CV;
5964  if (SrcVT == MVT::f64) {
5965    CV.push_back(ConstantFP::get(*Context, APFloat(APInt(64, 1ULL << 63))));
5966    CV.push_back(ConstantFP::get(*Context, APFloat(APInt(64, 0))));
5967  } else {
5968    CV.push_back(ConstantFP::get(*Context, APFloat(APInt(32, 1U << 31))));
5969    CV.push_back(ConstantFP::get(*Context, APFloat(APInt(32, 0))));
5970    CV.push_back(ConstantFP::get(*Context, APFloat(APInt(32, 0))));
5971    CV.push_back(ConstantFP::get(*Context, APFloat(APInt(32, 0))));
5972  }
5973  Constant *C = ConstantVector::get(CV);
5974  SDValue CPIdx = DAG.getConstantPool(C, getPointerTy(), 16);
5975  SDValue Mask1 = DAG.getLoad(SrcVT, dl, DAG.getEntryNode(), CPIdx,
5976                              PseudoSourceValue::getConstantPool(), 0,
5977                              false, false, 16);
5978  SDValue SignBit = DAG.getNode(X86ISD::FAND, dl, SrcVT, Op1, Mask1);
5979
5980  // Shift sign bit right or left if the two operands have different types.
5981  if (SrcVT.bitsGT(VT)) {
5982    // Op0 is MVT::f32, Op1 is MVT::f64.
5983    SignBit = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v2f64, SignBit);
5984    SignBit = DAG.getNode(X86ISD::FSRL, dl, MVT::v2f64, SignBit,
5985                          DAG.getConstant(32, MVT::i32));
5986    SignBit = DAG.getNode(ISD::BIT_CONVERT, dl, MVT::v4f32, SignBit);
5987    SignBit = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f32, SignBit,
5988                          DAG.getIntPtrConstant(0));
5989  }
5990
5991  // Clear first operand sign bit.
5992  CV.clear();
5993  if (VT == MVT::f64) {
5994    CV.push_back(ConstantFP::get(*Context, APFloat(APInt(64, ~(1ULL << 63)))));
5995    CV.push_back(ConstantFP::get(*Context, APFloat(APInt(64, 0))));
5996  } else {
5997    CV.push_back(ConstantFP::get(*Context, APFloat(APInt(32, ~(1U << 31)))));
5998    CV.push_back(ConstantFP::get(*Context, APFloat(APInt(32, 0))));
5999    CV.push_back(ConstantFP::get(*Context, APFloat(APInt(32, 0))));
6000    CV.push_back(ConstantFP::get(*Context, APFloat(APInt(32, 0))));
6001  }
6002  C = ConstantVector::get(CV);
6003  CPIdx = DAG.getConstantPool(C, getPointerTy(), 16);
6004  SDValue Mask2 = DAG.getLoad(VT, dl, DAG.getEntryNode(), CPIdx,
6005                              PseudoSourceValue::getConstantPool(), 0,
6006                              false, false, 16);
6007  SDValue Val = DAG.getNode(X86ISD::FAND, dl, VT, Op0, Mask2);
6008
6009  // Or the value with the sign bit.
6010  return DAG.getNode(X86ISD::FOR, dl, VT, Val, SignBit);
6011}
6012
6013/// Emit nodes that will be selected as "test Op0,Op0", or something
6014/// equivalent.
6015SDValue X86TargetLowering::EmitTest(SDValue Op, unsigned X86CC,
6016                                    SelectionDAG &DAG) const {
6017  DebugLoc dl = Op.getDebugLoc();
6018
6019  // CF and OF aren't always set the way we want. Determine which
6020  // of these we need.
6021  bool NeedCF = false;
6022  bool NeedOF = false;
6023  switch (X86CC) {
6024  case X86::COND_A: case X86::COND_AE:
6025  case X86::COND_B: case X86::COND_BE:
6026    NeedCF = true;
6027    break;
6028  case X86::COND_G: case X86::COND_GE:
6029  case X86::COND_L: case X86::COND_LE:
6030  case X86::COND_O: case X86::COND_NO:
6031    NeedOF = true;
6032    break;
6033  default: break;
6034  }
6035
6036  // See if we can use the EFLAGS value from the operand instead of
6037  // doing a separate TEST. TEST always sets OF and CF to 0, so unless
6038  // we prove that the arithmetic won't overflow, we can't use OF or CF.
6039  if (Op.getResNo() == 0 && !NeedOF && !NeedCF) {
6040    unsigned Opcode = 0;
6041    unsigned NumOperands = 0;
6042    switch (Op.getNode()->getOpcode()) {
6043    case ISD::ADD:
6044      // Due to an isel shortcoming, be conservative if this add is
6045      // likely to be selected as part of a load-modify-store
6046      // instruction. When the root node in a match is a store, isel
6047      // doesn't know how to remap non-chain non-flag uses of other
6048      // nodes in the match, such as the ADD in this case. This leads
6049      // to the ADD being left around and reselected, with the result
6050      // being two adds in the output.  Alas, even if none our users
6051      // are stores, that doesn't prove we're O.K.  Ergo, if we have
6052      // any parents that aren't CopyToReg or SETCC, eschew INC/DEC.
6053      // A better fix seems to require climbing the DAG back to the
6054      // root, and it doesn't seem to be worth the effort.
6055      for (SDNode::use_iterator UI = Op.getNode()->use_begin(),
6056             UE = Op.getNode()->use_end(); UI != UE; ++UI)
6057        if (UI->getOpcode() != ISD::CopyToReg && UI->getOpcode() != ISD::SETCC)
6058          goto default_case;
6059      if (ConstantSDNode *C =
6060            dyn_cast<ConstantSDNode>(Op.getNode()->getOperand(1))) {
6061        // An add of one will be selected as an INC.
6062        if (C->getAPIntValue() == 1) {
6063          Opcode = X86ISD::INC;
6064          NumOperands = 1;
6065          break;
6066        }
6067        // An add of negative one (subtract of one) will be selected as a DEC.
6068        if (C->getAPIntValue().isAllOnesValue()) {
6069          Opcode = X86ISD::DEC;
6070          NumOperands = 1;
6071          break;
6072        }
6073      }
6074      // Otherwise use a regular EFLAGS-setting add.
6075      Opcode = X86ISD::ADD;
6076      NumOperands = 2;
6077      break;
6078    case ISD::AND: {
6079      // If the primary and result isn't used, don't bother using X86ISD::AND,
6080      // because a TEST instruction will be better.
6081      bool NonFlagUse = false;
6082      for (SDNode::use_iterator UI = Op.getNode()->use_begin(),
6083             UE = Op.getNode()->use_end(); UI != UE; ++UI) {
6084        SDNode *User = *UI;
6085        unsigned UOpNo = UI.getOperandNo();
6086        if (User->getOpcode() == ISD::TRUNCATE && User->hasOneUse()) {
6087          // Look pass truncate.
6088          UOpNo = User->use_begin().getOperandNo();
6089          User = *User->use_begin();
6090        }
6091        if (User->getOpcode() != ISD::BRCOND &&
6092            User->getOpcode() != ISD::SETCC &&
6093            (User->getOpcode() != ISD::SELECT || UOpNo != 0)) {
6094          NonFlagUse = true;
6095          break;
6096        }
6097      }
6098      if (!NonFlagUse)
6099        break;
6100    }
6101    // FALL THROUGH
6102    case ISD::SUB:
6103    case ISD::OR:
6104    case ISD::XOR:
6105      // Due to the ISEL shortcoming noted above, be conservative if this op is
6106      // likely to be selected as part of a load-modify-store instruction.
6107      for (SDNode::use_iterator UI = Op.getNode()->use_begin(),
6108           UE = Op.getNode()->use_end(); UI != UE; ++UI)
6109        if (UI->getOpcode() == ISD::STORE)
6110          goto default_case;
6111      // Otherwise use a regular EFLAGS-setting instruction.
6112      switch (Op.getNode()->getOpcode()) {
6113      case ISD::SUB: Opcode = X86ISD::SUB; break;
6114      case ISD::OR:  Opcode = X86ISD::OR;  break;
6115      case ISD::XOR: Opcode = X86ISD::XOR; break;
6116      case ISD::AND: Opcode = X86ISD::AND; break;
6117      default: llvm_unreachable("unexpected operator!");
6118      }
6119      NumOperands = 2;
6120      break;
6121    case X86ISD::ADD:
6122    case X86ISD::SUB:
6123    case X86ISD::INC:
6124    case X86ISD::DEC:
6125    case X86ISD::OR:
6126    case X86ISD::XOR:
6127    case X86ISD::AND:
6128      return SDValue(Op.getNode(), 1);
6129    default:
6130    default_case:
6131      break;
6132    }
6133    if (Opcode != 0) {
6134      SDVTList VTs = DAG.getVTList(Op.getValueType(), MVT::i32);
6135      SmallVector<SDValue, 4> Ops;
6136      for (unsigned i = 0; i != NumOperands; ++i)
6137        Ops.push_back(Op.getOperand(i));
6138      SDValue New = DAG.getNode(Opcode, dl, VTs, &Ops[0], NumOperands);
6139      DAG.ReplaceAllUsesWith(Op, New);
6140      return SDValue(New.getNode(), 1);
6141    }
6142  }
6143
6144  // Otherwise just emit a CMP with 0, which is the TEST pattern.
6145  return DAG.getNode(X86ISD::CMP, dl, MVT::i32, Op,
6146                     DAG.getConstant(0, Op.getValueType()));
6147}
6148
6149/// Emit nodes that will be selected as "cmp Op0,Op1", or something
6150/// equivalent.
6151SDValue X86TargetLowering::EmitCmp(SDValue Op0, SDValue Op1, unsigned X86CC,
6152                                   SelectionDAG &DAG) const {
6153  if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op1))
6154    if (C->getAPIntValue() == 0)
6155      return EmitTest(Op0, X86CC, DAG);
6156
6157  DebugLoc dl = Op0.getDebugLoc();
6158  return DAG.getNode(X86ISD::CMP, dl, MVT::i32, Op0, Op1);
6159}
6160
6161/// LowerToBT - Result of 'and' is compared against zero. Turn it into a BT node
6162/// if it's possible.
6163SDValue X86TargetLowering::LowerToBT(SDValue And, ISD::CondCode CC,
6164                                     DebugLoc dl, SelectionDAG &DAG) const {
6165  SDValue Op0 = And.getOperand(0);
6166  SDValue Op1 = And.getOperand(1);
6167  if (Op0.getOpcode() == ISD::TRUNCATE)
6168    Op0 = Op0.getOperand(0);
6169  if (Op1.getOpcode() == ISD::TRUNCATE)
6170    Op1 = Op1.getOperand(0);
6171
6172  SDValue LHS, RHS;
6173  if (Op1.getOpcode() == ISD::SHL) {
6174    if (ConstantSDNode *And10C = dyn_cast<ConstantSDNode>(Op1.getOperand(0)))
6175      if (And10C->getZExtValue() == 1) {
6176        LHS = Op0;
6177        RHS = Op1.getOperand(1);
6178      }
6179  } else if (Op0.getOpcode() == ISD::SHL) {
6180    if (ConstantSDNode *And00C = dyn_cast<ConstantSDNode>(Op0.getOperand(0)))
6181      if (And00C->getZExtValue() == 1) {
6182        LHS = Op1;
6183        RHS = Op0.getOperand(1);
6184      }
6185  } else if (Op1.getOpcode() == ISD::Constant) {
6186    ConstantSDNode *AndRHS = cast<ConstantSDNode>(Op1);
6187    SDValue AndLHS = Op0;
6188    if (AndRHS->getZExtValue() == 1 && AndLHS.getOpcode() == ISD::SRL) {
6189      LHS = AndLHS.getOperand(0);
6190      RHS = AndLHS.getOperand(1);
6191    }
6192  }
6193
6194  if (LHS.getNode()) {
6195    // If LHS is i8, promote it to i32 with any_extend.  There is no i8 BT
6196    // instruction.  Since the shift amount is in-range-or-undefined, we know
6197    // that doing a bittest on the i32 value is ok.  We extend to i32 because
6198    // the encoding for the i16 version is larger than the i32 version.
6199    // Also promote i16 to i32 for performance / code size reason.
6200    if (LHS.getValueType() == MVT::i8 ||
6201        LHS.getValueType() == MVT::i16)
6202      LHS = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i32, LHS);
6203
6204    // If the operand types disagree, extend the shift amount to match.  Since
6205    // BT ignores high bits (like shifts) we can use anyextend.
6206    if (LHS.getValueType() != RHS.getValueType())
6207      RHS = DAG.getNode(ISD::ANY_EXTEND, dl, LHS.getValueType(), RHS);
6208
6209    SDValue BT = DAG.getNode(X86ISD::BT, dl, MVT::i32, LHS, RHS);
6210    unsigned Cond = CC == ISD::SETEQ ? X86::COND_AE : X86::COND_B;
6211    return DAG.getNode(X86ISD::SETCC, dl, MVT::i8,
6212                       DAG.getConstant(Cond, MVT::i8), BT);
6213  }
6214
6215  return SDValue();
6216}
6217
6218SDValue X86TargetLowering::LowerSETCC(SDValue Op, SelectionDAG &DAG) const {
6219  assert(Op.getValueType() == MVT::i8 && "SetCC type must be 8-bit integer");
6220  SDValue Op0 = Op.getOperand(0);
6221  SDValue Op1 = Op.getOperand(1);
6222  DebugLoc dl = Op.getDebugLoc();
6223  ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(2))->get();
6224
6225  // Optimize to BT if possible.
6226  // Lower (X & (1 << N)) == 0 to BT(X, N).
6227  // Lower ((X >>u N) & 1) != 0 to BT(X, N).
6228  // Lower ((X >>s N) & 1) != 0 to BT(X, N).
6229  if (Op0.getOpcode() == ISD::AND &&
6230      Op0.hasOneUse() &&
6231      Op1.getOpcode() == ISD::Constant &&
6232      cast<ConstantSDNode>(Op1)->isNullValue() &&
6233      (CC == ISD::SETEQ || CC == ISD::SETNE)) {
6234    SDValue NewSetCC = LowerToBT(Op0, CC, dl, DAG);
6235    if (NewSetCC.getNode())
6236      return NewSetCC;
6237  }
6238
6239  // Look for "(setcc) == / != 1" to avoid unncessary setcc.
6240  if (Op0.getOpcode() == X86ISD::SETCC &&
6241      Op1.getOpcode() == ISD::Constant &&
6242      (cast<ConstantSDNode>(Op1)->getZExtValue() == 1 ||
6243       cast<ConstantSDNode>(Op1)->isNullValue()) &&
6244      (CC == ISD::SETEQ || CC == ISD::SETNE)) {
6245    X86::CondCode CCode = (X86::CondCode)Op0.getConstantOperandVal(0);
6246    bool Invert = (CC == ISD::SETNE) ^
6247      cast<ConstantSDNode>(Op1)->isNullValue();
6248    if (Invert)
6249      CCode = X86::GetOppositeBranchCondition(CCode);
6250    return DAG.getNode(X86ISD::SETCC, dl, MVT::i8,
6251                       DAG.getConstant(CCode, MVT::i8), Op0.getOperand(1));
6252  }
6253
6254  bool isFP = Op1.getValueType().isFloatingPoint();
6255  unsigned X86CC = TranslateX86CC(CC, isFP, Op0, Op1, DAG);
6256  if (X86CC == X86::COND_INVALID)
6257    return SDValue();
6258
6259  SDValue Cond = EmitCmp(Op0, Op1, X86CC, DAG);
6260
6261  // Use sbb x, x to materialize carry bit into a GPR.
6262  if (X86CC == X86::COND_B)
6263    return DAG.getNode(ISD::AND, dl, MVT::i8,
6264                       DAG.getNode(X86ISD::SETCC_CARRY, dl, MVT::i8,
6265                                   DAG.getConstant(X86CC, MVT::i8), Cond),
6266                       DAG.getConstant(1, MVT::i8));
6267
6268  return DAG.getNode(X86ISD::SETCC, dl, MVT::i8,
6269                     DAG.getConstant(X86CC, MVT::i8), Cond);
6270}
6271
6272SDValue X86TargetLowering::LowerVSETCC(SDValue Op, SelectionDAG &DAG) const {
6273  SDValue Cond;
6274  SDValue Op0 = Op.getOperand(0);
6275  SDValue Op1 = Op.getOperand(1);
6276  SDValue CC = Op.getOperand(2);
6277  EVT VT = Op.getValueType();
6278  ISD::CondCode SetCCOpcode = cast<CondCodeSDNode>(CC)->get();
6279  bool isFP = Op.getOperand(1).getValueType().isFloatingPoint();
6280  DebugLoc dl = Op.getDebugLoc();
6281
6282  if (isFP) {
6283    unsigned SSECC = 8;
6284    EVT VT0 = Op0.getValueType();
6285    assert(VT0 == MVT::v4f32 || VT0 == MVT::v2f64);
6286    unsigned Opc = VT0 == MVT::v4f32 ? X86ISD::CMPPS : X86ISD::CMPPD;
6287    bool Swap = false;
6288
6289    switch (SetCCOpcode) {
6290    default: break;
6291    case ISD::SETOEQ:
6292    case ISD::SETEQ:  SSECC = 0; break;
6293    case ISD::SETOGT:
6294    case ISD::SETGT: Swap = true; // Fallthrough
6295    case ISD::SETLT:
6296    case ISD::SETOLT: SSECC = 1; break;
6297    case ISD::SETOGE:
6298    case ISD::SETGE: Swap = true; // Fallthrough
6299    case ISD::SETLE:
6300    case ISD::SETOLE: SSECC = 2; break;
6301    case ISD::SETUO:  SSECC = 3; break;
6302    case ISD::SETUNE:
6303    case ISD::SETNE:  SSECC = 4; break;
6304    case ISD::SETULE: Swap = true;
6305    case ISD::SETUGE: SSECC = 5; break;
6306    case ISD::SETULT: Swap = true;
6307    case ISD::SETUGT: SSECC = 6; break;
6308    case ISD::SETO:   SSECC = 7; break;
6309    }
6310    if (Swap)
6311      std::swap(Op0, Op1);
6312
6313    // In the two special cases we can't handle, emit two comparisons.
6314    if (SSECC == 8) {
6315      if (SetCCOpcode == ISD::SETUEQ) {
6316        SDValue UNORD, EQ;
6317        UNORD = DAG.getNode(Opc, dl, VT, Op0, Op1, DAG.getConstant(3, MVT::i8));
6318        EQ = DAG.getNode(Opc, dl, VT, Op0, Op1, DAG.getConstant(0, MVT::i8));
6319        return DAG.getNode(ISD::OR, dl, VT, UNORD, EQ);
6320      }
6321      else if (SetCCOpcode == ISD::SETONE) {
6322        SDValue ORD, NEQ;
6323        ORD = DAG.getNode(Opc, dl, VT, Op0, Op1, DAG.getConstant(7, MVT::i8));
6324        NEQ = DAG.getNode(Opc, dl, VT, Op0, Op1, DAG.getConstant(4, MVT::i8));
6325        return DAG.getNode(ISD::AND, dl, VT, ORD, NEQ);
6326      }
6327      llvm_unreachable("Illegal FP comparison");
6328    }
6329    // Handle all other FP comparisons here.
6330    return DAG.getNode(Opc, dl, VT, Op0, Op1, DAG.getConstant(SSECC, MVT::i8));
6331  }
6332
6333  // We are handling one of the integer comparisons here.  Since SSE only has
6334  // GT and EQ comparisons for integer, swapping operands and multiple
6335  // operations may be required for some comparisons.
6336  unsigned Opc = 0, EQOpc = 0, GTOpc = 0;
6337  bool Swap = false, Invert = false, FlipSigns = false;
6338
6339  switch (VT.getSimpleVT().SimpleTy) {
6340  default: break;
6341  case MVT::v8i8:
6342  case MVT::v16i8: EQOpc = X86ISD::PCMPEQB; GTOpc = X86ISD::PCMPGTB; break;
6343  case MVT::v4i16:
6344  case MVT::v8i16: EQOpc = X86ISD::PCMPEQW; GTOpc = X86ISD::PCMPGTW; break;
6345  case MVT::v2i32:
6346  case MVT::v4i32: EQOpc = X86ISD::PCMPEQD; GTOpc = X86ISD::PCMPGTD; break;
6347  case MVT::v2i64: EQOpc = X86ISD::PCMPEQQ; GTOpc = X86ISD::PCMPGTQ; break;
6348  }
6349
6350  switch (SetCCOpcode) {
6351  default: break;
6352  case ISD::SETNE:  Invert = true;
6353  case ISD::SETEQ:  Opc = EQOpc; break;
6354  case ISD::SETLT:  Swap = true;
6355  case ISD::SETGT:  Opc = GTOpc; break;
6356  case ISD::SETGE:  Swap = true;
6357  case ISD::SETLE:  Opc = GTOpc; Invert = true; break;
6358  case ISD::SETULT: Swap = true;
6359  case ISD::SETUGT: Opc = GTOpc; FlipSigns = true; break;
6360  case ISD::SETUGE: Swap = true;
6361  case ISD::SETULE: Opc = GTOpc; FlipSigns = true; Invert = true; break;
6362  }
6363  if (Swap)
6364    std::swap(Op0, Op1);
6365
6366  // Since SSE has no unsigned integer comparisons, we need to flip  the sign
6367  // bits of the inputs before performing those operations.
6368  if (FlipSigns) {
6369    EVT EltVT = VT.getVectorElementType();
6370    SDValue SignBit = DAG.getConstant(APInt::getSignBit(EltVT.getSizeInBits()),
6371                                      EltVT);
6372    std::vector<SDValue> SignBits(VT.getVectorNumElements(), SignBit);
6373    SDValue SignVec = DAG.getNode(ISD::BUILD_VECTOR, dl, VT, &SignBits[0],
6374                                    SignBits.size());
6375    Op0 = DAG.getNode(ISD::XOR, dl, VT, Op0, SignVec);
6376    Op1 = DAG.getNode(ISD::XOR, dl, VT, Op1, SignVec);
6377  }
6378
6379  SDValue Result = DAG.getNode(Opc, dl, VT, Op0, Op1);
6380
6381  // If the logical-not of the result is required, perform that now.
6382  if (Invert)
6383    Result = DAG.getNOT(dl, Result, VT);
6384
6385  return Result;
6386}
6387
6388// isX86LogicalCmp - Return true if opcode is a X86 logical comparison.
6389static bool isX86LogicalCmp(SDValue Op) {
6390  unsigned Opc = Op.getNode()->getOpcode();
6391  if (Opc == X86ISD::CMP || Opc == X86ISD::COMI || Opc == X86ISD::UCOMI)
6392    return true;
6393  if (Op.getResNo() == 1 &&
6394      (Opc == X86ISD::ADD ||
6395       Opc == X86ISD::SUB ||
6396       Opc == X86ISD::SMUL ||
6397       Opc == X86ISD::UMUL ||
6398       Opc == X86ISD::INC ||
6399       Opc == X86ISD::DEC ||
6400       Opc == X86ISD::OR ||
6401       Opc == X86ISD::XOR ||
6402       Opc == X86ISD::AND))
6403    return true;
6404
6405  return false;
6406}
6407
6408SDValue X86TargetLowering::LowerSELECT(SDValue Op, SelectionDAG &DAG) const {
6409  bool addTest = true;
6410  SDValue Cond  = Op.getOperand(0);
6411  DebugLoc dl = Op.getDebugLoc();
6412  SDValue CC;
6413
6414  if (Cond.getOpcode() == ISD::SETCC) {
6415    SDValue NewCond = LowerSETCC(Cond, DAG);
6416    if (NewCond.getNode())
6417      Cond = NewCond;
6418  }
6419
6420  // (select (x == 0), -1, 0) -> (sign_bit (x - 1))
6421  SDValue Op1 = Op.getOperand(1);
6422  SDValue Op2 = Op.getOperand(2);
6423  if (Cond.getOpcode() == X86ISD::SETCC &&
6424      cast<ConstantSDNode>(Cond.getOperand(0))->getZExtValue() == X86::COND_E) {
6425    SDValue Cmp = Cond.getOperand(1);
6426    if (Cmp.getOpcode() == X86ISD::CMP) {
6427      ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(Op1);
6428      ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(Op2);
6429      ConstantSDNode *RHSC =
6430        dyn_cast<ConstantSDNode>(Cmp.getOperand(1).getNode());
6431      if (N1C && N1C->isAllOnesValue() &&
6432          N2C && N2C->isNullValue() &&
6433          RHSC && RHSC->isNullValue()) {
6434        SDValue CmpOp0 = Cmp.getOperand(0);
6435        Cmp = DAG.getNode(X86ISD::CMP, dl, MVT::i32,
6436                          CmpOp0, DAG.getConstant(1, CmpOp0.getValueType()));
6437        return DAG.getNode(X86ISD::SETCC_CARRY, dl, Op.getValueType(),
6438                           DAG.getConstant(X86::COND_B, MVT::i8), Cmp);
6439      }
6440    }
6441  }
6442
6443  // Look pass (and (setcc_carry (cmp ...)), 1).
6444  if (Cond.getOpcode() == ISD::AND &&
6445      Cond.getOperand(0).getOpcode() == X86ISD::SETCC_CARRY) {
6446    ConstantSDNode *C = dyn_cast<ConstantSDNode>(Cond.getOperand(1));
6447    if (C && C->getAPIntValue() == 1)
6448      Cond = Cond.getOperand(0);
6449  }
6450
6451  // If condition flag is set by a X86ISD::CMP, then use it as the condition
6452  // setting operand in place of the X86ISD::SETCC.
6453  if (Cond.getOpcode() == X86ISD::SETCC ||
6454      Cond.getOpcode() == X86ISD::SETCC_CARRY) {
6455    CC = Cond.getOperand(0);
6456
6457    SDValue Cmp = Cond.getOperand(1);
6458    unsigned Opc = Cmp.getOpcode();
6459    EVT VT = Op.getValueType();
6460
6461    bool IllegalFPCMov = false;
6462    if (VT.isFloatingPoint() && !VT.isVector() &&
6463        !isScalarFPTypeInSSEReg(VT))  // FPStack?
6464      IllegalFPCMov = !hasFPCMov(cast<ConstantSDNode>(CC)->getSExtValue());
6465
6466    if ((isX86LogicalCmp(Cmp) && !IllegalFPCMov) ||
6467        Opc == X86ISD::BT) { // FIXME
6468      Cond = Cmp;
6469      addTest = false;
6470    }
6471  }
6472
6473  if (addTest) {
6474    // Look pass the truncate.
6475    if (Cond.getOpcode() == ISD::TRUNCATE)
6476      Cond = Cond.getOperand(0);
6477
6478    // We know the result of AND is compared against zero. Try to match
6479    // it to BT.
6480    if (Cond.getOpcode() == ISD::AND && Cond.hasOneUse()) {
6481      SDValue NewSetCC = LowerToBT(Cond, ISD::SETNE, dl, DAG);
6482      if (NewSetCC.getNode()) {
6483        CC = NewSetCC.getOperand(0);
6484        Cond = NewSetCC.getOperand(1);
6485        addTest = false;
6486      }
6487    }
6488  }
6489
6490  if (addTest) {
6491    CC = DAG.getConstant(X86::COND_NE, MVT::i8);
6492    Cond = EmitTest(Cond, X86::COND_NE, DAG);
6493  }
6494
6495  // X86ISD::CMOV means set the result (which is operand 1) to the RHS if
6496  // condition is true.
6497  SDVTList VTs = DAG.getVTList(Op.getValueType(), MVT::Flag);
6498  SDValue Ops[] = { Op2, Op1, CC, Cond };
6499  return DAG.getNode(X86ISD::CMOV, dl, VTs, Ops, array_lengthof(Ops));
6500}
6501
6502// isAndOrOfSingleUseSetCCs - Return true if node is an ISD::AND or
6503// ISD::OR of two X86ISD::SETCC nodes each of which has no other use apart
6504// from the AND / OR.
6505static bool isAndOrOfSetCCs(SDValue Op, unsigned &Opc) {
6506  Opc = Op.getOpcode();
6507  if (Opc != ISD::OR && Opc != ISD::AND)
6508    return false;
6509  return (Op.getOperand(0).getOpcode() == X86ISD::SETCC &&
6510          Op.getOperand(0).hasOneUse() &&
6511          Op.getOperand(1).getOpcode() == X86ISD::SETCC &&
6512          Op.getOperand(1).hasOneUse());
6513}
6514
6515// isXor1OfSetCC - Return true if node is an ISD::XOR of a X86ISD::SETCC and
6516// 1 and that the SETCC node has a single use.
6517static bool isXor1OfSetCC(SDValue Op) {
6518  if (Op.getOpcode() != ISD::XOR)
6519    return false;
6520  ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(Op.getOperand(1));
6521  if (N1C && N1C->getAPIntValue() == 1) {
6522    return Op.getOperand(0).getOpcode() == X86ISD::SETCC &&
6523      Op.getOperand(0).hasOneUse();
6524  }
6525  return false;
6526}
6527
6528SDValue X86TargetLowering::LowerBRCOND(SDValue Op, SelectionDAG &DAG) const {
6529  bool addTest = true;
6530  SDValue Chain = Op.getOperand(0);
6531  SDValue Cond  = Op.getOperand(1);
6532  SDValue Dest  = Op.getOperand(2);
6533  DebugLoc dl = Op.getDebugLoc();
6534  SDValue CC;
6535
6536  if (Cond.getOpcode() == ISD::SETCC) {
6537    SDValue NewCond = LowerSETCC(Cond, DAG);
6538    if (NewCond.getNode())
6539      Cond = NewCond;
6540  }
6541#if 0
6542  // FIXME: LowerXALUO doesn't handle these!!
6543  else if (Cond.getOpcode() == X86ISD::ADD  ||
6544           Cond.getOpcode() == X86ISD::SUB  ||
6545           Cond.getOpcode() == X86ISD::SMUL ||
6546           Cond.getOpcode() == X86ISD::UMUL)
6547    Cond = LowerXALUO(Cond, DAG);
6548#endif
6549
6550  // Look pass (and (setcc_carry (cmp ...)), 1).
6551  if (Cond.getOpcode() == ISD::AND &&
6552      Cond.getOperand(0).getOpcode() == X86ISD::SETCC_CARRY) {
6553    ConstantSDNode *C = dyn_cast<ConstantSDNode>(Cond.getOperand(1));
6554    if (C && C->getAPIntValue() == 1)
6555      Cond = Cond.getOperand(0);
6556  }
6557
6558  // If condition flag is set by a X86ISD::CMP, then use it as the condition
6559  // setting operand in place of the X86ISD::SETCC.
6560  if (Cond.getOpcode() == X86ISD::SETCC ||
6561      Cond.getOpcode() == X86ISD::SETCC_CARRY) {
6562    CC = Cond.getOperand(0);
6563
6564    SDValue Cmp = Cond.getOperand(1);
6565    unsigned Opc = Cmp.getOpcode();
6566    // FIXME: WHY THE SPECIAL CASING OF LogicalCmp??
6567    if (isX86LogicalCmp(Cmp) || Opc == X86ISD::BT) {
6568      Cond = Cmp;
6569      addTest = false;
6570    } else {
6571      switch (cast<ConstantSDNode>(CC)->getZExtValue()) {
6572      default: break;
6573      case X86::COND_O:
6574      case X86::COND_B:
6575        // These can only come from an arithmetic instruction with overflow,
6576        // e.g. SADDO, UADDO.
6577        Cond = Cond.getNode()->getOperand(1);
6578        addTest = false;
6579        break;
6580      }
6581    }
6582  } else {
6583    unsigned CondOpc;
6584    if (Cond.hasOneUse() && isAndOrOfSetCCs(Cond, CondOpc)) {
6585      SDValue Cmp = Cond.getOperand(0).getOperand(1);
6586      if (CondOpc == ISD::OR) {
6587        // Also, recognize the pattern generated by an FCMP_UNE. We can emit
6588        // two branches instead of an explicit OR instruction with a
6589        // separate test.
6590        if (Cmp == Cond.getOperand(1).getOperand(1) &&
6591            isX86LogicalCmp(Cmp)) {
6592          CC = Cond.getOperand(0).getOperand(0);
6593          Chain = DAG.getNode(X86ISD::BRCOND, dl, Op.getValueType(),
6594                              Chain, Dest, CC, Cmp);
6595          CC = Cond.getOperand(1).getOperand(0);
6596          Cond = Cmp;
6597          addTest = false;
6598        }
6599      } else { // ISD::AND
6600        // Also, recognize the pattern generated by an FCMP_OEQ. We can emit
6601        // two branches instead of an explicit AND instruction with a
6602        // separate test. However, we only do this if this block doesn't
6603        // have a fall-through edge, because this requires an explicit
6604        // jmp when the condition is false.
6605        if (Cmp == Cond.getOperand(1).getOperand(1) &&
6606            isX86LogicalCmp(Cmp) &&
6607            Op.getNode()->hasOneUse()) {
6608          X86::CondCode CCode =
6609            (X86::CondCode)Cond.getOperand(0).getConstantOperandVal(0);
6610          CCode = X86::GetOppositeBranchCondition(CCode);
6611          CC = DAG.getConstant(CCode, MVT::i8);
6612          SDValue User = SDValue(*Op.getNode()->use_begin(), 0);
6613          // Look for an unconditional branch following this conditional branch.
6614          // We need this because we need to reverse the successors in order
6615          // to implement FCMP_OEQ.
6616          if (User.getOpcode() == ISD::BR) {
6617            SDValue FalseBB = User.getOperand(1);
6618            SDValue NewBR =
6619              DAG.UpdateNodeOperands(User, User.getOperand(0), Dest);
6620            assert(NewBR == User);
6621            Dest = FalseBB;
6622
6623            Chain = DAG.getNode(X86ISD::BRCOND, dl, Op.getValueType(),
6624                                Chain, Dest, CC, Cmp);
6625            X86::CondCode CCode =
6626              (X86::CondCode)Cond.getOperand(1).getConstantOperandVal(0);
6627            CCode = X86::GetOppositeBranchCondition(CCode);
6628            CC = DAG.getConstant(CCode, MVT::i8);
6629            Cond = Cmp;
6630            addTest = false;
6631          }
6632        }
6633      }
6634    } else if (Cond.hasOneUse() && isXor1OfSetCC(Cond)) {
6635      // Recognize for xorb (setcc), 1 patterns. The xor inverts the condition.
6636      // It should be transformed during dag combiner except when the condition
6637      // is set by a arithmetics with overflow node.
6638      X86::CondCode CCode =
6639        (X86::CondCode)Cond.getOperand(0).getConstantOperandVal(0);
6640      CCode = X86::GetOppositeBranchCondition(CCode);
6641      CC = DAG.getConstant(CCode, MVT::i8);
6642      Cond = Cond.getOperand(0).getOperand(1);
6643      addTest = false;
6644    }
6645  }
6646
6647  if (addTest) {
6648    // Look pass the truncate.
6649    if (Cond.getOpcode() == ISD::TRUNCATE)
6650      Cond = Cond.getOperand(0);
6651
6652    // We know the result of AND is compared against zero. Try to match
6653    // it to BT.
6654    if (Cond.getOpcode() == ISD::AND && Cond.hasOneUse()) {
6655      SDValue NewSetCC = LowerToBT(Cond, ISD::SETNE, dl, DAG);
6656      if (NewSetCC.getNode()) {
6657        CC = NewSetCC.getOperand(0);
6658        Cond = NewSetCC.getOperand(1);
6659        addTest = false;
6660      }
6661    }
6662  }
6663
6664  if (addTest) {
6665    CC = DAG.getConstant(X86::COND_NE, MVT::i8);
6666    Cond = EmitTest(Cond, X86::COND_NE, DAG);
6667  }
6668  return DAG.getNode(X86ISD::BRCOND, dl, Op.getValueType(),
6669                     Chain, Dest, CC, Cond);
6670}
6671
6672
6673// Lower dynamic stack allocation to _alloca call for Cygwin/Mingw targets.
6674// Calls to _alloca is needed to probe the stack when allocating more than 4k
6675// bytes in one go. Touching the stack at 4K increments is necessary to ensure
6676// that the guard pages used by the OS virtual memory manager are allocated in
6677// correct sequence.
6678SDValue
6679X86TargetLowering::LowerDYNAMIC_STACKALLOC(SDValue Op,
6680                                           SelectionDAG &DAG) const {
6681  assert(Subtarget->isTargetCygMing() &&
6682         "This should be used only on Cygwin/Mingw targets");
6683  DebugLoc dl = Op.getDebugLoc();
6684
6685  // Get the inputs.
6686  SDValue Chain = Op.getOperand(0);
6687  SDValue Size  = Op.getOperand(1);
6688  // FIXME: Ensure alignment here
6689
6690  SDValue Flag;
6691
6692  EVT IntPtr = getPointerTy();
6693  EVT SPTy = Subtarget->is64Bit() ? MVT::i64 : MVT::i32;
6694
6695  Chain = DAG.getCopyToReg(Chain, dl, X86::EAX, Size, Flag);
6696  Flag = Chain.getValue(1);
6697
6698  SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Flag);
6699
6700  Chain = DAG.getNode(X86ISD::MINGW_ALLOCA, dl, NodeTys, Chain, Flag);
6701  Flag = Chain.getValue(1);
6702
6703  Chain = DAG.getCopyFromReg(Chain, dl, X86StackPtr, SPTy).getValue(1);
6704
6705  SDValue Ops1[2] = { Chain.getValue(0), Chain };
6706  return DAG.getMergeValues(Ops1, 2, dl);
6707}
6708
6709SDValue X86TargetLowering::LowerVASTART(SDValue Op, SelectionDAG &DAG) const {
6710  MachineFunction &MF = DAG.getMachineFunction();
6711  X86MachineFunctionInfo *FuncInfo = MF.getInfo<X86MachineFunctionInfo>();
6712
6713  const Value *SV = cast<SrcValueSDNode>(Op.getOperand(2))->getValue();
6714  DebugLoc dl = Op.getDebugLoc();
6715
6716  if (!Subtarget->is64Bit()) {
6717    // vastart just stores the address of the VarArgsFrameIndex slot into the
6718    // memory location argument.
6719    SDValue FR = DAG.getFrameIndex(FuncInfo->getVarArgsFrameIndex(),
6720                                   getPointerTy());
6721    return DAG.getStore(Op.getOperand(0), dl, FR, Op.getOperand(1), SV, 0,
6722                        false, false, 0);
6723  }
6724
6725  // __va_list_tag:
6726  //   gp_offset         (0 - 6 * 8)
6727  //   fp_offset         (48 - 48 + 8 * 16)
6728  //   overflow_arg_area (point to parameters coming in memory).
6729  //   reg_save_area
6730  SmallVector<SDValue, 8> MemOps;
6731  SDValue FIN = Op.getOperand(1);
6732  // Store gp_offset
6733  SDValue Store = DAG.getStore(Op.getOperand(0), dl,
6734                               DAG.getConstant(FuncInfo->getVarArgsGPOffset(),
6735                                               MVT::i32),
6736                               FIN, SV, 0, false, false, 0);
6737  MemOps.push_back(Store);
6738
6739  // Store fp_offset
6740  FIN = DAG.getNode(ISD::ADD, dl, getPointerTy(),
6741                    FIN, DAG.getIntPtrConstant(4));
6742  Store = DAG.getStore(Op.getOperand(0), dl,
6743                       DAG.getConstant(FuncInfo->getVarArgsFPOffset(),
6744                                       MVT::i32),
6745                       FIN, SV, 0, false, false, 0);
6746  MemOps.push_back(Store);
6747
6748  // Store ptr to overflow_arg_area
6749  FIN = DAG.getNode(ISD::ADD, dl, getPointerTy(),
6750                    FIN, DAG.getIntPtrConstant(4));
6751  SDValue OVFIN = DAG.getFrameIndex(FuncInfo->getVarArgsFrameIndex(),
6752                                    getPointerTy());
6753  Store = DAG.getStore(Op.getOperand(0), dl, OVFIN, FIN, SV, 0,
6754                       false, false, 0);
6755  MemOps.push_back(Store);
6756
6757  // Store ptr to reg_save_area.
6758  FIN = DAG.getNode(ISD::ADD, dl, getPointerTy(),
6759                    FIN, DAG.getIntPtrConstant(8));
6760  SDValue RSFIN = DAG.getFrameIndex(FuncInfo->getRegSaveFrameIndex(),
6761                                    getPointerTy());
6762  Store = DAG.getStore(Op.getOperand(0), dl, RSFIN, FIN, SV, 0,
6763                       false, false, 0);
6764  MemOps.push_back(Store);
6765  return DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
6766                     &MemOps[0], MemOps.size());
6767}
6768
6769SDValue X86TargetLowering::LowerVAARG(SDValue Op, SelectionDAG &DAG) const {
6770  // X86-64 va_list is a struct { i32, i32, i8*, i8* }.
6771  assert(Subtarget->is64Bit() && "This code only handles 64-bit va_arg!");
6772  SDValue Chain = Op.getOperand(0);
6773  SDValue SrcPtr = Op.getOperand(1);
6774  SDValue SrcSV = Op.getOperand(2);
6775
6776  report_fatal_error("VAArgInst is not yet implemented for x86-64!");
6777  return SDValue();
6778}
6779
6780SDValue X86TargetLowering::LowerVACOPY(SDValue Op, SelectionDAG &DAG) const {
6781  // X86-64 va_list is a struct { i32, i32, i8*, i8* }.
6782  assert(Subtarget->is64Bit() && "This code only handles 64-bit va_copy!");
6783  SDValue Chain = Op.getOperand(0);
6784  SDValue DstPtr = Op.getOperand(1);
6785  SDValue SrcPtr = Op.getOperand(2);
6786  const Value *DstSV = cast<SrcValueSDNode>(Op.getOperand(3))->getValue();
6787  const Value *SrcSV = cast<SrcValueSDNode>(Op.getOperand(4))->getValue();
6788  DebugLoc dl = Op.getDebugLoc();
6789
6790  return DAG.getMemcpy(Chain, dl, DstPtr, SrcPtr,
6791                       DAG.getIntPtrConstant(24), 8, /*isVolatile*/false,
6792                       false, DstSV, 0, SrcSV, 0);
6793}
6794
6795SDValue
6796X86TargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op, SelectionDAG &DAG) const {
6797  DebugLoc dl = Op.getDebugLoc();
6798  unsigned IntNo = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
6799  switch (IntNo) {
6800  default: return SDValue();    // Don't custom lower most intrinsics.
6801  // Comparison intrinsics.
6802  case Intrinsic::x86_sse_comieq_ss:
6803  case Intrinsic::x86_sse_comilt_ss:
6804  case Intrinsic::x86_sse_comile_ss:
6805  case Intrinsic::x86_sse_comigt_ss:
6806  case Intrinsic::x86_sse_comige_ss:
6807  case Intrinsic::x86_sse_comineq_ss:
6808  case Intrinsic::x86_sse_ucomieq_ss:
6809  case Intrinsic::x86_sse_ucomilt_ss:
6810  case Intrinsic::x86_sse_ucomile_ss:
6811  case Intrinsic::x86_sse_ucomigt_ss:
6812  case Intrinsic::x86_sse_ucomige_ss:
6813  case Intrinsic::x86_sse_ucomineq_ss:
6814  case Intrinsic::x86_sse2_comieq_sd:
6815  case Intrinsic::x86_sse2_comilt_sd:
6816  case Intrinsic::x86_sse2_comile_sd:
6817  case Intrinsic::x86_sse2_comigt_sd:
6818  case Intrinsic::x86_sse2_comige_sd:
6819  case Intrinsic::x86_sse2_comineq_sd:
6820  case Intrinsic::x86_sse2_ucomieq_sd:
6821  case Intrinsic::x86_sse2_ucomilt_sd:
6822  case Intrinsic::x86_sse2_ucomile_sd:
6823  case Intrinsic::x86_sse2_ucomigt_sd:
6824  case Intrinsic::x86_sse2_ucomige_sd:
6825  case Intrinsic::x86_sse2_ucomineq_sd: {
6826    unsigned Opc = 0;
6827    ISD::CondCode CC = ISD::SETCC_INVALID;
6828    switch (IntNo) {
6829    default: break;
6830    case Intrinsic::x86_sse_comieq_ss:
6831    case Intrinsic::x86_sse2_comieq_sd:
6832      Opc = X86ISD::COMI;
6833      CC = ISD::SETEQ;
6834      break;
6835    case Intrinsic::x86_sse_comilt_ss:
6836    case Intrinsic::x86_sse2_comilt_sd:
6837      Opc = X86ISD::COMI;
6838      CC = ISD::SETLT;
6839      break;
6840    case Intrinsic::x86_sse_comile_ss:
6841    case Intrinsic::x86_sse2_comile_sd:
6842      Opc = X86ISD::COMI;
6843      CC = ISD::SETLE;
6844      break;
6845    case Intrinsic::x86_sse_comigt_ss:
6846    case Intrinsic::x86_sse2_comigt_sd:
6847      Opc = X86ISD::COMI;
6848      CC = ISD::SETGT;
6849      break;
6850    case Intrinsic::x86_sse_comige_ss:
6851    case Intrinsic::x86_sse2_comige_sd:
6852      Opc = X86ISD::COMI;
6853      CC = ISD::SETGE;
6854      break;
6855    case Intrinsic::x86_sse_comineq_ss:
6856    case Intrinsic::x86_sse2_comineq_sd:
6857      Opc = X86ISD::COMI;
6858      CC = ISD::SETNE;
6859      break;
6860    case Intrinsic::x86_sse_ucomieq_ss:
6861    case Intrinsic::x86_sse2_ucomieq_sd:
6862      Opc = X86ISD::UCOMI;
6863      CC = ISD::SETEQ;
6864      break;
6865    case Intrinsic::x86_sse_ucomilt_ss:
6866    case Intrinsic::x86_sse2_ucomilt_sd:
6867      Opc = X86ISD::UCOMI;
6868      CC = ISD::SETLT;
6869      break;
6870    case Intrinsic::x86_sse_ucomile_ss:
6871    case Intrinsic::x86_sse2_ucomile_sd:
6872      Opc = X86ISD::UCOMI;
6873      CC = ISD::SETLE;
6874      break;
6875    case Intrinsic::x86_sse_ucomigt_ss:
6876    case Intrinsic::x86_sse2_ucomigt_sd:
6877      Opc = X86ISD::UCOMI;
6878      CC = ISD::SETGT;
6879      break;
6880    case Intrinsic::x86_sse_ucomige_ss:
6881    case Intrinsic::x86_sse2_ucomige_sd:
6882      Opc = X86ISD::UCOMI;
6883      CC = ISD::SETGE;
6884      break;
6885    case Intrinsic::x86_sse_ucomineq_ss:
6886    case Intrinsic::x86_sse2_ucomineq_sd:
6887      Opc = X86ISD::UCOMI;
6888      CC = ISD::SETNE;
6889      break;
6890    }
6891
6892    SDValue LHS = Op.getOperand(1);
6893    SDValue RHS = Op.getOperand(2);
6894    unsigned X86CC = TranslateX86CC(CC, true, LHS, RHS, DAG);
6895    assert(X86CC != X86::COND_INVALID && "Unexpected illegal condition!");
6896    SDValue Cond = DAG.getNode(Opc, dl, MVT::i32, LHS, RHS);
6897    SDValue SetCC = DAG.getNode(X86ISD::SETCC, dl, MVT::i8,
6898                                DAG.getConstant(X86CC, MVT::i8), Cond);
6899    return DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i32, SetCC);
6900  }
6901  // ptest intrinsics. The intrinsic these come from are designed to return
6902  // an integer value, not just an instruction so lower it to the ptest
6903  // pattern and a setcc for the result.
6904  case Intrinsic::x86_sse41_ptestz:
6905  case Intrinsic::x86_sse41_ptestc:
6906  case Intrinsic::x86_sse41_ptestnzc:{
6907    unsigned X86CC = 0;
6908    switch (IntNo) {
6909    default: llvm_unreachable("Bad fallthrough in Intrinsic lowering.");
6910    case Intrinsic::x86_sse41_ptestz:
6911      // ZF = 1
6912      X86CC = X86::COND_E;
6913      break;
6914    case Intrinsic::x86_sse41_ptestc:
6915      // CF = 1
6916      X86CC = X86::COND_B;
6917      break;
6918    case Intrinsic::x86_sse41_ptestnzc:
6919      // ZF and CF = 0
6920      X86CC = X86::COND_A;
6921      break;
6922    }
6923
6924    SDValue LHS = Op.getOperand(1);
6925    SDValue RHS = Op.getOperand(2);
6926    SDValue Test = DAG.getNode(X86ISD::PTEST, dl, MVT::i32, LHS, RHS);
6927    SDValue CC = DAG.getConstant(X86CC, MVT::i8);
6928    SDValue SetCC = DAG.getNode(X86ISD::SETCC, dl, MVT::i8, CC, Test);
6929    return DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i32, SetCC);
6930  }
6931
6932  // Fix vector shift instructions where the last operand is a non-immediate
6933  // i32 value.
6934  case Intrinsic::x86_sse2_pslli_w:
6935  case Intrinsic::x86_sse2_pslli_d:
6936  case Intrinsic::x86_sse2_pslli_q:
6937  case Intrinsic::x86_sse2_psrli_w:
6938  case Intrinsic::x86_sse2_psrli_d:
6939  case Intrinsic::x86_sse2_psrli_q:
6940  case Intrinsic::x86_sse2_psrai_w:
6941  case Intrinsic::x86_sse2_psrai_d:
6942  case Intrinsic::x86_mmx_pslli_w:
6943  case Intrinsic::x86_mmx_pslli_d:
6944  case Intrinsic::x86_mmx_pslli_q:
6945  case Intrinsic::x86_mmx_psrli_w:
6946  case Intrinsic::x86_mmx_psrli_d:
6947  case Intrinsic::x86_mmx_psrli_q:
6948  case Intrinsic::x86_mmx_psrai_w:
6949  case Intrinsic::x86_mmx_psrai_d: {
6950    SDValue ShAmt = Op.getOperand(2);
6951    if (isa<ConstantSDNode>(ShAmt))
6952      return SDValue();
6953
6954    unsigned NewIntNo = 0;
6955    EVT ShAmtVT = MVT::v4i32;
6956    switch (IntNo) {
6957    case Intrinsic::x86_sse2_pslli_w:
6958      NewIntNo = Intrinsic::x86_sse2_psll_w;
6959      break;
6960    case Intrinsic::x86_sse2_pslli_d:
6961      NewIntNo = Intrinsic::x86_sse2_psll_d;
6962      break;
6963    case Intrinsic::x86_sse2_pslli_q:
6964      NewIntNo = Intrinsic::x86_sse2_psll_q;
6965      break;
6966    case Intrinsic::x86_sse2_psrli_w:
6967      NewIntNo = Intrinsic::x86_sse2_psrl_w;
6968      break;
6969    case Intrinsic::x86_sse2_psrli_d:
6970      NewIntNo = Intrinsic::x86_sse2_psrl_d;
6971      break;
6972    case Intrinsic::x86_sse2_psrli_q:
6973      NewIntNo = Intrinsic::x86_sse2_psrl_q;
6974      break;
6975    case Intrinsic::x86_sse2_psrai_w:
6976      NewIntNo = Intrinsic::x86_sse2_psra_w;
6977      break;
6978    case Intrinsic::x86_sse2_psrai_d:
6979      NewIntNo = Intrinsic::x86_sse2_psra_d;
6980      break;
6981    default: {
6982      ShAmtVT = MVT::v2i32;
6983      switch (IntNo) {
6984      case Intrinsic::x86_mmx_pslli_w:
6985        NewIntNo = Intrinsic::x86_mmx_psll_w;
6986        break;
6987      case Intrinsic::x86_mmx_pslli_d:
6988        NewIntNo = Intrinsic::x86_mmx_psll_d;
6989        break;
6990      case Intrinsic::x86_mmx_pslli_q:
6991        NewIntNo = Intrinsic::x86_mmx_psll_q;
6992        break;
6993      case Intrinsic::x86_mmx_psrli_w:
6994        NewIntNo = Intrinsic::x86_mmx_psrl_w;
6995        break;
6996      case Intrinsic::x86_mmx_psrli_d:
6997        NewIntNo = Intrinsic::x86_mmx_psrl_d;
6998        break;
6999      case Intrinsic::x86_mmx_psrli_q:
7000        NewIntNo = Intrinsic::x86_mmx_psrl_q;
7001        break;
7002      case Intrinsic::x86_mmx_psrai_w:
7003        NewIntNo = Intrinsic::x86_mmx_psra_w;
7004        break;
7005      case Intrinsic::x86_mmx_psrai_d:
7006        NewIntNo = Intrinsic::x86_mmx_psra_d;
7007        break;
7008      default: llvm_unreachable("Impossible intrinsic");  // Can't reach here.
7009      }
7010      break;
7011    }
7012    }
7013
7014    // The vector shift intrinsics with scalars uses 32b shift amounts but
7015    // the sse2/mmx shift instructions reads 64 bits. Set the upper 32 bits
7016    // to be zero.
7017    SDValue ShOps[4];
7018    ShOps[0] = ShAmt;
7019    ShOps[1] = DAG.getConstant(0, MVT::i32);
7020    if (ShAmtVT == MVT::v4i32) {
7021      ShOps[2] = DAG.getUNDEF(MVT::i32);
7022      ShOps[3] = DAG.getUNDEF(MVT::i32);
7023      ShAmt =  DAG.getNode(ISD::BUILD_VECTOR, dl, ShAmtVT, &ShOps[0], 4);
7024    } else {
7025      ShAmt =  DAG.getNode(ISD::BUILD_VECTOR, dl, ShAmtVT, &ShOps[0], 2);
7026    }
7027
7028    EVT VT = Op.getValueType();
7029    ShAmt = DAG.getNode(ISD::BIT_CONVERT, dl, VT, ShAmt);
7030    return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, VT,
7031                       DAG.getConstant(NewIntNo, MVT::i32),
7032                       Op.getOperand(1), ShAmt);
7033  }
7034  }
7035}
7036
7037SDValue X86TargetLowering::LowerRETURNADDR(SDValue Op,
7038                                           SelectionDAG &DAG) const {
7039  MachineFrameInfo *MFI = DAG.getMachineFunction().getFrameInfo();
7040  MFI->setReturnAddressIsTaken(true);
7041
7042  unsigned Depth = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
7043  DebugLoc dl = Op.getDebugLoc();
7044
7045  if (Depth > 0) {
7046    SDValue FrameAddr = LowerFRAMEADDR(Op, DAG);
7047    SDValue Offset =
7048      DAG.getConstant(TD->getPointerSize(),
7049                      Subtarget->is64Bit() ? MVT::i64 : MVT::i32);
7050    return DAG.getLoad(getPointerTy(), dl, DAG.getEntryNode(),
7051                       DAG.getNode(ISD::ADD, dl, getPointerTy(),
7052                                   FrameAddr, Offset),
7053                       NULL, 0, false, false, 0);
7054  }
7055
7056  // Just load the return address.
7057  SDValue RetAddrFI = getReturnAddressFrameIndex(DAG);
7058  return DAG.getLoad(getPointerTy(), dl, DAG.getEntryNode(),
7059                     RetAddrFI, NULL, 0, false, false, 0);
7060}
7061
7062SDValue X86TargetLowering::LowerFRAMEADDR(SDValue Op, SelectionDAG &DAG) const {
7063  MachineFrameInfo *MFI = DAG.getMachineFunction().getFrameInfo();
7064  MFI->setFrameAddressIsTaken(true);
7065
7066  EVT VT = Op.getValueType();
7067  DebugLoc dl = Op.getDebugLoc();  // FIXME probably not meaningful
7068  unsigned Depth = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
7069  unsigned FrameReg = Subtarget->is64Bit() ? X86::RBP : X86::EBP;
7070  SDValue FrameAddr = DAG.getCopyFromReg(DAG.getEntryNode(), dl, FrameReg, VT);
7071  while (Depth--)
7072    FrameAddr = DAG.getLoad(VT, dl, DAG.getEntryNode(), FrameAddr, NULL, 0,
7073                            false, false, 0);
7074  return FrameAddr;
7075}
7076
7077SDValue X86TargetLowering::LowerFRAME_TO_ARGS_OFFSET(SDValue Op,
7078                                                     SelectionDAG &DAG) const {
7079  return DAG.getIntPtrConstant(2*TD->getPointerSize());
7080}
7081
7082SDValue X86TargetLowering::LowerEH_RETURN(SDValue Op, SelectionDAG &DAG) const {
7083  MachineFunction &MF = DAG.getMachineFunction();
7084  SDValue Chain     = Op.getOperand(0);
7085  SDValue Offset    = Op.getOperand(1);
7086  SDValue Handler   = Op.getOperand(2);
7087  DebugLoc dl       = Op.getDebugLoc();
7088
7089  SDValue Frame = DAG.getRegister(Subtarget->is64Bit() ? X86::RBP : X86::EBP,
7090                                  getPointerTy());
7091  unsigned StoreAddrReg = (Subtarget->is64Bit() ? X86::RCX : X86::ECX);
7092
7093  SDValue StoreAddr = DAG.getNode(ISD::SUB, dl, getPointerTy(), Frame,
7094                                  DAG.getIntPtrConstant(-TD->getPointerSize()));
7095  StoreAddr = DAG.getNode(ISD::ADD, dl, getPointerTy(), StoreAddr, Offset);
7096  Chain = DAG.getStore(Chain, dl, Handler, StoreAddr, NULL, 0, false, false, 0);
7097  Chain = DAG.getCopyToReg(Chain, dl, StoreAddrReg, StoreAddr);
7098  MF.getRegInfo().addLiveOut(StoreAddrReg);
7099
7100  return DAG.getNode(X86ISD::EH_RETURN, dl,
7101                     MVT::Other,
7102                     Chain, DAG.getRegister(StoreAddrReg, getPointerTy()));
7103}
7104
7105SDValue X86TargetLowering::LowerTRAMPOLINE(SDValue Op,
7106                                             SelectionDAG &DAG) const {
7107  SDValue Root = Op.getOperand(0);
7108  SDValue Trmp = Op.getOperand(1); // trampoline
7109  SDValue FPtr = Op.getOperand(2); // nested function
7110  SDValue Nest = Op.getOperand(3); // 'nest' parameter value
7111  DebugLoc dl  = Op.getDebugLoc();
7112
7113  const Value *TrmpAddr = cast<SrcValueSDNode>(Op.getOperand(4))->getValue();
7114
7115  if (Subtarget->is64Bit()) {
7116    SDValue OutChains[6];
7117
7118    // Large code-model.
7119    const unsigned char JMP64r  = 0xFF; // 64-bit jmp through register opcode.
7120    const unsigned char MOV64ri = 0xB8; // X86::MOV64ri opcode.
7121
7122    const unsigned char N86R10 = RegInfo->getX86RegNum(X86::R10);
7123    const unsigned char N86R11 = RegInfo->getX86RegNum(X86::R11);
7124
7125    const unsigned char REX_WB = 0x40 | 0x08 | 0x01; // REX prefix
7126
7127    // Load the pointer to the nested function into R11.
7128    unsigned OpCode = ((MOV64ri | N86R11) << 8) | REX_WB; // movabsq r11
7129    SDValue Addr = Trmp;
7130    OutChains[0] = DAG.getStore(Root, dl, DAG.getConstant(OpCode, MVT::i16),
7131                                Addr, TrmpAddr, 0, false, false, 0);
7132
7133    Addr = DAG.getNode(ISD::ADD, dl, MVT::i64, Trmp,
7134                       DAG.getConstant(2, MVT::i64));
7135    OutChains[1] = DAG.getStore(Root, dl, FPtr, Addr, TrmpAddr, 2,
7136                                false, false, 2);
7137
7138    // Load the 'nest' parameter value into R10.
7139    // R10 is specified in X86CallingConv.td
7140    OpCode = ((MOV64ri | N86R10) << 8) | REX_WB; // movabsq r10
7141    Addr = DAG.getNode(ISD::ADD, dl, MVT::i64, Trmp,
7142                       DAG.getConstant(10, MVT::i64));
7143    OutChains[2] = DAG.getStore(Root, dl, DAG.getConstant(OpCode, MVT::i16),
7144                                Addr, TrmpAddr, 10, false, false, 0);
7145
7146    Addr = DAG.getNode(ISD::ADD, dl, MVT::i64, Trmp,
7147                       DAG.getConstant(12, MVT::i64));
7148    OutChains[3] = DAG.getStore(Root, dl, Nest, Addr, TrmpAddr, 12,
7149                                false, false, 2);
7150
7151    // Jump to the nested function.
7152    OpCode = (JMP64r << 8) | REX_WB; // jmpq *...
7153    Addr = DAG.getNode(ISD::ADD, dl, MVT::i64, Trmp,
7154                       DAG.getConstant(20, MVT::i64));
7155    OutChains[4] = DAG.getStore(Root, dl, DAG.getConstant(OpCode, MVT::i16),
7156                                Addr, TrmpAddr, 20, false, false, 0);
7157
7158    unsigned char ModRM = N86R11 | (4 << 3) | (3 << 6); // ...r11
7159    Addr = DAG.getNode(ISD::ADD, dl, MVT::i64, Trmp,
7160                       DAG.getConstant(22, MVT::i64));
7161    OutChains[5] = DAG.getStore(Root, dl, DAG.getConstant(ModRM, MVT::i8), Addr,
7162                                TrmpAddr, 22, false, false, 0);
7163
7164    SDValue Ops[] =
7165      { Trmp, DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains, 6) };
7166    return DAG.getMergeValues(Ops, 2, dl);
7167  } else {
7168    const Function *Func =
7169      cast<Function>(cast<SrcValueSDNode>(Op.getOperand(5))->getValue());
7170    CallingConv::ID CC = Func->getCallingConv();
7171    unsigned NestReg;
7172
7173    switch (CC) {
7174    default:
7175      llvm_unreachable("Unsupported calling convention");
7176    case CallingConv::C:
7177    case CallingConv::X86_StdCall: {
7178      // Pass 'nest' parameter in ECX.
7179      // Must be kept in sync with X86CallingConv.td
7180      NestReg = X86::ECX;
7181
7182      // Check that ECX wasn't needed by an 'inreg' parameter.
7183      const FunctionType *FTy = Func->getFunctionType();
7184      const AttrListPtr &Attrs = Func->getAttributes();
7185
7186      if (!Attrs.isEmpty() && !Func->isVarArg()) {
7187        unsigned InRegCount = 0;
7188        unsigned Idx = 1;
7189
7190        for (FunctionType::param_iterator I = FTy->param_begin(),
7191             E = FTy->param_end(); I != E; ++I, ++Idx)
7192          if (Attrs.paramHasAttr(Idx, Attribute::InReg))
7193            // FIXME: should only count parameters that are lowered to integers.
7194            InRegCount += (TD->getTypeSizeInBits(*I) + 31) / 32;
7195
7196        if (InRegCount > 2) {
7197          report_fatal_error("Nest register in use - reduce number of inreg parameters!");
7198        }
7199      }
7200      break;
7201    }
7202    case CallingConv::X86_FastCall:
7203    case CallingConv::X86_ThisCall:
7204    case CallingConv::Fast:
7205      // Pass 'nest' parameter in EAX.
7206      // Must be kept in sync with X86CallingConv.td
7207      NestReg = X86::EAX;
7208      break;
7209    }
7210
7211    SDValue OutChains[4];
7212    SDValue Addr, Disp;
7213
7214    Addr = DAG.getNode(ISD::ADD, dl, MVT::i32, Trmp,
7215                       DAG.getConstant(10, MVT::i32));
7216    Disp = DAG.getNode(ISD::SUB, dl, MVT::i32, FPtr, Addr);
7217
7218    // This is storing the opcode for MOV32ri.
7219    const unsigned char MOV32ri = 0xB8; // X86::MOV32ri's opcode byte.
7220    const unsigned char N86Reg = RegInfo->getX86RegNum(NestReg);
7221    OutChains[0] = DAG.getStore(Root, dl,
7222                                DAG.getConstant(MOV32ri|N86Reg, MVT::i8),
7223                                Trmp, TrmpAddr, 0, false, false, 0);
7224
7225    Addr = DAG.getNode(ISD::ADD, dl, MVT::i32, Trmp,
7226                       DAG.getConstant(1, MVT::i32));
7227    OutChains[1] = DAG.getStore(Root, dl, Nest, Addr, TrmpAddr, 1,
7228                                false, false, 1);
7229
7230    const unsigned char JMP = 0xE9; // jmp <32bit dst> opcode.
7231    Addr = DAG.getNode(ISD::ADD, dl, MVT::i32, Trmp,
7232                       DAG.getConstant(5, MVT::i32));
7233    OutChains[2] = DAG.getStore(Root, dl, DAG.getConstant(JMP, MVT::i8), Addr,
7234                                TrmpAddr, 5, false, false, 1);
7235
7236    Addr = DAG.getNode(ISD::ADD, dl, MVT::i32, Trmp,
7237                       DAG.getConstant(6, MVT::i32));
7238    OutChains[3] = DAG.getStore(Root, dl, Disp, Addr, TrmpAddr, 6,
7239                                false, false, 1);
7240
7241    SDValue Ops[] =
7242      { Trmp, DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains, 4) };
7243    return DAG.getMergeValues(Ops, 2, dl);
7244  }
7245}
7246
7247SDValue X86TargetLowering::LowerFLT_ROUNDS_(SDValue Op,
7248                                            SelectionDAG &DAG) const {
7249  /*
7250   The rounding mode is in bits 11:10 of FPSR, and has the following
7251   settings:
7252     00 Round to nearest
7253     01 Round to -inf
7254     10 Round to +inf
7255     11 Round to 0
7256
7257  FLT_ROUNDS, on the other hand, expects the following:
7258    -1 Undefined
7259     0 Round to 0
7260     1 Round to nearest
7261     2 Round to +inf
7262     3 Round to -inf
7263
7264  To perform the conversion, we do:
7265    (((((FPSR & 0x800) >> 11) | ((FPSR & 0x400) >> 9)) + 1) & 3)
7266  */
7267
7268  MachineFunction &MF = DAG.getMachineFunction();
7269  const TargetMachine &TM = MF.getTarget();
7270  const TargetFrameInfo &TFI = *TM.getFrameInfo();
7271  unsigned StackAlignment = TFI.getStackAlignment();
7272  EVT VT = Op.getValueType();
7273  DebugLoc dl = Op.getDebugLoc();
7274
7275  // Save FP Control Word to stack slot
7276  int SSFI = MF.getFrameInfo()->CreateStackObject(2, StackAlignment, false);
7277  SDValue StackSlot = DAG.getFrameIndex(SSFI, getPointerTy());
7278
7279  SDValue Chain = DAG.getNode(X86ISD::FNSTCW16m, dl, MVT::Other,
7280                              DAG.getEntryNode(), StackSlot);
7281
7282  // Load FP Control Word from stack slot
7283  SDValue CWD = DAG.getLoad(MVT::i16, dl, Chain, StackSlot, NULL, 0,
7284                            false, false, 0);
7285
7286  // Transform as necessary
7287  SDValue CWD1 =
7288    DAG.getNode(ISD::SRL, dl, MVT::i16,
7289                DAG.getNode(ISD::AND, dl, MVT::i16,
7290                            CWD, DAG.getConstant(0x800, MVT::i16)),
7291                DAG.getConstant(11, MVT::i8));
7292  SDValue CWD2 =
7293    DAG.getNode(ISD::SRL, dl, MVT::i16,
7294                DAG.getNode(ISD::AND, dl, MVT::i16,
7295                            CWD, DAG.getConstant(0x400, MVT::i16)),
7296                DAG.getConstant(9, MVT::i8));
7297
7298  SDValue RetVal =
7299    DAG.getNode(ISD::AND, dl, MVT::i16,
7300                DAG.getNode(ISD::ADD, dl, MVT::i16,
7301                            DAG.getNode(ISD::OR, dl, MVT::i16, CWD1, CWD2),
7302                            DAG.getConstant(1, MVT::i16)),
7303                DAG.getConstant(3, MVT::i16));
7304
7305
7306  return DAG.getNode((VT.getSizeInBits() < 16 ?
7307                      ISD::TRUNCATE : ISD::ZERO_EXTEND), dl, VT, RetVal);
7308}
7309
7310SDValue X86TargetLowering::LowerCTLZ(SDValue Op, SelectionDAG &DAG) const {
7311  EVT VT = Op.getValueType();
7312  EVT OpVT = VT;
7313  unsigned NumBits = VT.getSizeInBits();
7314  DebugLoc dl = Op.getDebugLoc();
7315
7316  Op = Op.getOperand(0);
7317  if (VT == MVT::i8) {
7318    // Zero extend to i32 since there is not an i8 bsr.
7319    OpVT = MVT::i32;
7320    Op = DAG.getNode(ISD::ZERO_EXTEND, dl, OpVT, Op);
7321  }
7322
7323  // Issue a bsr (scan bits in reverse) which also sets EFLAGS.
7324  SDVTList VTs = DAG.getVTList(OpVT, MVT::i32);
7325  Op = DAG.getNode(X86ISD::BSR, dl, VTs, Op);
7326
7327  // If src is zero (i.e. bsr sets ZF), returns NumBits.
7328  SDValue Ops[] = {
7329    Op,
7330    DAG.getConstant(NumBits+NumBits-1, OpVT),
7331    DAG.getConstant(X86::COND_E, MVT::i8),
7332    Op.getValue(1)
7333  };
7334  Op = DAG.getNode(X86ISD::CMOV, dl, OpVT, Ops, array_lengthof(Ops));
7335
7336  // Finally xor with NumBits-1.
7337  Op = DAG.getNode(ISD::XOR, dl, OpVT, Op, DAG.getConstant(NumBits-1, OpVT));
7338
7339  if (VT == MVT::i8)
7340    Op = DAG.getNode(ISD::TRUNCATE, dl, MVT::i8, Op);
7341  return Op;
7342}
7343
7344SDValue X86TargetLowering::LowerCTTZ(SDValue Op, SelectionDAG &DAG) const {
7345  EVT VT = Op.getValueType();
7346  EVT OpVT = VT;
7347  unsigned NumBits = VT.getSizeInBits();
7348  DebugLoc dl = Op.getDebugLoc();
7349
7350  Op = Op.getOperand(0);
7351  if (VT == MVT::i8) {
7352    OpVT = MVT::i32;
7353    Op = DAG.getNode(ISD::ZERO_EXTEND, dl, OpVT, Op);
7354  }
7355
7356  // Issue a bsf (scan bits forward) which also sets EFLAGS.
7357  SDVTList VTs = DAG.getVTList(OpVT, MVT::i32);
7358  Op = DAG.getNode(X86ISD::BSF, dl, VTs, Op);
7359
7360  // If src is zero (i.e. bsf sets ZF), returns NumBits.
7361  SDValue Ops[] = {
7362    Op,
7363    DAG.getConstant(NumBits, OpVT),
7364    DAG.getConstant(X86::COND_E, MVT::i8),
7365    Op.getValue(1)
7366  };
7367  Op = DAG.getNode(X86ISD::CMOV, dl, OpVT, Ops, array_lengthof(Ops));
7368
7369  if (VT == MVT::i8)
7370    Op = DAG.getNode(ISD::TRUNCATE, dl, MVT::i8, Op);
7371  return Op;
7372}
7373
7374SDValue X86TargetLowering::LowerMUL_V2I64(SDValue Op, SelectionDAG &DAG) const {
7375  EVT VT = Op.getValueType();
7376  assert(VT == MVT::v2i64 && "Only know how to lower V2I64 multiply");
7377  DebugLoc dl = Op.getDebugLoc();
7378
7379  //  ulong2 Ahi = __builtin_ia32_psrlqi128( a, 32);
7380  //  ulong2 Bhi = __builtin_ia32_psrlqi128( b, 32);
7381  //  ulong2 AloBlo = __builtin_ia32_pmuludq128( a, b );
7382  //  ulong2 AloBhi = __builtin_ia32_pmuludq128( a, Bhi );
7383  //  ulong2 AhiBlo = __builtin_ia32_pmuludq128( Ahi, b );
7384  //
7385  //  AloBhi = __builtin_ia32_psllqi128( AloBhi, 32 );
7386  //  AhiBlo = __builtin_ia32_psllqi128( AhiBlo, 32 );
7387  //  return AloBlo + AloBhi + AhiBlo;
7388
7389  SDValue A = Op.getOperand(0);
7390  SDValue B = Op.getOperand(1);
7391
7392  SDValue Ahi = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, VT,
7393                       DAG.getConstant(Intrinsic::x86_sse2_psrli_q, MVT::i32),
7394                       A, DAG.getConstant(32, MVT::i32));
7395  SDValue Bhi = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, VT,
7396                       DAG.getConstant(Intrinsic::x86_sse2_psrli_q, MVT::i32),
7397                       B, DAG.getConstant(32, MVT::i32));
7398  SDValue AloBlo = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, VT,
7399                       DAG.getConstant(Intrinsic::x86_sse2_pmulu_dq, MVT::i32),
7400                       A, B);
7401  SDValue AloBhi = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, VT,
7402                       DAG.getConstant(Intrinsic::x86_sse2_pmulu_dq, MVT::i32),
7403                       A, Bhi);
7404  SDValue AhiBlo = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, VT,
7405                       DAG.getConstant(Intrinsic::x86_sse2_pmulu_dq, MVT::i32),
7406                       Ahi, B);
7407  AloBhi = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, VT,
7408                       DAG.getConstant(Intrinsic::x86_sse2_pslli_q, MVT::i32),
7409                       AloBhi, DAG.getConstant(32, MVT::i32));
7410  AhiBlo = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, VT,
7411                       DAG.getConstant(Intrinsic::x86_sse2_pslli_q, MVT::i32),
7412                       AhiBlo, DAG.getConstant(32, MVT::i32));
7413  SDValue Res = DAG.getNode(ISD::ADD, dl, VT, AloBlo, AloBhi);
7414  Res = DAG.getNode(ISD::ADD, dl, VT, Res, AhiBlo);
7415  return Res;
7416}
7417
7418
7419SDValue X86TargetLowering::LowerXALUO(SDValue Op, SelectionDAG &DAG) const {
7420  // Lower the "add/sub/mul with overflow" instruction into a regular ins plus
7421  // a "setcc" instruction that checks the overflow flag. The "brcond" lowering
7422  // looks for this combo and may remove the "setcc" instruction if the "setcc"
7423  // has only one use.
7424  SDNode *N = Op.getNode();
7425  SDValue LHS = N->getOperand(0);
7426  SDValue RHS = N->getOperand(1);
7427  unsigned BaseOp = 0;
7428  unsigned Cond = 0;
7429  DebugLoc dl = Op.getDebugLoc();
7430
7431  switch (Op.getOpcode()) {
7432  default: llvm_unreachable("Unknown ovf instruction!");
7433  case ISD::SADDO:
7434    // A subtract of one will be selected as a INC. Note that INC doesn't
7435    // set CF, so we can't do this for UADDO.
7436    if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op))
7437      if (C->getAPIntValue() == 1) {
7438        BaseOp = X86ISD::INC;
7439        Cond = X86::COND_O;
7440        break;
7441      }
7442    BaseOp = X86ISD::ADD;
7443    Cond = X86::COND_O;
7444    break;
7445  case ISD::UADDO:
7446    BaseOp = X86ISD::ADD;
7447    Cond = X86::COND_B;
7448    break;
7449  case ISD::SSUBO:
7450    // A subtract of one will be selected as a DEC. Note that DEC doesn't
7451    // set CF, so we can't do this for USUBO.
7452    if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op))
7453      if (C->getAPIntValue() == 1) {
7454        BaseOp = X86ISD::DEC;
7455        Cond = X86::COND_O;
7456        break;
7457      }
7458    BaseOp = X86ISD::SUB;
7459    Cond = X86::COND_O;
7460    break;
7461  case ISD::USUBO:
7462    BaseOp = X86ISD::SUB;
7463    Cond = X86::COND_B;
7464    break;
7465  case ISD::SMULO:
7466    BaseOp = X86ISD::SMUL;
7467    Cond = X86::COND_O;
7468    break;
7469  case ISD::UMULO:
7470    BaseOp = X86ISD::UMUL;
7471    Cond = X86::COND_B;
7472    break;
7473  }
7474
7475  // Also sets EFLAGS.
7476  SDVTList VTs = DAG.getVTList(N->getValueType(0), MVT::i32);
7477  SDValue Sum = DAG.getNode(BaseOp, dl, VTs, LHS, RHS);
7478
7479  SDValue SetCC =
7480    DAG.getNode(X86ISD::SETCC, dl, N->getValueType(1),
7481                DAG.getConstant(Cond, MVT::i32), SDValue(Sum.getNode(), 1));
7482
7483  DAG.ReplaceAllUsesOfValueWith(SDValue(N, 1), SetCC);
7484  return Sum;
7485}
7486
7487SDValue X86TargetLowering::LowerCMP_SWAP(SDValue Op, SelectionDAG &DAG) const {
7488  EVT T = Op.getValueType();
7489  DebugLoc dl = Op.getDebugLoc();
7490  unsigned Reg = 0;
7491  unsigned size = 0;
7492  switch(T.getSimpleVT().SimpleTy) {
7493  default:
7494    assert(false && "Invalid value type!");
7495  case MVT::i8:  Reg = X86::AL;  size = 1; break;
7496  case MVT::i16: Reg = X86::AX;  size = 2; break;
7497  case MVT::i32: Reg = X86::EAX; size = 4; break;
7498  case MVT::i64:
7499    assert(Subtarget->is64Bit() && "Node not type legal!");
7500    Reg = X86::RAX; size = 8;
7501    break;
7502  }
7503  SDValue cpIn = DAG.getCopyToReg(Op.getOperand(0), dl, Reg,
7504                                    Op.getOperand(2), SDValue());
7505  SDValue Ops[] = { cpIn.getValue(0),
7506                    Op.getOperand(1),
7507                    Op.getOperand(3),
7508                    DAG.getTargetConstant(size, MVT::i8),
7509                    cpIn.getValue(1) };
7510  SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Flag);
7511  SDValue Result = DAG.getNode(X86ISD::LCMPXCHG_DAG, dl, Tys, Ops, 5);
7512  SDValue cpOut =
7513    DAG.getCopyFromReg(Result.getValue(0), dl, Reg, T, Result.getValue(1));
7514  return cpOut;
7515}
7516
7517SDValue X86TargetLowering::LowerREADCYCLECOUNTER(SDValue Op,
7518                                                 SelectionDAG &DAG) const {
7519  assert(Subtarget->is64Bit() && "Result not type legalized?");
7520  SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Flag);
7521  SDValue TheChain = Op.getOperand(0);
7522  DebugLoc dl = Op.getDebugLoc();
7523  SDValue rd = DAG.getNode(X86ISD::RDTSC_DAG, dl, Tys, &TheChain, 1);
7524  SDValue rax = DAG.getCopyFromReg(rd, dl, X86::RAX, MVT::i64, rd.getValue(1));
7525  SDValue rdx = DAG.getCopyFromReg(rax.getValue(1), dl, X86::RDX, MVT::i64,
7526                                   rax.getValue(2));
7527  SDValue Tmp = DAG.getNode(ISD::SHL, dl, MVT::i64, rdx,
7528                            DAG.getConstant(32, MVT::i8));
7529  SDValue Ops[] = {
7530    DAG.getNode(ISD::OR, dl, MVT::i64, rax, Tmp),
7531    rdx.getValue(1)
7532  };
7533  return DAG.getMergeValues(Ops, 2, dl);
7534}
7535
7536SDValue X86TargetLowering::LowerBIT_CONVERT(SDValue Op,
7537                                            SelectionDAG &DAG) const {
7538  EVT SrcVT = Op.getOperand(0).getValueType();
7539  EVT DstVT = Op.getValueType();
7540  assert((Subtarget->is64Bit() && !Subtarget->hasSSE2() &&
7541          Subtarget->hasMMX() && !DisableMMX) &&
7542         "Unexpected custom BIT_CONVERT");
7543  assert((DstVT == MVT::i64 ||
7544          (DstVT.isVector() && DstVT.getSizeInBits()==64)) &&
7545         "Unexpected custom BIT_CONVERT");
7546  // i64 <=> MMX conversions are Legal.
7547  if (SrcVT==MVT::i64 && DstVT.isVector())
7548    return Op;
7549  if (DstVT==MVT::i64 && SrcVT.isVector())
7550    return Op;
7551  // MMX <=> MMX conversions are Legal.
7552  if (SrcVT.isVector() && DstVT.isVector())
7553    return Op;
7554  // All other conversions need to be expanded.
7555  return SDValue();
7556}
7557SDValue X86TargetLowering::LowerLOAD_SUB(SDValue Op, SelectionDAG &DAG) const {
7558  SDNode *Node = Op.getNode();
7559  DebugLoc dl = Node->getDebugLoc();
7560  EVT T = Node->getValueType(0);
7561  SDValue negOp = DAG.getNode(ISD::SUB, dl, T,
7562                              DAG.getConstant(0, T), Node->getOperand(2));
7563  return DAG.getAtomic(ISD::ATOMIC_LOAD_ADD, dl,
7564                       cast<AtomicSDNode>(Node)->getMemoryVT(),
7565                       Node->getOperand(0),
7566                       Node->getOperand(1), negOp,
7567                       cast<AtomicSDNode>(Node)->getSrcValue(),
7568                       cast<AtomicSDNode>(Node)->getAlignment());
7569}
7570
7571/// LowerOperation - Provide custom lowering hooks for some operations.
7572///
7573SDValue X86TargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
7574  switch (Op.getOpcode()) {
7575  default: llvm_unreachable("Should not custom lower this!");
7576  case ISD::ATOMIC_CMP_SWAP:    return LowerCMP_SWAP(Op,DAG);
7577  case ISD::ATOMIC_LOAD_SUB:    return LowerLOAD_SUB(Op,DAG);
7578  case ISD::BUILD_VECTOR:       return LowerBUILD_VECTOR(Op, DAG);
7579  case ISD::CONCAT_VECTORS:     return LowerCONCAT_VECTORS(Op, DAG);
7580  case ISD::VECTOR_SHUFFLE:     return LowerVECTOR_SHUFFLE(Op, DAG);
7581  case ISD::EXTRACT_VECTOR_ELT: return LowerEXTRACT_VECTOR_ELT(Op, DAG);
7582  case ISD::INSERT_VECTOR_ELT:  return LowerINSERT_VECTOR_ELT(Op, DAG);
7583  case ISD::SCALAR_TO_VECTOR:   return LowerSCALAR_TO_VECTOR(Op, DAG);
7584  case ISD::ConstantPool:       return LowerConstantPool(Op, DAG);
7585  case ISD::GlobalAddress:      return LowerGlobalAddress(Op, DAG);
7586  case ISD::GlobalTLSAddress:   return LowerGlobalTLSAddress(Op, DAG);
7587  case ISD::ExternalSymbol:     return LowerExternalSymbol(Op, DAG);
7588  case ISD::BlockAddress:       return LowerBlockAddress(Op, DAG);
7589  case ISD::SHL_PARTS:
7590  case ISD::SRA_PARTS:
7591  case ISD::SRL_PARTS:          return LowerShift(Op, DAG);
7592  case ISD::SINT_TO_FP:         return LowerSINT_TO_FP(Op, DAG);
7593  case ISD::UINT_TO_FP:         return LowerUINT_TO_FP(Op, DAG);
7594  case ISD::FP_TO_SINT:         return LowerFP_TO_SINT(Op, DAG);
7595  case ISD::FP_TO_UINT:         return LowerFP_TO_UINT(Op, DAG);
7596  case ISD::FABS:               return LowerFABS(Op, DAG);
7597  case ISD::FNEG:               return LowerFNEG(Op, DAG);
7598  case ISD::FCOPYSIGN:          return LowerFCOPYSIGN(Op, DAG);
7599  case ISD::SETCC:              return LowerSETCC(Op, DAG);
7600  case ISD::VSETCC:             return LowerVSETCC(Op, DAG);
7601  case ISD::SELECT:             return LowerSELECT(Op, DAG);
7602  case ISD::BRCOND:             return LowerBRCOND(Op, DAG);
7603  case ISD::JumpTable:          return LowerJumpTable(Op, DAG);
7604  case ISD::VASTART:            return LowerVASTART(Op, DAG);
7605  case ISD::VAARG:              return LowerVAARG(Op, DAG);
7606  case ISD::VACOPY:             return LowerVACOPY(Op, DAG);
7607  case ISD::INTRINSIC_WO_CHAIN: return LowerINTRINSIC_WO_CHAIN(Op, DAG);
7608  case ISD::RETURNADDR:         return LowerRETURNADDR(Op, DAG);
7609  case ISD::FRAMEADDR:          return LowerFRAMEADDR(Op, DAG);
7610  case ISD::FRAME_TO_ARGS_OFFSET:
7611                                return LowerFRAME_TO_ARGS_OFFSET(Op, DAG);
7612  case ISD::DYNAMIC_STACKALLOC: return LowerDYNAMIC_STACKALLOC(Op, DAG);
7613  case ISD::EH_RETURN:          return LowerEH_RETURN(Op, DAG);
7614  case ISD::TRAMPOLINE:         return LowerTRAMPOLINE(Op, DAG);
7615  case ISD::FLT_ROUNDS_:        return LowerFLT_ROUNDS_(Op, DAG);
7616  case ISD::CTLZ:               return LowerCTLZ(Op, DAG);
7617  case ISD::CTTZ:               return LowerCTTZ(Op, DAG);
7618  case ISD::MUL:                return LowerMUL_V2I64(Op, DAG);
7619  case ISD::SADDO:
7620  case ISD::UADDO:
7621  case ISD::SSUBO:
7622  case ISD::USUBO:
7623  case ISD::SMULO:
7624  case ISD::UMULO:              return LowerXALUO(Op, DAG);
7625  case ISD::READCYCLECOUNTER:   return LowerREADCYCLECOUNTER(Op, DAG);
7626  case ISD::BIT_CONVERT:        return LowerBIT_CONVERT(Op, DAG);
7627  }
7628}
7629
7630void X86TargetLowering::
7631ReplaceATOMIC_BINARY_64(SDNode *Node, SmallVectorImpl<SDValue>&Results,
7632                        SelectionDAG &DAG, unsigned NewOp) const {
7633  EVT T = Node->getValueType(0);
7634  DebugLoc dl = Node->getDebugLoc();
7635  assert (T == MVT::i64 && "Only know how to expand i64 atomics");
7636
7637  SDValue Chain = Node->getOperand(0);
7638  SDValue In1 = Node->getOperand(1);
7639  SDValue In2L = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32,
7640                             Node->getOperand(2), DAG.getIntPtrConstant(0));
7641  SDValue In2H = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32,
7642                             Node->getOperand(2), DAG.getIntPtrConstant(1));
7643  SDValue Ops[] = { Chain, In1, In2L, In2H };
7644  SDVTList Tys = DAG.getVTList(MVT::i32, MVT::i32, MVT::Other);
7645  SDValue Result =
7646    DAG.getMemIntrinsicNode(NewOp, dl, Tys, Ops, 4, MVT::i64,
7647                            cast<MemSDNode>(Node)->getMemOperand());
7648  SDValue OpsF[] = { Result.getValue(0), Result.getValue(1)};
7649  Results.push_back(DAG.getNode(ISD::BUILD_PAIR, dl, MVT::i64, OpsF, 2));
7650  Results.push_back(Result.getValue(2));
7651}
7652
7653/// ReplaceNodeResults - Replace a node with an illegal result type
7654/// with a new node built out of custom code.
7655void X86TargetLowering::ReplaceNodeResults(SDNode *N,
7656                                           SmallVectorImpl<SDValue>&Results,
7657                                           SelectionDAG &DAG) const {
7658  DebugLoc dl = N->getDebugLoc();
7659  switch (N->getOpcode()) {
7660  default:
7661    assert(false && "Do not know how to custom type legalize this operation!");
7662    return;
7663  case ISD::FP_TO_SINT: {
7664    std::pair<SDValue,SDValue> Vals =
7665        FP_TO_INTHelper(SDValue(N, 0), DAG, true);
7666    SDValue FIST = Vals.first, StackSlot = Vals.second;
7667    if (FIST.getNode() != 0) {
7668      EVT VT = N->getValueType(0);
7669      // Return a load from the stack slot.
7670      Results.push_back(DAG.getLoad(VT, dl, FIST, StackSlot, NULL, 0,
7671                                    false, false, 0));
7672    }
7673    return;
7674  }
7675  case ISD::READCYCLECOUNTER: {
7676    SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Flag);
7677    SDValue TheChain = N->getOperand(0);
7678    SDValue rd = DAG.getNode(X86ISD::RDTSC_DAG, dl, Tys, &TheChain, 1);
7679    SDValue eax = DAG.getCopyFromReg(rd, dl, X86::EAX, MVT::i32,
7680                                     rd.getValue(1));
7681    SDValue edx = DAG.getCopyFromReg(eax.getValue(1), dl, X86::EDX, MVT::i32,
7682                                     eax.getValue(2));
7683    // Use a buildpair to merge the two 32-bit values into a 64-bit one.
7684    SDValue Ops[] = { eax, edx };
7685    Results.push_back(DAG.getNode(ISD::BUILD_PAIR, dl, MVT::i64, Ops, 2));
7686    Results.push_back(edx.getValue(1));
7687    return;
7688  }
7689  case ISD::ATOMIC_CMP_SWAP: {
7690    EVT T = N->getValueType(0);
7691    assert (T == MVT::i64 && "Only know how to expand i64 Cmp and Swap");
7692    SDValue cpInL, cpInH;
7693    cpInL = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32, N->getOperand(2),
7694                        DAG.getConstant(0, MVT::i32));
7695    cpInH = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32, N->getOperand(2),
7696                        DAG.getConstant(1, MVT::i32));
7697    cpInL = DAG.getCopyToReg(N->getOperand(0), dl, X86::EAX, cpInL, SDValue());
7698    cpInH = DAG.getCopyToReg(cpInL.getValue(0), dl, X86::EDX, cpInH,
7699                             cpInL.getValue(1));
7700    SDValue swapInL, swapInH;
7701    swapInL = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32, N->getOperand(3),
7702                          DAG.getConstant(0, MVT::i32));
7703    swapInH = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32, N->getOperand(3),
7704                          DAG.getConstant(1, MVT::i32));
7705    swapInL = DAG.getCopyToReg(cpInH.getValue(0), dl, X86::EBX, swapInL,
7706                               cpInH.getValue(1));
7707    swapInH = DAG.getCopyToReg(swapInL.getValue(0), dl, X86::ECX, swapInH,
7708                               swapInL.getValue(1));
7709    SDValue Ops[] = { swapInH.getValue(0),
7710                      N->getOperand(1),
7711                      swapInH.getValue(1) };
7712    SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Flag);
7713    SDValue Result = DAG.getNode(X86ISD::LCMPXCHG8_DAG, dl, Tys, Ops, 3);
7714    SDValue cpOutL = DAG.getCopyFromReg(Result.getValue(0), dl, X86::EAX,
7715                                        MVT::i32, Result.getValue(1));
7716    SDValue cpOutH = DAG.getCopyFromReg(cpOutL.getValue(1), dl, X86::EDX,
7717                                        MVT::i32, cpOutL.getValue(2));
7718    SDValue OpsF[] = { cpOutL.getValue(0), cpOutH.getValue(0)};
7719    Results.push_back(DAG.getNode(ISD::BUILD_PAIR, dl, MVT::i64, OpsF, 2));
7720    Results.push_back(cpOutH.getValue(1));
7721    return;
7722  }
7723  case ISD::ATOMIC_LOAD_ADD:
7724    ReplaceATOMIC_BINARY_64(N, Results, DAG, X86ISD::ATOMADD64_DAG);
7725    return;
7726  case ISD::ATOMIC_LOAD_AND:
7727    ReplaceATOMIC_BINARY_64(N, Results, DAG, X86ISD::ATOMAND64_DAG);
7728    return;
7729  case ISD::ATOMIC_LOAD_NAND:
7730    ReplaceATOMIC_BINARY_64(N, Results, DAG, X86ISD::ATOMNAND64_DAG);
7731    return;
7732  case ISD::ATOMIC_LOAD_OR:
7733    ReplaceATOMIC_BINARY_64(N, Results, DAG, X86ISD::ATOMOR64_DAG);
7734    return;
7735  case ISD::ATOMIC_LOAD_SUB:
7736    ReplaceATOMIC_BINARY_64(N, Results, DAG, X86ISD::ATOMSUB64_DAG);
7737    return;
7738  case ISD::ATOMIC_LOAD_XOR:
7739    ReplaceATOMIC_BINARY_64(N, Results, DAG, X86ISD::ATOMXOR64_DAG);
7740    return;
7741  case ISD::ATOMIC_SWAP:
7742    ReplaceATOMIC_BINARY_64(N, Results, DAG, X86ISD::ATOMSWAP64_DAG);
7743    return;
7744  }
7745}
7746
7747const char *X86TargetLowering::getTargetNodeName(unsigned Opcode) const {
7748  switch (Opcode) {
7749  default: return NULL;
7750  case X86ISD::BSF:                return "X86ISD::BSF";
7751  case X86ISD::BSR:                return "X86ISD::BSR";
7752  case X86ISD::SHLD:               return "X86ISD::SHLD";
7753  case X86ISD::SHRD:               return "X86ISD::SHRD";
7754  case X86ISD::FAND:               return "X86ISD::FAND";
7755  case X86ISD::FOR:                return "X86ISD::FOR";
7756  case X86ISD::FXOR:               return "X86ISD::FXOR";
7757  case X86ISD::FSRL:               return "X86ISD::FSRL";
7758  case X86ISD::FILD:               return "X86ISD::FILD";
7759  case X86ISD::FILD_FLAG:          return "X86ISD::FILD_FLAG";
7760  case X86ISD::FP_TO_INT16_IN_MEM: return "X86ISD::FP_TO_INT16_IN_MEM";
7761  case X86ISD::FP_TO_INT32_IN_MEM: return "X86ISD::FP_TO_INT32_IN_MEM";
7762  case X86ISD::FP_TO_INT64_IN_MEM: return "X86ISD::FP_TO_INT64_IN_MEM";
7763  case X86ISD::FLD:                return "X86ISD::FLD";
7764  case X86ISD::FST:                return "X86ISD::FST";
7765  case X86ISD::CALL:               return "X86ISD::CALL";
7766  case X86ISD::RDTSC_DAG:          return "X86ISD::RDTSC_DAG";
7767  case X86ISD::BT:                 return "X86ISD::BT";
7768  case X86ISD::CMP:                return "X86ISD::CMP";
7769  case X86ISD::COMI:               return "X86ISD::COMI";
7770  case X86ISD::UCOMI:              return "X86ISD::UCOMI";
7771  case X86ISD::SETCC:              return "X86ISD::SETCC";
7772  case X86ISD::SETCC_CARRY:        return "X86ISD::SETCC_CARRY";
7773  case X86ISD::CMOV:               return "X86ISD::CMOV";
7774  case X86ISD::BRCOND:             return "X86ISD::BRCOND";
7775  case X86ISD::RET_FLAG:           return "X86ISD::RET_FLAG";
7776  case X86ISD::REP_STOS:           return "X86ISD::REP_STOS";
7777  case X86ISD::REP_MOVS:           return "X86ISD::REP_MOVS";
7778  case X86ISD::GlobalBaseReg:      return "X86ISD::GlobalBaseReg";
7779  case X86ISD::Wrapper:            return "X86ISD::Wrapper";
7780  case X86ISD::WrapperRIP:         return "X86ISD::WrapperRIP";
7781  case X86ISD::PEXTRB:             return "X86ISD::PEXTRB";
7782  case X86ISD::PEXTRW:             return "X86ISD::PEXTRW";
7783  case X86ISD::INSERTPS:           return "X86ISD::INSERTPS";
7784  case X86ISD::PINSRB:             return "X86ISD::PINSRB";
7785  case X86ISD::PINSRW:             return "X86ISD::PINSRW";
7786  case X86ISD::MMX_PINSRW:         return "X86ISD::MMX_PINSRW";
7787  case X86ISD::PSHUFB:             return "X86ISD::PSHUFB";
7788  case X86ISD::FMAX:               return "X86ISD::FMAX";
7789  case X86ISD::FMIN:               return "X86ISD::FMIN";
7790  case X86ISD::FRSQRT:             return "X86ISD::FRSQRT";
7791  case X86ISD::FRCP:               return "X86ISD::FRCP";
7792  case X86ISD::TLSADDR:            return "X86ISD::TLSADDR";
7793  case X86ISD::TLSCALL:            return "X86ISD::TLSCALL";
7794  case X86ISD::SegmentBaseAddress: return "X86ISD::SegmentBaseAddress";
7795  case X86ISD::EH_RETURN:          return "X86ISD::EH_RETURN";
7796  case X86ISD::TC_RETURN:          return "X86ISD::TC_RETURN";
7797  case X86ISD::FNSTCW16m:          return "X86ISD::FNSTCW16m";
7798  case X86ISD::LCMPXCHG_DAG:       return "X86ISD::LCMPXCHG_DAG";
7799  case X86ISD::LCMPXCHG8_DAG:      return "X86ISD::LCMPXCHG8_DAG";
7800  case X86ISD::ATOMADD64_DAG:      return "X86ISD::ATOMADD64_DAG";
7801  case X86ISD::ATOMSUB64_DAG:      return "X86ISD::ATOMSUB64_DAG";
7802  case X86ISD::ATOMOR64_DAG:       return "X86ISD::ATOMOR64_DAG";
7803  case X86ISD::ATOMXOR64_DAG:      return "X86ISD::ATOMXOR64_DAG";
7804  case X86ISD::ATOMAND64_DAG:      return "X86ISD::ATOMAND64_DAG";
7805  case X86ISD::ATOMNAND64_DAG:     return "X86ISD::ATOMNAND64_DAG";
7806  case X86ISD::VZEXT_MOVL:         return "X86ISD::VZEXT_MOVL";
7807  case X86ISD::VZEXT_LOAD:         return "X86ISD::VZEXT_LOAD";
7808  case X86ISD::VSHL:               return "X86ISD::VSHL";
7809  case X86ISD::VSRL:               return "X86ISD::VSRL";
7810  case X86ISD::CMPPD:              return "X86ISD::CMPPD";
7811  case X86ISD::CMPPS:              return "X86ISD::CMPPS";
7812  case X86ISD::PCMPEQB:            return "X86ISD::PCMPEQB";
7813  case X86ISD::PCMPEQW:            return "X86ISD::PCMPEQW";
7814  case X86ISD::PCMPEQD:            return "X86ISD::PCMPEQD";
7815  case X86ISD::PCMPEQQ:            return "X86ISD::PCMPEQQ";
7816  case X86ISD::PCMPGTB:            return "X86ISD::PCMPGTB";
7817  case X86ISD::PCMPGTW:            return "X86ISD::PCMPGTW";
7818  case X86ISD::PCMPGTD:            return "X86ISD::PCMPGTD";
7819  case X86ISD::PCMPGTQ:            return "X86ISD::PCMPGTQ";
7820  case X86ISD::ADD:                return "X86ISD::ADD";
7821  case X86ISD::SUB:                return "X86ISD::SUB";
7822  case X86ISD::SMUL:               return "X86ISD::SMUL";
7823  case X86ISD::UMUL:               return "X86ISD::UMUL";
7824  case X86ISD::INC:                return "X86ISD::INC";
7825  case X86ISD::DEC:                return "X86ISD::DEC";
7826  case X86ISD::OR:                 return "X86ISD::OR";
7827  case X86ISD::XOR:                return "X86ISD::XOR";
7828  case X86ISD::AND:                return "X86ISD::AND";
7829  case X86ISD::MUL_IMM:            return "X86ISD::MUL_IMM";
7830  case X86ISD::PTEST:              return "X86ISD::PTEST";
7831  case X86ISD::VASTART_SAVE_XMM_REGS: return "X86ISD::VASTART_SAVE_XMM_REGS";
7832  case X86ISD::MINGW_ALLOCA:       return "X86ISD::MINGW_ALLOCA";
7833  }
7834}
7835
7836// isLegalAddressingMode - Return true if the addressing mode represented
7837// by AM is legal for this target, for a load/store of the specified type.
7838bool X86TargetLowering::isLegalAddressingMode(const AddrMode &AM,
7839                                              const Type *Ty) const {
7840  // X86 supports extremely general addressing modes.
7841  CodeModel::Model M = getTargetMachine().getCodeModel();
7842
7843  // X86 allows a sign-extended 32-bit immediate field as a displacement.
7844  if (!X86::isOffsetSuitableForCodeModel(AM.BaseOffs, M, AM.BaseGV != NULL))
7845    return false;
7846
7847  if (AM.BaseGV) {
7848    unsigned GVFlags =
7849      Subtarget->ClassifyGlobalReference(AM.BaseGV, getTargetMachine());
7850
7851    // If a reference to this global requires an extra load, we can't fold it.
7852    if (isGlobalStubReference(GVFlags))
7853      return false;
7854
7855    // If BaseGV requires a register for the PIC base, we cannot also have a
7856    // BaseReg specified.
7857    if (AM.HasBaseReg && isGlobalRelativeToPICBase(GVFlags))
7858      return false;
7859
7860    // If lower 4G is not available, then we must use rip-relative addressing.
7861    if (Subtarget->is64Bit() && (AM.BaseOffs || AM.Scale > 1))
7862      return false;
7863  }
7864
7865  switch (AM.Scale) {
7866  case 0:
7867  case 1:
7868  case 2:
7869  case 4:
7870  case 8:
7871    // These scales always work.
7872    break;
7873  case 3:
7874  case 5:
7875  case 9:
7876    // These scales are formed with basereg+scalereg.  Only accept if there is
7877    // no basereg yet.
7878    if (AM.HasBaseReg)
7879      return false;
7880    break;
7881  default:  // Other stuff never works.
7882    return false;
7883  }
7884
7885  return true;
7886}
7887
7888
7889bool X86TargetLowering::isTruncateFree(const Type *Ty1, const Type *Ty2) const {
7890  if (!Ty1->isIntegerTy() || !Ty2->isIntegerTy())
7891    return false;
7892  unsigned NumBits1 = Ty1->getPrimitiveSizeInBits();
7893  unsigned NumBits2 = Ty2->getPrimitiveSizeInBits();
7894  if (NumBits1 <= NumBits2)
7895    return false;
7896  return true;
7897}
7898
7899bool X86TargetLowering::isTruncateFree(EVT VT1, EVT VT2) const {
7900  if (!VT1.isInteger() || !VT2.isInteger())
7901    return false;
7902  unsigned NumBits1 = VT1.getSizeInBits();
7903  unsigned NumBits2 = VT2.getSizeInBits();
7904  if (NumBits1 <= NumBits2)
7905    return false;
7906  return true;
7907}
7908
7909bool X86TargetLowering::isZExtFree(const Type *Ty1, const Type *Ty2) const {
7910  // x86-64 implicitly zero-extends 32-bit results in 64-bit registers.
7911  return Ty1->isIntegerTy(32) && Ty2->isIntegerTy(64) && Subtarget->is64Bit();
7912}
7913
7914bool X86TargetLowering::isZExtFree(EVT VT1, EVT VT2) const {
7915  // x86-64 implicitly zero-extends 32-bit results in 64-bit registers.
7916  return VT1 == MVT::i32 && VT2 == MVT::i64 && Subtarget->is64Bit();
7917}
7918
7919bool X86TargetLowering::isNarrowingProfitable(EVT VT1, EVT VT2) const {
7920  // i16 instructions are longer (0x66 prefix) and potentially slower.
7921  return !(VT1 == MVT::i32 && VT2 == MVT::i16);
7922}
7923
7924/// isShuffleMaskLegal - Targets can use this to indicate that they only
7925/// support *some* VECTOR_SHUFFLE operations, those with specific masks.
7926/// By default, if a target supports the VECTOR_SHUFFLE node, all mask values
7927/// are assumed to be legal.
7928bool
7929X86TargetLowering::isShuffleMaskLegal(const SmallVectorImpl<int> &M,
7930                                      EVT VT) const {
7931  // Very little shuffling can be done for 64-bit vectors right now.
7932  if (VT.getSizeInBits() == 64)
7933    return isPALIGNRMask(M, VT, Subtarget->hasSSSE3());
7934
7935  // FIXME: pshufb, blends, shifts.
7936  return (VT.getVectorNumElements() == 2 ||
7937          ShuffleVectorSDNode::isSplatMask(&M[0], VT) ||
7938          isMOVLMask(M, VT) ||
7939          isSHUFPMask(M, VT) ||
7940          isPSHUFDMask(M, VT) ||
7941          isPSHUFHWMask(M, VT) ||
7942          isPSHUFLWMask(M, VT) ||
7943          isPALIGNRMask(M, VT, Subtarget->hasSSSE3()) ||
7944          isUNPCKLMask(M, VT) ||
7945          isUNPCKHMask(M, VT) ||
7946          isUNPCKL_v_undef_Mask(M, VT) ||
7947          isUNPCKH_v_undef_Mask(M, VT));
7948}
7949
7950bool
7951X86TargetLowering::isVectorClearMaskLegal(const SmallVectorImpl<int> &Mask,
7952                                          EVT VT) const {
7953  unsigned NumElts = VT.getVectorNumElements();
7954  // FIXME: This collection of masks seems suspect.
7955  if (NumElts == 2)
7956    return true;
7957  if (NumElts == 4 && VT.getSizeInBits() == 128) {
7958    return (isMOVLMask(Mask, VT)  ||
7959            isCommutedMOVLMask(Mask, VT, true) ||
7960            isSHUFPMask(Mask, VT) ||
7961            isCommutedSHUFPMask(Mask, VT));
7962  }
7963  return false;
7964}
7965
7966//===----------------------------------------------------------------------===//
7967//                           X86 Scheduler Hooks
7968//===----------------------------------------------------------------------===//
7969
7970// private utility function
7971MachineBasicBlock *
7972X86TargetLowering::EmitAtomicBitwiseWithCustomInserter(MachineInstr *bInstr,
7973                                                       MachineBasicBlock *MBB,
7974                                                       unsigned regOpc,
7975                                                       unsigned immOpc,
7976                                                       unsigned LoadOpc,
7977                                                       unsigned CXchgOpc,
7978                                                       unsigned copyOpc,
7979                                                       unsigned notOpc,
7980                                                       unsigned EAXreg,
7981                                                       TargetRegisterClass *RC,
7982                                                       bool invSrc) const {
7983  // For the atomic bitwise operator, we generate
7984  //   thisMBB:
7985  //   newMBB:
7986  //     ld  t1 = [bitinstr.addr]
7987  //     op  t2 = t1, [bitinstr.val]
7988  //     mov EAX = t1
7989  //     lcs dest = [bitinstr.addr], t2  [EAX is implicit]
7990  //     bz  newMBB
7991  //     fallthrough -->nextMBB
7992  const TargetInstrInfo *TII = getTargetMachine().getInstrInfo();
7993  const BasicBlock *LLVM_BB = MBB->getBasicBlock();
7994  MachineFunction::iterator MBBIter = MBB;
7995  ++MBBIter;
7996
7997  /// First build the CFG
7998  MachineFunction *F = MBB->getParent();
7999  MachineBasicBlock *thisMBB = MBB;
8000  MachineBasicBlock *newMBB = F->CreateMachineBasicBlock(LLVM_BB);
8001  MachineBasicBlock *nextMBB = F->CreateMachineBasicBlock(LLVM_BB);
8002  F->insert(MBBIter, newMBB);
8003  F->insert(MBBIter, nextMBB);
8004
8005  // Move all successors to thisMBB to nextMBB
8006  nextMBB->transferSuccessors(thisMBB);
8007
8008  // Update thisMBB to fall through to newMBB
8009  thisMBB->addSuccessor(newMBB);
8010
8011  // newMBB jumps to itself and fall through to nextMBB
8012  newMBB->addSuccessor(nextMBB);
8013  newMBB->addSuccessor(newMBB);
8014
8015  // Insert instructions into newMBB based on incoming instruction
8016  assert(bInstr->getNumOperands() < X86AddrNumOperands + 4 &&
8017         "unexpected number of operands");
8018  DebugLoc dl = bInstr->getDebugLoc();
8019  MachineOperand& destOper = bInstr->getOperand(0);
8020  MachineOperand* argOpers[2 + X86AddrNumOperands];
8021  int numArgs = bInstr->getNumOperands() - 1;
8022  for (int i=0; i < numArgs; ++i)
8023    argOpers[i] = &bInstr->getOperand(i+1);
8024
8025  // x86 address has 4 operands: base, index, scale, and displacement
8026  int lastAddrIndx = X86AddrNumOperands - 1; // [0,3]
8027  int valArgIndx = lastAddrIndx + 1;
8028
8029  unsigned t1 = F->getRegInfo().createVirtualRegister(RC);
8030  MachineInstrBuilder MIB = BuildMI(newMBB, dl, TII->get(LoadOpc), t1);
8031  for (int i=0; i <= lastAddrIndx; ++i)
8032    (*MIB).addOperand(*argOpers[i]);
8033
8034  unsigned tt = F->getRegInfo().createVirtualRegister(RC);
8035  if (invSrc) {
8036    MIB = BuildMI(newMBB, dl, TII->get(notOpc), tt).addReg(t1);
8037  }
8038  else
8039    tt = t1;
8040
8041  unsigned t2 = F->getRegInfo().createVirtualRegister(RC);
8042  assert((argOpers[valArgIndx]->isReg() ||
8043          argOpers[valArgIndx]->isImm()) &&
8044         "invalid operand");
8045  if (argOpers[valArgIndx]->isReg())
8046    MIB = BuildMI(newMBB, dl, TII->get(regOpc), t2);
8047  else
8048    MIB = BuildMI(newMBB, dl, TII->get(immOpc), t2);
8049  MIB.addReg(tt);
8050  (*MIB).addOperand(*argOpers[valArgIndx]);
8051
8052  MIB = BuildMI(newMBB, dl, TII->get(copyOpc), EAXreg);
8053  MIB.addReg(t1);
8054
8055  MIB = BuildMI(newMBB, dl, TII->get(CXchgOpc));
8056  for (int i=0; i <= lastAddrIndx; ++i)
8057    (*MIB).addOperand(*argOpers[i]);
8058  MIB.addReg(t2);
8059  assert(bInstr->hasOneMemOperand() && "Unexpected number of memoperand");
8060  (*MIB).setMemRefs(bInstr->memoperands_begin(),
8061                    bInstr->memoperands_end());
8062
8063  MIB = BuildMI(newMBB, dl, TII->get(copyOpc), destOper.getReg());
8064  MIB.addReg(EAXreg);
8065
8066  // insert branch
8067  BuildMI(newMBB, dl, TII->get(X86::JNE_4)).addMBB(newMBB);
8068
8069  F->DeleteMachineInstr(bInstr);   // The pseudo instruction is gone now.
8070  return nextMBB;
8071}
8072
8073// private utility function:  64 bit atomics on 32 bit host.
8074MachineBasicBlock *
8075X86TargetLowering::EmitAtomicBit6432WithCustomInserter(MachineInstr *bInstr,
8076                                                       MachineBasicBlock *MBB,
8077                                                       unsigned regOpcL,
8078                                                       unsigned regOpcH,
8079                                                       unsigned immOpcL,
8080                                                       unsigned immOpcH,
8081                                                       bool invSrc) const {
8082  // For the atomic bitwise operator, we generate
8083  //   thisMBB (instructions are in pairs, except cmpxchg8b)
8084  //     ld t1,t2 = [bitinstr.addr]
8085  //   newMBB:
8086  //     out1, out2 = phi (thisMBB, t1/t2) (newMBB, t3/t4)
8087  //     op  t5, t6 <- out1, out2, [bitinstr.val]
8088  //      (for SWAP, substitute:  mov t5, t6 <- [bitinstr.val])
8089  //     mov ECX, EBX <- t5, t6
8090  //     mov EAX, EDX <- t1, t2
8091  //     cmpxchg8b [bitinstr.addr]  [EAX, EDX, EBX, ECX implicit]
8092  //     mov t3, t4 <- EAX, EDX
8093  //     bz  newMBB
8094  //     result in out1, out2
8095  //     fallthrough -->nextMBB
8096
8097  const TargetRegisterClass *RC = X86::GR32RegisterClass;
8098  const unsigned LoadOpc = X86::MOV32rm;
8099  const unsigned copyOpc = X86::MOV32rr;
8100  const unsigned NotOpc = X86::NOT32r;
8101  const TargetInstrInfo *TII = getTargetMachine().getInstrInfo();
8102  const BasicBlock *LLVM_BB = MBB->getBasicBlock();
8103  MachineFunction::iterator MBBIter = MBB;
8104  ++MBBIter;
8105
8106  /// First build the CFG
8107  MachineFunction *F = MBB->getParent();
8108  MachineBasicBlock *thisMBB = MBB;
8109  MachineBasicBlock *newMBB = F->CreateMachineBasicBlock(LLVM_BB);
8110  MachineBasicBlock *nextMBB = F->CreateMachineBasicBlock(LLVM_BB);
8111  F->insert(MBBIter, newMBB);
8112  F->insert(MBBIter, nextMBB);
8113
8114  // Move all successors to thisMBB to nextMBB
8115  nextMBB->transferSuccessors(thisMBB);
8116
8117  // Update thisMBB to fall through to newMBB
8118  thisMBB->addSuccessor(newMBB);
8119
8120  // newMBB jumps to itself and fall through to nextMBB
8121  newMBB->addSuccessor(nextMBB);
8122  newMBB->addSuccessor(newMBB);
8123
8124  DebugLoc dl = bInstr->getDebugLoc();
8125  // Insert instructions into newMBB based on incoming instruction
8126  // There are 8 "real" operands plus 9 implicit def/uses, ignored here.
8127  assert(bInstr->getNumOperands() < X86AddrNumOperands + 14 &&
8128         "unexpected number of operands");
8129  MachineOperand& dest1Oper = bInstr->getOperand(0);
8130  MachineOperand& dest2Oper = bInstr->getOperand(1);
8131  MachineOperand* argOpers[2 + X86AddrNumOperands];
8132  for (int i=0; i < 2 + X86AddrNumOperands; ++i) {
8133    argOpers[i] = &bInstr->getOperand(i+2);
8134
8135    // We use some of the operands multiple times, so conservatively just
8136    // clear any kill flags that might be present.
8137    if (argOpers[i]->isReg() && argOpers[i]->isUse())
8138      argOpers[i]->setIsKill(false);
8139  }
8140
8141  // x86 address has 5 operands: base, index, scale, displacement, and segment.
8142  int lastAddrIndx = X86AddrNumOperands - 1; // [0,3]
8143
8144  unsigned t1 = F->getRegInfo().createVirtualRegister(RC);
8145  MachineInstrBuilder MIB = BuildMI(thisMBB, dl, TII->get(LoadOpc), t1);
8146  for (int i=0; i <= lastAddrIndx; ++i)
8147    (*MIB).addOperand(*argOpers[i]);
8148  unsigned t2 = F->getRegInfo().createVirtualRegister(RC);
8149  MIB = BuildMI(thisMBB, dl, TII->get(LoadOpc), t2);
8150  // add 4 to displacement.
8151  for (int i=0; i <= lastAddrIndx-2; ++i)
8152    (*MIB).addOperand(*argOpers[i]);
8153  MachineOperand newOp3 = *(argOpers[3]);
8154  if (newOp3.isImm())
8155    newOp3.setImm(newOp3.getImm()+4);
8156  else
8157    newOp3.setOffset(newOp3.getOffset()+4);
8158  (*MIB).addOperand(newOp3);
8159  (*MIB).addOperand(*argOpers[lastAddrIndx]);
8160
8161  // t3/4 are defined later, at the bottom of the loop
8162  unsigned t3 = F->getRegInfo().createVirtualRegister(RC);
8163  unsigned t4 = F->getRegInfo().createVirtualRegister(RC);
8164  BuildMI(newMBB, dl, TII->get(X86::PHI), dest1Oper.getReg())
8165    .addReg(t1).addMBB(thisMBB).addReg(t3).addMBB(newMBB);
8166  BuildMI(newMBB, dl, TII->get(X86::PHI), dest2Oper.getReg())
8167    .addReg(t2).addMBB(thisMBB).addReg(t4).addMBB(newMBB);
8168
8169  // The subsequent operations should be using the destination registers of
8170  //the PHI instructions.
8171  if (invSrc) {
8172    t1 = F->getRegInfo().createVirtualRegister(RC);
8173    t2 = F->getRegInfo().createVirtualRegister(RC);
8174    MIB = BuildMI(newMBB, dl, TII->get(NotOpc), t1).addReg(dest1Oper.getReg());
8175    MIB = BuildMI(newMBB, dl, TII->get(NotOpc), t2).addReg(dest2Oper.getReg());
8176  } else {
8177    t1 = dest1Oper.getReg();
8178    t2 = dest2Oper.getReg();
8179  }
8180
8181  int valArgIndx = lastAddrIndx + 1;
8182  assert((argOpers[valArgIndx]->isReg() ||
8183          argOpers[valArgIndx]->isImm()) &&
8184         "invalid operand");
8185  unsigned t5 = F->getRegInfo().createVirtualRegister(RC);
8186  unsigned t6 = F->getRegInfo().createVirtualRegister(RC);
8187  if (argOpers[valArgIndx]->isReg())
8188    MIB = BuildMI(newMBB, dl, TII->get(regOpcL), t5);
8189  else
8190    MIB = BuildMI(newMBB, dl, TII->get(immOpcL), t5);
8191  if (regOpcL != X86::MOV32rr)
8192    MIB.addReg(t1);
8193  (*MIB).addOperand(*argOpers[valArgIndx]);
8194  assert(argOpers[valArgIndx + 1]->isReg() ==
8195         argOpers[valArgIndx]->isReg());
8196  assert(argOpers[valArgIndx + 1]->isImm() ==
8197         argOpers[valArgIndx]->isImm());
8198  if (argOpers[valArgIndx + 1]->isReg())
8199    MIB = BuildMI(newMBB, dl, TII->get(regOpcH), t6);
8200  else
8201    MIB = BuildMI(newMBB, dl, TII->get(immOpcH), t6);
8202  if (regOpcH != X86::MOV32rr)
8203    MIB.addReg(t2);
8204  (*MIB).addOperand(*argOpers[valArgIndx + 1]);
8205
8206  MIB = BuildMI(newMBB, dl, TII->get(copyOpc), X86::EAX);
8207  MIB.addReg(t1);
8208  MIB = BuildMI(newMBB, dl, TII->get(copyOpc), X86::EDX);
8209  MIB.addReg(t2);
8210
8211  MIB = BuildMI(newMBB, dl, TII->get(copyOpc), X86::EBX);
8212  MIB.addReg(t5);
8213  MIB = BuildMI(newMBB, dl, TII->get(copyOpc), X86::ECX);
8214  MIB.addReg(t6);
8215
8216  MIB = BuildMI(newMBB, dl, TII->get(X86::LCMPXCHG8B));
8217  for (int i=0; i <= lastAddrIndx; ++i)
8218    (*MIB).addOperand(*argOpers[i]);
8219
8220  assert(bInstr->hasOneMemOperand() && "Unexpected number of memoperand");
8221  (*MIB).setMemRefs(bInstr->memoperands_begin(),
8222                    bInstr->memoperands_end());
8223
8224  MIB = BuildMI(newMBB, dl, TII->get(copyOpc), t3);
8225  MIB.addReg(X86::EAX);
8226  MIB = BuildMI(newMBB, dl, TII->get(copyOpc), t4);
8227  MIB.addReg(X86::EDX);
8228
8229  // insert branch
8230  BuildMI(newMBB, dl, TII->get(X86::JNE_4)).addMBB(newMBB);
8231
8232  F->DeleteMachineInstr(bInstr);   // The pseudo instruction is gone now.
8233  return nextMBB;
8234}
8235
8236// private utility function
8237MachineBasicBlock *
8238X86TargetLowering::EmitAtomicMinMaxWithCustomInserter(MachineInstr *mInstr,
8239                                                      MachineBasicBlock *MBB,
8240                                                      unsigned cmovOpc) const {
8241  // For the atomic min/max operator, we generate
8242  //   thisMBB:
8243  //   newMBB:
8244  //     ld t1 = [min/max.addr]
8245  //     mov t2 = [min/max.val]
8246  //     cmp  t1, t2
8247  //     cmov[cond] t2 = t1
8248  //     mov EAX = t1
8249  //     lcs dest = [bitinstr.addr], t2  [EAX is implicit]
8250  //     bz   newMBB
8251  //     fallthrough -->nextMBB
8252  //
8253  const TargetInstrInfo *TII = getTargetMachine().getInstrInfo();
8254  const BasicBlock *LLVM_BB = MBB->getBasicBlock();
8255  MachineFunction::iterator MBBIter = MBB;
8256  ++MBBIter;
8257
8258  /// First build the CFG
8259  MachineFunction *F = MBB->getParent();
8260  MachineBasicBlock *thisMBB = MBB;
8261  MachineBasicBlock *newMBB = F->CreateMachineBasicBlock(LLVM_BB);
8262  MachineBasicBlock *nextMBB = F->CreateMachineBasicBlock(LLVM_BB);
8263  F->insert(MBBIter, newMBB);
8264  F->insert(MBBIter, nextMBB);
8265
8266  // Move all successors of thisMBB to nextMBB
8267  nextMBB->transferSuccessors(thisMBB);
8268
8269  // Update thisMBB to fall through to newMBB
8270  thisMBB->addSuccessor(newMBB);
8271
8272  // newMBB jumps to newMBB and fall through to nextMBB
8273  newMBB->addSuccessor(nextMBB);
8274  newMBB->addSuccessor(newMBB);
8275
8276  DebugLoc dl = mInstr->getDebugLoc();
8277  // Insert instructions into newMBB based on incoming instruction
8278  assert(mInstr->getNumOperands() < X86AddrNumOperands + 4 &&
8279         "unexpected number of operands");
8280  MachineOperand& destOper = mInstr->getOperand(0);
8281  MachineOperand* argOpers[2 + X86AddrNumOperands];
8282  int numArgs = mInstr->getNumOperands() - 1;
8283  for (int i=0; i < numArgs; ++i)
8284    argOpers[i] = &mInstr->getOperand(i+1);
8285
8286  // x86 address has 4 operands: base, index, scale, and displacement
8287  int lastAddrIndx = X86AddrNumOperands - 1; // [0,3]
8288  int valArgIndx = lastAddrIndx + 1;
8289
8290  unsigned t1 = F->getRegInfo().createVirtualRegister(X86::GR32RegisterClass);
8291  MachineInstrBuilder MIB = BuildMI(newMBB, dl, TII->get(X86::MOV32rm), t1);
8292  for (int i=0; i <= lastAddrIndx; ++i)
8293    (*MIB).addOperand(*argOpers[i]);
8294
8295  // We only support register and immediate values
8296  assert((argOpers[valArgIndx]->isReg() ||
8297          argOpers[valArgIndx]->isImm()) &&
8298         "invalid operand");
8299
8300  unsigned t2 = F->getRegInfo().createVirtualRegister(X86::GR32RegisterClass);
8301  if (argOpers[valArgIndx]->isReg())
8302    MIB = BuildMI(newMBB, dl, TII->get(X86::MOV32rr), t2);
8303  else
8304    MIB = BuildMI(newMBB, dl, TII->get(X86::MOV32rr), t2);
8305  (*MIB).addOperand(*argOpers[valArgIndx]);
8306
8307  MIB = BuildMI(newMBB, dl, TII->get(X86::MOV32rr), X86::EAX);
8308  MIB.addReg(t1);
8309
8310  MIB = BuildMI(newMBB, dl, TII->get(X86::CMP32rr));
8311  MIB.addReg(t1);
8312  MIB.addReg(t2);
8313
8314  // Generate movc
8315  unsigned t3 = F->getRegInfo().createVirtualRegister(X86::GR32RegisterClass);
8316  MIB = BuildMI(newMBB, dl, TII->get(cmovOpc),t3);
8317  MIB.addReg(t2);
8318  MIB.addReg(t1);
8319
8320  // Cmp and exchange if none has modified the memory location
8321  MIB = BuildMI(newMBB, dl, TII->get(X86::LCMPXCHG32));
8322  for (int i=0; i <= lastAddrIndx; ++i)
8323    (*MIB).addOperand(*argOpers[i]);
8324  MIB.addReg(t3);
8325  assert(mInstr->hasOneMemOperand() && "Unexpected number of memoperand");
8326  (*MIB).setMemRefs(mInstr->memoperands_begin(),
8327                    mInstr->memoperands_end());
8328
8329  MIB = BuildMI(newMBB, dl, TII->get(X86::MOV32rr), destOper.getReg());
8330  MIB.addReg(X86::EAX);
8331
8332  // insert branch
8333  BuildMI(newMBB, dl, TII->get(X86::JNE_4)).addMBB(newMBB);
8334
8335  F->DeleteMachineInstr(mInstr);   // The pseudo instruction is gone now.
8336  return nextMBB;
8337}
8338
8339// FIXME: When we get size specific XMM0 registers, i.e. XMM0_V16I8
8340// all of this code can be replaced with that in the .td file.
8341MachineBasicBlock *
8342X86TargetLowering::EmitPCMP(MachineInstr *MI, MachineBasicBlock *BB,
8343                            unsigned numArgs, bool memArg) const {
8344
8345  MachineFunction *F = BB->getParent();
8346  DebugLoc dl = MI->getDebugLoc();
8347  const TargetInstrInfo *TII = getTargetMachine().getInstrInfo();
8348
8349  unsigned Opc;
8350  if (memArg)
8351    Opc = numArgs == 3 ? X86::PCMPISTRM128rm : X86::PCMPESTRM128rm;
8352  else
8353    Opc = numArgs == 3 ? X86::PCMPISTRM128rr : X86::PCMPESTRM128rr;
8354
8355  MachineInstrBuilder MIB = BuildMI(BB, dl, TII->get(Opc));
8356
8357  for (unsigned i = 0; i < numArgs; ++i) {
8358    MachineOperand &Op = MI->getOperand(i+1);
8359
8360    if (!(Op.isReg() && Op.isImplicit()))
8361      MIB.addOperand(Op);
8362  }
8363
8364  BuildMI(BB, dl, TII->get(X86::MOVAPSrr), MI->getOperand(0).getReg())
8365    .addReg(X86::XMM0);
8366
8367  F->DeleteMachineInstr(MI);
8368
8369  return BB;
8370}
8371
8372MachineBasicBlock *
8373X86TargetLowering::EmitVAStartSaveXMMRegsWithCustomInserter(
8374                                                 MachineInstr *MI,
8375                                                 MachineBasicBlock *MBB) const {
8376  // Emit code to save XMM registers to the stack. The ABI says that the
8377  // number of registers to save is given in %al, so it's theoretically
8378  // possible to do an indirect jump trick to avoid saving all of them,
8379  // however this code takes a simpler approach and just executes all
8380  // of the stores if %al is non-zero. It's less code, and it's probably
8381  // easier on the hardware branch predictor, and stores aren't all that
8382  // expensive anyway.
8383
8384  // Create the new basic blocks. One block contains all the XMM stores,
8385  // and one block is the final destination regardless of whether any
8386  // stores were performed.
8387  const BasicBlock *LLVM_BB = MBB->getBasicBlock();
8388  MachineFunction *F = MBB->getParent();
8389  MachineFunction::iterator MBBIter = MBB;
8390  ++MBBIter;
8391  MachineBasicBlock *XMMSaveMBB = F->CreateMachineBasicBlock(LLVM_BB);
8392  MachineBasicBlock *EndMBB = F->CreateMachineBasicBlock(LLVM_BB);
8393  F->insert(MBBIter, XMMSaveMBB);
8394  F->insert(MBBIter, EndMBB);
8395
8396  // Set up the CFG.
8397  // Move any original successors of MBB to the end block.
8398  EndMBB->transferSuccessors(MBB);
8399  // The original block will now fall through to the XMM save block.
8400  MBB->addSuccessor(XMMSaveMBB);
8401  // The XMMSaveMBB will fall through to the end block.
8402  XMMSaveMBB->addSuccessor(EndMBB);
8403
8404  // Now add the instructions.
8405  const TargetInstrInfo *TII = getTargetMachine().getInstrInfo();
8406  DebugLoc DL = MI->getDebugLoc();
8407
8408  unsigned CountReg = MI->getOperand(0).getReg();
8409  int64_t RegSaveFrameIndex = MI->getOperand(1).getImm();
8410  int64_t VarArgsFPOffset = MI->getOperand(2).getImm();
8411
8412  if (!Subtarget->isTargetWin64()) {
8413    // If %al is 0, branch around the XMM save block.
8414    BuildMI(MBB, DL, TII->get(X86::TEST8rr)).addReg(CountReg).addReg(CountReg);
8415    BuildMI(MBB, DL, TII->get(X86::JE_4)).addMBB(EndMBB);
8416    MBB->addSuccessor(EndMBB);
8417  }
8418
8419  // In the XMM save block, save all the XMM argument registers.
8420  for (int i = 3, e = MI->getNumOperands(); i != e; ++i) {
8421    int64_t Offset = (i - 3) * 16 + VarArgsFPOffset;
8422    MachineMemOperand *MMO =
8423      F->getMachineMemOperand(
8424        PseudoSourceValue::getFixedStack(RegSaveFrameIndex),
8425        MachineMemOperand::MOStore, Offset,
8426        /*Size=*/16, /*Align=*/16);
8427    BuildMI(XMMSaveMBB, DL, TII->get(X86::MOVAPSmr))
8428      .addFrameIndex(RegSaveFrameIndex)
8429      .addImm(/*Scale=*/1)
8430      .addReg(/*IndexReg=*/0)
8431      .addImm(/*Disp=*/Offset)
8432      .addReg(/*Segment=*/0)
8433      .addReg(MI->getOperand(i).getReg())
8434      .addMemOperand(MMO);
8435  }
8436
8437  F->DeleteMachineInstr(MI);   // The pseudo instruction is gone now.
8438
8439  return EndMBB;
8440}
8441
8442MachineBasicBlock *
8443X86TargetLowering::EmitLoweredSelect(MachineInstr *MI,
8444                                     MachineBasicBlock *BB) const {
8445  const TargetInstrInfo *TII = getTargetMachine().getInstrInfo();
8446  DebugLoc DL = MI->getDebugLoc();
8447
8448  // To "insert" a SELECT_CC instruction, we actually have to insert the
8449  // diamond control-flow pattern.  The incoming instruction knows the
8450  // destination vreg to set, the condition code register to branch on, the
8451  // true/false values to select between, and a branch opcode to use.
8452  const BasicBlock *LLVM_BB = BB->getBasicBlock();
8453  MachineFunction::iterator It = BB;
8454  ++It;
8455
8456  //  thisMBB:
8457  //  ...
8458  //   TrueVal = ...
8459  //   cmpTY ccX, r1, r2
8460  //   bCC copy1MBB
8461  //   fallthrough --> copy0MBB
8462  MachineBasicBlock *thisMBB = BB;
8463  MachineFunction *F = BB->getParent();
8464  MachineBasicBlock *copy0MBB = F->CreateMachineBasicBlock(LLVM_BB);
8465  MachineBasicBlock *sinkMBB = F->CreateMachineBasicBlock(LLVM_BB);
8466  unsigned Opc =
8467    X86::GetCondBranchFromCond((X86::CondCode)MI->getOperand(3).getImm());
8468
8469  BuildMI(BB, DL, TII->get(Opc)).addMBB(sinkMBB);
8470  F->insert(It, copy0MBB);
8471  F->insert(It, sinkMBB);
8472
8473  // Update machine-CFG edges by first adding all successors of the current
8474  // block to the new block which will contain the Phi node for the select.
8475  for (MachineBasicBlock::succ_iterator I = BB->succ_begin(),
8476         E = BB->succ_end(); I != E; ++I)
8477    sinkMBB->addSuccessor(*I);
8478
8479  // Next, remove all successors of the current block, and add the true
8480  // and fallthrough blocks as its successors.
8481  while (!BB->succ_empty())
8482    BB->removeSuccessor(BB->succ_begin());
8483
8484  // Add the true and fallthrough blocks as its successors.
8485  BB->addSuccessor(copy0MBB);
8486  BB->addSuccessor(sinkMBB);
8487
8488  // If the EFLAGS register isn't dead in the terminator, then claim that it's
8489  // live into the sink and copy blocks.
8490  const MachineFunction *MF = BB->getParent();
8491  const TargetRegisterInfo *TRI = MF->getTarget().getRegisterInfo();
8492  BitVector ReservedRegs = TRI->getReservedRegs(*MF);
8493  const MachineInstr *Term = BB->getFirstTerminator();
8494
8495  for (unsigned I = 0, E = Term->getNumOperands(); I != E; ++I) {
8496    const MachineOperand &MO = Term->getOperand(I);
8497    if (!MO.isReg() || MO.isKill() || MO.isDead()) continue;
8498    unsigned Reg = MO.getReg();
8499    if (Reg != X86::EFLAGS) continue;
8500    copy0MBB->addLiveIn(Reg);
8501    sinkMBB->addLiveIn(Reg);
8502  }
8503
8504  //  copy0MBB:
8505  //   %FalseValue = ...
8506  //   # fallthrough to sinkMBB
8507  copy0MBB->addSuccessor(sinkMBB);
8508
8509  //  sinkMBB:
8510  //   %Result = phi [ %FalseValue, copy0MBB ], [ %TrueValue, thisMBB ]
8511  //  ...
8512  BuildMI(sinkMBB, DL, TII->get(X86::PHI), MI->getOperand(0).getReg())
8513    .addReg(MI->getOperand(1).getReg()).addMBB(copy0MBB)
8514    .addReg(MI->getOperand(2).getReg()).addMBB(thisMBB);
8515
8516  F->DeleteMachineInstr(MI);   // The pseudo instruction is gone now.
8517  return sinkMBB;
8518}
8519
8520MachineBasicBlock *
8521X86TargetLowering::EmitLoweredMingwAlloca(MachineInstr *MI,
8522                                          MachineBasicBlock *BB) const {
8523  const TargetInstrInfo *TII = getTargetMachine().getInstrInfo();
8524  DebugLoc DL = MI->getDebugLoc();
8525  MachineFunction *F = BB->getParent();
8526
8527  // The lowering is pretty easy: we're just emitting the call to _alloca.  The
8528  // non-trivial part is impdef of ESP.
8529  // FIXME: The code should be tweaked as soon as we'll try to do codegen for
8530  // mingw-w64.
8531
8532  BuildMI(BB, DL, TII->get(X86::CALLpcrel32))
8533    .addExternalSymbol("_alloca")
8534    .addReg(X86::EAX, RegState::Implicit)
8535    .addReg(X86::ESP, RegState::Implicit)
8536    .addReg(X86::EAX, RegState::Define | RegState::Implicit)
8537    .addReg(X86::ESP, RegState::Define | RegState::Implicit);
8538
8539  F->DeleteMachineInstr(MI);   // The pseudo instruction is gone now.
8540  return BB;
8541}
8542
8543MachineBasicBlock *
8544X86TargetLowering::EmitLoweredTLSCall(MachineInstr *MI,
8545                                      MachineBasicBlock *BB) const {
8546  // This is pretty easy.  We're taking the value that we received from
8547  // our load from the relocation, sticking it in either RDI (x86-64)
8548  // or EAX and doing an indirect call.  The return value will then
8549  // be in the normal return register.
8550  const X86InstrInfo *TII
8551    = static_cast<const X86InstrInfo*>(getTargetMachine().getInstrInfo());
8552  DebugLoc DL = MI->getDebugLoc();
8553  MachineFunction *F = BB->getParent();
8554
8555  assert(MI->getOperand(3).isGlobal() && "This should be a global");
8556
8557  if (Subtarget->is64Bit()) {
8558    MachineInstrBuilder MIB = BuildMI(BB, DL, TII->get(X86::MOV64rm), X86::RDI)
8559    .addReg(X86::RIP)
8560    .addImm(0).addReg(0)
8561    .addGlobalAddress(MI->getOperand(3).getGlobal(), 0,
8562                      MI->getOperand(3).getTargetFlags())
8563    .addReg(0);
8564    MIB = BuildMI(BB, DL, TII->get(X86::CALL64m));
8565    addDirectMem(MIB, X86::RDI).addReg(0);
8566  } else if (getTargetMachine().getRelocationModel() != Reloc::PIC_) {
8567    MachineInstrBuilder MIB = BuildMI(BB, DL, TII->get(X86::MOV32rm), X86::EAX)
8568    .addReg(0)
8569    .addImm(0).addReg(0)
8570    .addGlobalAddress(MI->getOperand(3).getGlobal(), 0,
8571                      MI->getOperand(3).getTargetFlags())
8572    .addReg(0);
8573    MIB = BuildMI(BB, DL, TII->get(X86::CALL32m));
8574    addDirectMem(MIB, X86::EAX).addReg(0);
8575  } else {
8576    MachineInstrBuilder MIB = BuildMI(BB, DL, TII->get(X86::MOV32rm), X86::EAX)
8577    .addReg(TII->getGlobalBaseReg(F))
8578    .addImm(0).addReg(0)
8579    .addGlobalAddress(MI->getOperand(3).getGlobal(), 0,
8580                      MI->getOperand(3).getTargetFlags())
8581    .addReg(0);
8582    MIB = BuildMI(BB, DL, TII->get(X86::CALL32m));
8583    addDirectMem(MIB, X86::EAX).addReg(0);
8584  }
8585
8586  F->DeleteMachineInstr(MI); // The pseudo instruction is gone now.
8587  return BB;
8588}
8589
8590MachineBasicBlock *
8591X86TargetLowering::EmitInstrWithCustomInserter(MachineInstr *MI,
8592                                               MachineBasicBlock *BB) const {
8593  switch (MI->getOpcode()) {
8594  default: assert(false && "Unexpected instr type to insert");
8595  case X86::MINGW_ALLOCA:
8596    return EmitLoweredMingwAlloca(MI, BB);
8597  case X86::TLSCall_32:
8598  case X86::TLSCall_64:
8599    return EmitLoweredTLSCall(MI, BB);
8600  case X86::CMOV_GR8:
8601  case X86::CMOV_V1I64:
8602  case X86::CMOV_FR32:
8603  case X86::CMOV_FR64:
8604  case X86::CMOV_V4F32:
8605  case X86::CMOV_V2F64:
8606  case X86::CMOV_V2I64:
8607  case X86::CMOV_GR16:
8608  case X86::CMOV_GR32:
8609  case X86::CMOV_RFP32:
8610  case X86::CMOV_RFP64:
8611  case X86::CMOV_RFP80:
8612    return EmitLoweredSelect(MI, BB);
8613
8614  case X86::FP32_TO_INT16_IN_MEM:
8615  case X86::FP32_TO_INT32_IN_MEM:
8616  case X86::FP32_TO_INT64_IN_MEM:
8617  case X86::FP64_TO_INT16_IN_MEM:
8618  case X86::FP64_TO_INT32_IN_MEM:
8619  case X86::FP64_TO_INT64_IN_MEM:
8620  case X86::FP80_TO_INT16_IN_MEM:
8621  case X86::FP80_TO_INT32_IN_MEM:
8622  case X86::FP80_TO_INT64_IN_MEM: {
8623    const TargetInstrInfo *TII = getTargetMachine().getInstrInfo();
8624    DebugLoc DL = MI->getDebugLoc();
8625
8626    // Change the floating point control register to use "round towards zero"
8627    // mode when truncating to an integer value.
8628    MachineFunction *F = BB->getParent();
8629    int CWFrameIdx = F->getFrameInfo()->CreateStackObject(2, 2, false);
8630    addFrameReference(BuildMI(BB, DL, TII->get(X86::FNSTCW16m)), CWFrameIdx);
8631
8632    // Load the old value of the high byte of the control word...
8633    unsigned OldCW =
8634      F->getRegInfo().createVirtualRegister(X86::GR16RegisterClass);
8635    addFrameReference(BuildMI(BB, DL, TII->get(X86::MOV16rm), OldCW),
8636                      CWFrameIdx);
8637
8638    // Set the high part to be round to zero...
8639    addFrameReference(BuildMI(BB, DL, TII->get(X86::MOV16mi)), CWFrameIdx)
8640      .addImm(0xC7F);
8641
8642    // Reload the modified control word now...
8643    addFrameReference(BuildMI(BB, DL, TII->get(X86::FLDCW16m)), CWFrameIdx);
8644
8645    // Restore the memory image of control word to original value
8646    addFrameReference(BuildMI(BB, DL, TII->get(X86::MOV16mr)), CWFrameIdx)
8647      .addReg(OldCW);
8648
8649    // Get the X86 opcode to use.
8650    unsigned Opc;
8651    switch (MI->getOpcode()) {
8652    default: llvm_unreachable("illegal opcode!");
8653    case X86::FP32_TO_INT16_IN_MEM: Opc = X86::IST_Fp16m32; break;
8654    case X86::FP32_TO_INT32_IN_MEM: Opc = X86::IST_Fp32m32; break;
8655    case X86::FP32_TO_INT64_IN_MEM: Opc = X86::IST_Fp64m32; break;
8656    case X86::FP64_TO_INT16_IN_MEM: Opc = X86::IST_Fp16m64; break;
8657    case X86::FP64_TO_INT32_IN_MEM: Opc = X86::IST_Fp32m64; break;
8658    case X86::FP64_TO_INT64_IN_MEM: Opc = X86::IST_Fp64m64; break;
8659    case X86::FP80_TO_INT16_IN_MEM: Opc = X86::IST_Fp16m80; break;
8660    case X86::FP80_TO_INT32_IN_MEM: Opc = X86::IST_Fp32m80; break;
8661    case X86::FP80_TO_INT64_IN_MEM: Opc = X86::IST_Fp64m80; break;
8662    }
8663
8664    X86AddressMode AM;
8665    MachineOperand &Op = MI->getOperand(0);
8666    if (Op.isReg()) {
8667      AM.BaseType = X86AddressMode::RegBase;
8668      AM.Base.Reg = Op.getReg();
8669    } else {
8670      AM.BaseType = X86AddressMode::FrameIndexBase;
8671      AM.Base.FrameIndex = Op.getIndex();
8672    }
8673    Op = MI->getOperand(1);
8674    if (Op.isImm())
8675      AM.Scale = Op.getImm();
8676    Op = MI->getOperand(2);
8677    if (Op.isImm())
8678      AM.IndexReg = Op.getImm();
8679    Op = MI->getOperand(3);
8680    if (Op.isGlobal()) {
8681      AM.GV = Op.getGlobal();
8682    } else {
8683      AM.Disp = Op.getImm();
8684    }
8685    addFullAddress(BuildMI(BB, DL, TII->get(Opc)), AM)
8686                      .addReg(MI->getOperand(X86AddrNumOperands).getReg());
8687
8688    // Reload the original control word now.
8689    addFrameReference(BuildMI(BB, DL, TII->get(X86::FLDCW16m)), CWFrameIdx);
8690
8691    F->DeleteMachineInstr(MI);   // The pseudo instruction is gone now.
8692    return BB;
8693  }
8694    // String/text processing lowering.
8695  case X86::PCMPISTRM128REG:
8696    return EmitPCMP(MI, BB, 3, false /* in-mem */);
8697  case X86::PCMPISTRM128MEM:
8698    return EmitPCMP(MI, BB, 3, true /* in-mem */);
8699  case X86::PCMPESTRM128REG:
8700    return EmitPCMP(MI, BB, 5, false /* in mem */);
8701  case X86::PCMPESTRM128MEM:
8702    return EmitPCMP(MI, BB, 5, true /* in mem */);
8703
8704    // Atomic Lowering.
8705  case X86::ATOMAND32:
8706    return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::AND32rr,
8707                                               X86::AND32ri, X86::MOV32rm,
8708                                               X86::LCMPXCHG32, X86::MOV32rr,
8709                                               X86::NOT32r, X86::EAX,
8710                                               X86::GR32RegisterClass);
8711  case X86::ATOMOR32:
8712    return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::OR32rr,
8713                                               X86::OR32ri, X86::MOV32rm,
8714                                               X86::LCMPXCHG32, X86::MOV32rr,
8715                                               X86::NOT32r, X86::EAX,
8716                                               X86::GR32RegisterClass);
8717  case X86::ATOMXOR32:
8718    return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::XOR32rr,
8719                                               X86::XOR32ri, X86::MOV32rm,
8720                                               X86::LCMPXCHG32, X86::MOV32rr,
8721                                               X86::NOT32r, X86::EAX,
8722                                               X86::GR32RegisterClass);
8723  case X86::ATOMNAND32:
8724    return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::AND32rr,
8725                                               X86::AND32ri, X86::MOV32rm,
8726                                               X86::LCMPXCHG32, X86::MOV32rr,
8727                                               X86::NOT32r, X86::EAX,
8728                                               X86::GR32RegisterClass, true);
8729  case X86::ATOMMIN32:
8730    return EmitAtomicMinMaxWithCustomInserter(MI, BB, X86::CMOVL32rr);
8731  case X86::ATOMMAX32:
8732    return EmitAtomicMinMaxWithCustomInserter(MI, BB, X86::CMOVG32rr);
8733  case X86::ATOMUMIN32:
8734    return EmitAtomicMinMaxWithCustomInserter(MI, BB, X86::CMOVB32rr);
8735  case X86::ATOMUMAX32:
8736    return EmitAtomicMinMaxWithCustomInserter(MI, BB, X86::CMOVA32rr);
8737
8738  case X86::ATOMAND16:
8739    return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::AND16rr,
8740                                               X86::AND16ri, X86::MOV16rm,
8741                                               X86::LCMPXCHG16, X86::MOV16rr,
8742                                               X86::NOT16r, X86::AX,
8743                                               X86::GR16RegisterClass);
8744  case X86::ATOMOR16:
8745    return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::OR16rr,
8746                                               X86::OR16ri, X86::MOV16rm,
8747                                               X86::LCMPXCHG16, X86::MOV16rr,
8748                                               X86::NOT16r, X86::AX,
8749                                               X86::GR16RegisterClass);
8750  case X86::ATOMXOR16:
8751    return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::XOR16rr,
8752                                               X86::XOR16ri, X86::MOV16rm,
8753                                               X86::LCMPXCHG16, X86::MOV16rr,
8754                                               X86::NOT16r, X86::AX,
8755                                               X86::GR16RegisterClass);
8756  case X86::ATOMNAND16:
8757    return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::AND16rr,
8758                                               X86::AND16ri, X86::MOV16rm,
8759                                               X86::LCMPXCHG16, X86::MOV16rr,
8760                                               X86::NOT16r, X86::AX,
8761                                               X86::GR16RegisterClass, true);
8762  case X86::ATOMMIN16:
8763    return EmitAtomicMinMaxWithCustomInserter(MI, BB, X86::CMOVL16rr);
8764  case X86::ATOMMAX16:
8765    return EmitAtomicMinMaxWithCustomInserter(MI, BB, X86::CMOVG16rr);
8766  case X86::ATOMUMIN16:
8767    return EmitAtomicMinMaxWithCustomInserter(MI, BB, X86::CMOVB16rr);
8768  case X86::ATOMUMAX16:
8769    return EmitAtomicMinMaxWithCustomInserter(MI, BB, X86::CMOVA16rr);
8770
8771  case X86::ATOMAND8:
8772    return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::AND8rr,
8773                                               X86::AND8ri, X86::MOV8rm,
8774                                               X86::LCMPXCHG8, X86::MOV8rr,
8775                                               X86::NOT8r, X86::AL,
8776                                               X86::GR8RegisterClass);
8777  case X86::ATOMOR8:
8778    return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::OR8rr,
8779                                               X86::OR8ri, X86::MOV8rm,
8780                                               X86::LCMPXCHG8, X86::MOV8rr,
8781                                               X86::NOT8r, X86::AL,
8782                                               X86::GR8RegisterClass);
8783  case X86::ATOMXOR8:
8784    return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::XOR8rr,
8785                                               X86::XOR8ri, X86::MOV8rm,
8786                                               X86::LCMPXCHG8, X86::MOV8rr,
8787                                               X86::NOT8r, X86::AL,
8788                                               X86::GR8RegisterClass);
8789  case X86::ATOMNAND8:
8790    return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::AND8rr,
8791                                               X86::AND8ri, X86::MOV8rm,
8792                                               X86::LCMPXCHG8, X86::MOV8rr,
8793                                               X86::NOT8r, X86::AL,
8794                                               X86::GR8RegisterClass, true);
8795  // FIXME: There are no CMOV8 instructions; MIN/MAX need some other way.
8796  // This group is for 64-bit host.
8797  case X86::ATOMAND64:
8798    return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::AND64rr,
8799                                               X86::AND64ri32, X86::MOV64rm,
8800                                               X86::LCMPXCHG64, X86::MOV64rr,
8801                                               X86::NOT64r, X86::RAX,
8802                                               X86::GR64RegisterClass);
8803  case X86::ATOMOR64:
8804    return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::OR64rr,
8805                                               X86::OR64ri32, X86::MOV64rm,
8806                                               X86::LCMPXCHG64, X86::MOV64rr,
8807                                               X86::NOT64r, X86::RAX,
8808                                               X86::GR64RegisterClass);
8809  case X86::ATOMXOR64:
8810    return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::XOR64rr,
8811                                               X86::XOR64ri32, X86::MOV64rm,
8812                                               X86::LCMPXCHG64, X86::MOV64rr,
8813                                               X86::NOT64r, X86::RAX,
8814                                               X86::GR64RegisterClass);
8815  case X86::ATOMNAND64:
8816    return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::AND64rr,
8817                                               X86::AND64ri32, X86::MOV64rm,
8818                                               X86::LCMPXCHG64, X86::MOV64rr,
8819                                               X86::NOT64r, X86::RAX,
8820                                               X86::GR64RegisterClass, true);
8821  case X86::ATOMMIN64:
8822    return EmitAtomicMinMaxWithCustomInserter(MI, BB, X86::CMOVL64rr);
8823  case X86::ATOMMAX64:
8824    return EmitAtomicMinMaxWithCustomInserter(MI, BB, X86::CMOVG64rr);
8825  case X86::ATOMUMIN64:
8826    return EmitAtomicMinMaxWithCustomInserter(MI, BB, X86::CMOVB64rr);
8827  case X86::ATOMUMAX64:
8828    return EmitAtomicMinMaxWithCustomInserter(MI, BB, X86::CMOVA64rr);
8829
8830  // This group does 64-bit operations on a 32-bit host.
8831  case X86::ATOMAND6432:
8832    return EmitAtomicBit6432WithCustomInserter(MI, BB,
8833                                               X86::AND32rr, X86::AND32rr,
8834                                               X86::AND32ri, X86::AND32ri,
8835                                               false);
8836  case X86::ATOMOR6432:
8837    return EmitAtomicBit6432WithCustomInserter(MI, BB,
8838                                               X86::OR32rr, X86::OR32rr,
8839                                               X86::OR32ri, X86::OR32ri,
8840                                               false);
8841  case X86::ATOMXOR6432:
8842    return EmitAtomicBit6432WithCustomInserter(MI, BB,
8843                                               X86::XOR32rr, X86::XOR32rr,
8844                                               X86::XOR32ri, X86::XOR32ri,
8845                                               false);
8846  case X86::ATOMNAND6432:
8847    return EmitAtomicBit6432WithCustomInserter(MI, BB,
8848                                               X86::AND32rr, X86::AND32rr,
8849                                               X86::AND32ri, X86::AND32ri,
8850                                               true);
8851  case X86::ATOMADD6432:
8852    return EmitAtomicBit6432WithCustomInserter(MI, BB,
8853                                               X86::ADD32rr, X86::ADC32rr,
8854                                               X86::ADD32ri, X86::ADC32ri,
8855                                               false);
8856  case X86::ATOMSUB6432:
8857    return EmitAtomicBit6432WithCustomInserter(MI, BB,
8858                                               X86::SUB32rr, X86::SBB32rr,
8859                                               X86::SUB32ri, X86::SBB32ri,
8860                                               false);
8861  case X86::ATOMSWAP6432:
8862    return EmitAtomicBit6432WithCustomInserter(MI, BB,
8863                                               X86::MOV32rr, X86::MOV32rr,
8864                                               X86::MOV32ri, X86::MOV32ri,
8865                                               false);
8866  case X86::VASTART_SAVE_XMM_REGS:
8867    return EmitVAStartSaveXMMRegsWithCustomInserter(MI, BB);
8868  }
8869}
8870
8871//===----------------------------------------------------------------------===//
8872//                           X86 Optimization Hooks
8873//===----------------------------------------------------------------------===//
8874
8875void X86TargetLowering::computeMaskedBitsForTargetNode(const SDValue Op,
8876                                                       const APInt &Mask,
8877                                                       APInt &KnownZero,
8878                                                       APInt &KnownOne,
8879                                                       const SelectionDAG &DAG,
8880                                                       unsigned Depth) const {
8881  unsigned Opc = Op.getOpcode();
8882  assert((Opc >= ISD::BUILTIN_OP_END ||
8883          Opc == ISD::INTRINSIC_WO_CHAIN ||
8884          Opc == ISD::INTRINSIC_W_CHAIN ||
8885          Opc == ISD::INTRINSIC_VOID) &&
8886         "Should use MaskedValueIsZero if you don't know whether Op"
8887         " is a target node!");
8888
8889  KnownZero = KnownOne = APInt(Mask.getBitWidth(), 0);   // Don't know anything.
8890  switch (Opc) {
8891  default: break;
8892  case X86ISD::ADD:
8893  case X86ISD::SUB:
8894  case X86ISD::SMUL:
8895  case X86ISD::UMUL:
8896  case X86ISD::INC:
8897  case X86ISD::DEC:
8898  case X86ISD::OR:
8899  case X86ISD::XOR:
8900  case X86ISD::AND:
8901    // These nodes' second result is a boolean.
8902    if (Op.getResNo() == 0)
8903      break;
8904    // Fallthrough
8905  case X86ISD::SETCC:
8906    KnownZero |= APInt::getHighBitsSet(Mask.getBitWidth(),
8907                                       Mask.getBitWidth() - 1);
8908    break;
8909  }
8910}
8911
8912/// isGAPlusOffset - Returns true (and the GlobalValue and the offset) if the
8913/// node is a GlobalAddress + offset.
8914bool X86TargetLowering::isGAPlusOffset(SDNode *N,
8915                                       const GlobalValue* &GA,
8916                                       int64_t &Offset) const {
8917  if (N->getOpcode() == X86ISD::Wrapper) {
8918    if (isa<GlobalAddressSDNode>(N->getOperand(0))) {
8919      GA = cast<GlobalAddressSDNode>(N->getOperand(0))->getGlobal();
8920      Offset = cast<GlobalAddressSDNode>(N->getOperand(0))->getOffset();
8921      return true;
8922    }
8923  }
8924  return TargetLowering::isGAPlusOffset(N, GA, Offset);
8925}
8926
8927/// PerformShuffleCombine - Combine a vector_shuffle that is equal to
8928/// build_vector load1, load2, load3, load4, <0, 1, 2, 3> into a 128-bit load
8929/// if the load addresses are consecutive, non-overlapping, and in the right
8930/// order.
8931static SDValue PerformShuffleCombine(SDNode *N, SelectionDAG &DAG,
8932                                     const TargetLowering &TLI) {
8933  DebugLoc dl = N->getDebugLoc();
8934  EVT VT = N->getValueType(0);
8935  ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(N);
8936
8937  if (VT.getSizeInBits() != 128)
8938    return SDValue();
8939
8940  SmallVector<SDValue, 16> Elts;
8941  for (unsigned i = 0, e = VT.getVectorNumElements(); i != e; ++i)
8942    Elts.push_back(DAG.getShuffleScalarElt(SVN, i));
8943
8944  return EltsFromConsecutiveLoads(VT, Elts, dl, DAG);
8945}
8946
8947/// PerformShuffleCombine - Detect vector gather/scatter index generation
8948/// and convert it from being a bunch of shuffles and extracts to a simple
8949/// store and scalar loads to extract the elements.
8950static SDValue PerformEXTRACT_VECTOR_ELTCombine(SDNode *N, SelectionDAG &DAG,
8951                                                const TargetLowering &TLI) {
8952  SDValue InputVector = N->getOperand(0);
8953
8954  // Only operate on vectors of 4 elements, where the alternative shuffling
8955  // gets to be more expensive.
8956  if (InputVector.getValueType() != MVT::v4i32)
8957    return SDValue();
8958
8959  // Check whether every use of InputVector is an EXTRACT_VECTOR_ELT with a
8960  // single use which is a sign-extend or zero-extend, and all elements are
8961  // used.
8962  SmallVector<SDNode *, 4> Uses;
8963  unsigned ExtractedElements = 0;
8964  for (SDNode::use_iterator UI = InputVector.getNode()->use_begin(),
8965       UE = InputVector.getNode()->use_end(); UI != UE; ++UI) {
8966    if (UI.getUse().getResNo() != InputVector.getResNo())
8967      return SDValue();
8968
8969    SDNode *Extract = *UI;
8970    if (Extract->getOpcode() != ISD::EXTRACT_VECTOR_ELT)
8971      return SDValue();
8972
8973    if (Extract->getValueType(0) != MVT::i32)
8974      return SDValue();
8975    if (!Extract->hasOneUse())
8976      return SDValue();
8977    if (Extract->use_begin()->getOpcode() != ISD::SIGN_EXTEND &&
8978        Extract->use_begin()->getOpcode() != ISD::ZERO_EXTEND)
8979      return SDValue();
8980    if (!isa<ConstantSDNode>(Extract->getOperand(1)))
8981      return SDValue();
8982
8983    // Record which element was extracted.
8984    ExtractedElements |=
8985      1 << cast<ConstantSDNode>(Extract->getOperand(1))->getZExtValue();
8986
8987    Uses.push_back(Extract);
8988  }
8989
8990  // If not all the elements were used, this may not be worthwhile.
8991  if (ExtractedElements != 15)
8992    return SDValue();
8993
8994  // Ok, we've now decided to do the transformation.
8995  DebugLoc dl = InputVector.getDebugLoc();
8996
8997  // Store the value to a temporary stack slot.
8998  SDValue StackPtr = DAG.CreateStackTemporary(InputVector.getValueType());
8999  SDValue Ch = DAG.getStore(DAG.getEntryNode(), dl, InputVector, StackPtr, NULL, 0,
9000                            false, false, 0);
9001
9002  // Replace each use (extract) with a load of the appropriate element.
9003  for (SmallVectorImpl<SDNode *>::iterator UI = Uses.begin(),
9004       UE = Uses.end(); UI != UE; ++UI) {
9005    SDNode *Extract = *UI;
9006
9007    // Compute the element's address.
9008    SDValue Idx = Extract->getOperand(1);
9009    unsigned EltSize =
9010        InputVector.getValueType().getVectorElementType().getSizeInBits()/8;
9011    uint64_t Offset = EltSize * cast<ConstantSDNode>(Idx)->getZExtValue();
9012    SDValue OffsetVal = DAG.getConstant(Offset, TLI.getPointerTy());
9013
9014    SDValue ScalarAddr = DAG.getNode(ISD::ADD, dl, Idx.getValueType(), OffsetVal, StackPtr);
9015
9016    // Load the scalar.
9017    SDValue LoadScalar = DAG.getLoad(Extract->getValueType(0), dl, Ch, ScalarAddr,
9018                          NULL, 0, false, false, 0);
9019
9020    // Replace the exact with the load.
9021    DAG.ReplaceAllUsesOfValueWith(SDValue(Extract, 0), LoadScalar);
9022  }
9023
9024  // The replacement was made in place; don't return anything.
9025  return SDValue();
9026}
9027
9028/// PerformSELECTCombine - Do target-specific dag combines on SELECT nodes.
9029static SDValue PerformSELECTCombine(SDNode *N, SelectionDAG &DAG,
9030                                    const X86Subtarget *Subtarget) {
9031  DebugLoc DL = N->getDebugLoc();
9032  SDValue Cond = N->getOperand(0);
9033  // Get the LHS/RHS of the select.
9034  SDValue LHS = N->getOperand(1);
9035  SDValue RHS = N->getOperand(2);
9036
9037  // If we have SSE[12] support, try to form min/max nodes. SSE min/max
9038  // instructions match the semantics of the common C idiom x<y?x:y but not
9039  // x<=y?x:y, because of how they handle negative zero (which can be
9040  // ignored in unsafe-math mode).
9041  if (Subtarget->hasSSE2() &&
9042      (LHS.getValueType() == MVT::f32 || LHS.getValueType() == MVT::f64) &&
9043      Cond.getOpcode() == ISD::SETCC) {
9044    ISD::CondCode CC = cast<CondCodeSDNode>(Cond.getOperand(2))->get();
9045
9046    unsigned Opcode = 0;
9047    // Check for x CC y ? x : y.
9048    if (DAG.isEqualTo(LHS, Cond.getOperand(0)) &&
9049        DAG.isEqualTo(RHS, Cond.getOperand(1))) {
9050      switch (CC) {
9051      default: break;
9052      case ISD::SETULT:
9053        // Converting this to a min would handle NaNs incorrectly, and swapping
9054        // the operands would cause it to handle comparisons between positive
9055        // and negative zero incorrectly.
9056        if (!FiniteOnlyFPMath() &&
9057            (!DAG.isKnownNeverNaN(LHS) || !DAG.isKnownNeverNaN(RHS))) {
9058          if (!UnsafeFPMath &&
9059              !(DAG.isKnownNeverZero(LHS) || DAG.isKnownNeverZero(RHS)))
9060            break;
9061          std::swap(LHS, RHS);
9062        }
9063        Opcode = X86ISD::FMIN;
9064        break;
9065      case ISD::SETOLE:
9066        // Converting this to a min would handle comparisons between positive
9067        // and negative zero incorrectly.
9068        if (!UnsafeFPMath &&
9069            !DAG.isKnownNeverZero(LHS) && !DAG.isKnownNeverZero(RHS))
9070          break;
9071        Opcode = X86ISD::FMIN;
9072        break;
9073      case ISD::SETULE:
9074        // Converting this to a min would handle both negative zeros and NaNs
9075        // incorrectly, but we can swap the operands to fix both.
9076        std::swap(LHS, RHS);
9077      case ISD::SETOLT:
9078      case ISD::SETLT:
9079      case ISD::SETLE:
9080        Opcode = X86ISD::FMIN;
9081        break;
9082
9083      case ISD::SETOGE:
9084        // Converting this to a max would handle comparisons between positive
9085        // and negative zero incorrectly.
9086        if (!UnsafeFPMath &&
9087            !DAG.isKnownNeverZero(LHS) && !DAG.isKnownNeverZero(LHS))
9088          break;
9089        Opcode = X86ISD::FMAX;
9090        break;
9091      case ISD::SETUGT:
9092        // Converting this to a max would handle NaNs incorrectly, and swapping
9093        // the operands would cause it to handle comparisons between positive
9094        // and negative zero incorrectly.
9095        if (!FiniteOnlyFPMath() &&
9096            (!DAG.isKnownNeverNaN(LHS) || !DAG.isKnownNeverNaN(RHS))) {
9097          if (!UnsafeFPMath &&
9098              !(DAG.isKnownNeverZero(LHS) || DAG.isKnownNeverZero(RHS)))
9099            break;
9100          std::swap(LHS, RHS);
9101        }
9102        Opcode = X86ISD::FMAX;
9103        break;
9104      case ISD::SETUGE:
9105        // Converting this to a max would handle both negative zeros and NaNs
9106        // incorrectly, but we can swap the operands to fix both.
9107        std::swap(LHS, RHS);
9108      case ISD::SETOGT:
9109      case ISD::SETGT:
9110      case ISD::SETGE:
9111        Opcode = X86ISD::FMAX;
9112        break;
9113      }
9114    // Check for x CC y ? y : x -- a min/max with reversed arms.
9115    } else if (DAG.isEqualTo(LHS, Cond.getOperand(1)) &&
9116               DAG.isEqualTo(RHS, Cond.getOperand(0))) {
9117      switch (CC) {
9118      default: break;
9119      case ISD::SETOGE:
9120        // Converting this to a min would handle comparisons between positive
9121        // and negative zero incorrectly, and swapping the operands would
9122        // cause it to handle NaNs incorrectly.
9123        if (!UnsafeFPMath &&
9124            !(DAG.isKnownNeverZero(LHS) || DAG.isKnownNeverZero(RHS))) {
9125          if (!FiniteOnlyFPMath() &&
9126              (!DAG.isKnownNeverNaN(LHS) || !DAG.isKnownNeverNaN(RHS)))
9127            break;
9128          std::swap(LHS, RHS);
9129        }
9130        Opcode = X86ISD::FMIN;
9131        break;
9132      case ISD::SETUGT:
9133        // Converting this to a min would handle NaNs incorrectly.
9134        if (!UnsafeFPMath &&
9135            (!DAG.isKnownNeverNaN(LHS) || !DAG.isKnownNeverNaN(RHS)))
9136          break;
9137        Opcode = X86ISD::FMIN;
9138        break;
9139      case ISD::SETUGE:
9140        // Converting this to a min would handle both negative zeros and NaNs
9141        // incorrectly, but we can swap the operands to fix both.
9142        std::swap(LHS, RHS);
9143      case ISD::SETOGT:
9144      case ISD::SETGT:
9145      case ISD::SETGE:
9146        Opcode = X86ISD::FMIN;
9147        break;
9148
9149      case ISD::SETULT:
9150        // Converting this to a max would handle NaNs incorrectly.
9151        if (!FiniteOnlyFPMath() &&
9152            (!DAG.isKnownNeverNaN(LHS) || !DAG.isKnownNeverNaN(RHS)))
9153          break;
9154        Opcode = X86ISD::FMAX;
9155        break;
9156      case ISD::SETOLE:
9157        // Converting this to a max would handle comparisons between positive
9158        // and negative zero incorrectly, and swapping the operands would
9159        // cause it to handle NaNs incorrectly.
9160        if (!UnsafeFPMath &&
9161            !DAG.isKnownNeverZero(LHS) && !DAG.isKnownNeverZero(RHS)) {
9162          if (!FiniteOnlyFPMath() &&
9163              (!DAG.isKnownNeverNaN(LHS) || !DAG.isKnownNeverNaN(RHS)))
9164            break;
9165          std::swap(LHS, RHS);
9166        }
9167        Opcode = X86ISD::FMAX;
9168        break;
9169      case ISD::SETULE:
9170        // Converting this to a max would handle both negative zeros and NaNs
9171        // incorrectly, but we can swap the operands to fix both.
9172        std::swap(LHS, RHS);
9173      case ISD::SETOLT:
9174      case ISD::SETLT:
9175      case ISD::SETLE:
9176        Opcode = X86ISD::FMAX;
9177        break;
9178      }
9179    }
9180
9181    if (Opcode)
9182      return DAG.getNode(Opcode, DL, N->getValueType(0), LHS, RHS);
9183  }
9184
9185  // If this is a select between two integer constants, try to do some
9186  // optimizations.
9187  if (ConstantSDNode *TrueC = dyn_cast<ConstantSDNode>(LHS)) {
9188    if (ConstantSDNode *FalseC = dyn_cast<ConstantSDNode>(RHS))
9189      // Don't do this for crazy integer types.
9190      if (DAG.getTargetLoweringInfo().isTypeLegal(LHS.getValueType())) {
9191        // If this is efficiently invertible, canonicalize the LHSC/RHSC values
9192        // so that TrueC (the true value) is larger than FalseC.
9193        bool NeedsCondInvert = false;
9194
9195        if (TrueC->getAPIntValue().ult(FalseC->getAPIntValue()) &&
9196            // Efficiently invertible.
9197            (Cond.getOpcode() == ISD::SETCC ||  // setcc -> invertible.
9198             (Cond.getOpcode() == ISD::XOR &&   // xor(X, C) -> invertible.
9199              isa<ConstantSDNode>(Cond.getOperand(1))))) {
9200          NeedsCondInvert = true;
9201          std::swap(TrueC, FalseC);
9202        }
9203
9204        // Optimize C ? 8 : 0 -> zext(C) << 3.  Likewise for any pow2/0.
9205        if (FalseC->getAPIntValue() == 0 &&
9206            TrueC->getAPIntValue().isPowerOf2()) {
9207          if (NeedsCondInvert) // Invert the condition if needed.
9208            Cond = DAG.getNode(ISD::XOR, DL, Cond.getValueType(), Cond,
9209                               DAG.getConstant(1, Cond.getValueType()));
9210
9211          // Zero extend the condition if needed.
9212          Cond = DAG.getNode(ISD::ZERO_EXTEND, DL, LHS.getValueType(), Cond);
9213
9214          unsigned ShAmt = TrueC->getAPIntValue().logBase2();
9215          return DAG.getNode(ISD::SHL, DL, LHS.getValueType(), Cond,
9216                             DAG.getConstant(ShAmt, MVT::i8));
9217        }
9218
9219        // Optimize Cond ? cst+1 : cst -> zext(setcc(C)+cst.
9220        if (FalseC->getAPIntValue()+1 == TrueC->getAPIntValue()) {
9221          if (NeedsCondInvert) // Invert the condition if needed.
9222            Cond = DAG.getNode(ISD::XOR, DL, Cond.getValueType(), Cond,
9223                               DAG.getConstant(1, Cond.getValueType()));
9224
9225          // Zero extend the condition if needed.
9226          Cond = DAG.getNode(ISD::ZERO_EXTEND, DL,
9227                             FalseC->getValueType(0), Cond);
9228          return DAG.getNode(ISD::ADD, DL, Cond.getValueType(), Cond,
9229                             SDValue(FalseC, 0));
9230        }
9231
9232        // Optimize cases that will turn into an LEA instruction.  This requires
9233        // an i32 or i64 and an efficient multiplier (1, 2, 3, 4, 5, 8, 9).
9234        if (N->getValueType(0) == MVT::i32 || N->getValueType(0) == MVT::i64) {
9235          uint64_t Diff = TrueC->getZExtValue()-FalseC->getZExtValue();
9236          if (N->getValueType(0) == MVT::i32) Diff = (unsigned)Diff;
9237
9238          bool isFastMultiplier = false;
9239          if (Diff < 10) {
9240            switch ((unsigned char)Diff) {
9241              default: break;
9242              case 1:  // result = add base, cond
9243              case 2:  // result = lea base(    , cond*2)
9244              case 3:  // result = lea base(cond, cond*2)
9245              case 4:  // result = lea base(    , cond*4)
9246              case 5:  // result = lea base(cond, cond*4)
9247              case 8:  // result = lea base(    , cond*8)
9248              case 9:  // result = lea base(cond, cond*8)
9249                isFastMultiplier = true;
9250                break;
9251            }
9252          }
9253
9254          if (isFastMultiplier) {
9255            APInt Diff = TrueC->getAPIntValue()-FalseC->getAPIntValue();
9256            if (NeedsCondInvert) // Invert the condition if needed.
9257              Cond = DAG.getNode(ISD::XOR, DL, Cond.getValueType(), Cond,
9258                                 DAG.getConstant(1, Cond.getValueType()));
9259
9260            // Zero extend the condition if needed.
9261            Cond = DAG.getNode(ISD::ZERO_EXTEND, DL, FalseC->getValueType(0),
9262                               Cond);
9263            // Scale the condition by the difference.
9264            if (Diff != 1)
9265              Cond = DAG.getNode(ISD::MUL, DL, Cond.getValueType(), Cond,
9266                                 DAG.getConstant(Diff, Cond.getValueType()));
9267
9268            // Add the base if non-zero.
9269            if (FalseC->getAPIntValue() != 0)
9270              Cond = DAG.getNode(ISD::ADD, DL, Cond.getValueType(), Cond,
9271                                 SDValue(FalseC, 0));
9272            return Cond;
9273          }
9274        }
9275      }
9276  }
9277
9278  return SDValue();
9279}
9280
9281/// Optimize X86ISD::CMOV [LHS, RHS, CONDCODE (e.g. X86::COND_NE), CONDVAL]
9282static SDValue PerformCMOVCombine(SDNode *N, SelectionDAG &DAG,
9283                                  TargetLowering::DAGCombinerInfo &DCI) {
9284  DebugLoc DL = N->getDebugLoc();
9285
9286  // If the flag operand isn't dead, don't touch this CMOV.
9287  if (N->getNumValues() == 2 && !SDValue(N, 1).use_empty())
9288    return SDValue();
9289
9290  // If this is a select between two integer constants, try to do some
9291  // optimizations.  Note that the operands are ordered the opposite of SELECT
9292  // operands.
9293  if (ConstantSDNode *TrueC = dyn_cast<ConstantSDNode>(N->getOperand(1))) {
9294    if (ConstantSDNode *FalseC = dyn_cast<ConstantSDNode>(N->getOperand(0))) {
9295      // Canonicalize the TrueC/FalseC values so that TrueC (the true value) is
9296      // larger than FalseC (the false value).
9297      X86::CondCode CC = (X86::CondCode)N->getConstantOperandVal(2);
9298
9299      if (TrueC->getAPIntValue().ult(FalseC->getAPIntValue())) {
9300        CC = X86::GetOppositeBranchCondition(CC);
9301        std::swap(TrueC, FalseC);
9302      }
9303
9304      // Optimize C ? 8 : 0 -> zext(setcc(C)) << 3.  Likewise for any pow2/0.
9305      // This is efficient for any integer data type (including i8/i16) and
9306      // shift amount.
9307      if (FalseC->getAPIntValue() == 0 && TrueC->getAPIntValue().isPowerOf2()) {
9308        SDValue Cond = N->getOperand(3);
9309        Cond = DAG.getNode(X86ISD::SETCC, DL, MVT::i8,
9310                           DAG.getConstant(CC, MVT::i8), Cond);
9311
9312        // Zero extend the condition if needed.
9313        Cond = DAG.getNode(ISD::ZERO_EXTEND, DL, TrueC->getValueType(0), Cond);
9314
9315        unsigned ShAmt = TrueC->getAPIntValue().logBase2();
9316        Cond = DAG.getNode(ISD::SHL, DL, Cond.getValueType(), Cond,
9317                           DAG.getConstant(ShAmt, MVT::i8));
9318        if (N->getNumValues() == 2)  // Dead flag value?
9319          return DCI.CombineTo(N, Cond, SDValue());
9320        return Cond;
9321      }
9322
9323      // Optimize Cond ? cst+1 : cst -> zext(setcc(C)+cst.  This is efficient
9324      // for any integer data type, including i8/i16.
9325      if (FalseC->getAPIntValue()+1 == TrueC->getAPIntValue()) {
9326        SDValue Cond = N->getOperand(3);
9327        Cond = DAG.getNode(X86ISD::SETCC, DL, MVT::i8,
9328                           DAG.getConstant(CC, MVT::i8), Cond);
9329
9330        // Zero extend the condition if needed.
9331        Cond = DAG.getNode(ISD::ZERO_EXTEND, DL,
9332                           FalseC->getValueType(0), Cond);
9333        Cond = DAG.getNode(ISD::ADD, DL, Cond.getValueType(), Cond,
9334                           SDValue(FalseC, 0));
9335
9336        if (N->getNumValues() == 2)  // Dead flag value?
9337          return DCI.CombineTo(N, Cond, SDValue());
9338        return Cond;
9339      }
9340
9341      // Optimize cases that will turn into an LEA instruction.  This requires
9342      // an i32 or i64 and an efficient multiplier (1, 2, 3, 4, 5, 8, 9).
9343      if (N->getValueType(0) == MVT::i32 || N->getValueType(0) == MVT::i64) {
9344        uint64_t Diff = TrueC->getZExtValue()-FalseC->getZExtValue();
9345        if (N->getValueType(0) == MVT::i32) Diff = (unsigned)Diff;
9346
9347        bool isFastMultiplier = false;
9348        if (Diff < 10) {
9349          switch ((unsigned char)Diff) {
9350          default: break;
9351          case 1:  // result = add base, cond
9352          case 2:  // result = lea base(    , cond*2)
9353          case 3:  // result = lea base(cond, cond*2)
9354          case 4:  // result = lea base(    , cond*4)
9355          case 5:  // result = lea base(cond, cond*4)
9356          case 8:  // result = lea base(    , cond*8)
9357          case 9:  // result = lea base(cond, cond*8)
9358            isFastMultiplier = true;
9359            break;
9360          }
9361        }
9362
9363        if (isFastMultiplier) {
9364          APInt Diff = TrueC->getAPIntValue()-FalseC->getAPIntValue();
9365          SDValue Cond = N->getOperand(3);
9366          Cond = DAG.getNode(X86ISD::SETCC, DL, MVT::i8,
9367                             DAG.getConstant(CC, MVT::i8), Cond);
9368          // Zero extend the condition if needed.
9369          Cond = DAG.getNode(ISD::ZERO_EXTEND, DL, FalseC->getValueType(0),
9370                             Cond);
9371          // Scale the condition by the difference.
9372          if (Diff != 1)
9373            Cond = DAG.getNode(ISD::MUL, DL, Cond.getValueType(), Cond,
9374                               DAG.getConstant(Diff, Cond.getValueType()));
9375
9376          // Add the base if non-zero.
9377          if (FalseC->getAPIntValue() != 0)
9378            Cond = DAG.getNode(ISD::ADD, DL, Cond.getValueType(), Cond,
9379                               SDValue(FalseC, 0));
9380          if (N->getNumValues() == 2)  // Dead flag value?
9381            return DCI.CombineTo(N, Cond, SDValue());
9382          return Cond;
9383        }
9384      }
9385    }
9386  }
9387  return SDValue();
9388}
9389
9390
9391/// PerformMulCombine - Optimize a single multiply with constant into two
9392/// in order to implement it with two cheaper instructions, e.g.
9393/// LEA + SHL, LEA + LEA.
9394static SDValue PerformMulCombine(SDNode *N, SelectionDAG &DAG,
9395                                 TargetLowering::DAGCombinerInfo &DCI) {
9396  if (DCI.isBeforeLegalize() || DCI.isCalledByLegalizer())
9397    return SDValue();
9398
9399  EVT VT = N->getValueType(0);
9400  if (VT != MVT::i64)
9401    return SDValue();
9402
9403  ConstantSDNode *C = dyn_cast<ConstantSDNode>(N->getOperand(1));
9404  if (!C)
9405    return SDValue();
9406  uint64_t MulAmt = C->getZExtValue();
9407  if (isPowerOf2_64(MulAmt) || MulAmt == 3 || MulAmt == 5 || MulAmt == 9)
9408    return SDValue();
9409
9410  uint64_t MulAmt1 = 0;
9411  uint64_t MulAmt2 = 0;
9412  if ((MulAmt % 9) == 0) {
9413    MulAmt1 = 9;
9414    MulAmt2 = MulAmt / 9;
9415  } else if ((MulAmt % 5) == 0) {
9416    MulAmt1 = 5;
9417    MulAmt2 = MulAmt / 5;
9418  } else if ((MulAmt % 3) == 0) {
9419    MulAmt1 = 3;
9420    MulAmt2 = MulAmt / 3;
9421  }
9422  if (MulAmt2 &&
9423      (isPowerOf2_64(MulAmt2) || MulAmt2 == 3 || MulAmt2 == 5 || MulAmt2 == 9)){
9424    DebugLoc DL = N->getDebugLoc();
9425
9426    if (isPowerOf2_64(MulAmt2) &&
9427        !(N->hasOneUse() && N->use_begin()->getOpcode() == ISD::ADD))
9428      // If second multiplifer is pow2, issue it first. We want the multiply by
9429      // 3, 5, or 9 to be folded into the addressing mode unless the lone use
9430      // is an add.
9431      std::swap(MulAmt1, MulAmt2);
9432
9433    SDValue NewMul;
9434    if (isPowerOf2_64(MulAmt1))
9435      NewMul = DAG.getNode(ISD::SHL, DL, VT, N->getOperand(0),
9436                           DAG.getConstant(Log2_64(MulAmt1), MVT::i8));
9437    else
9438      NewMul = DAG.getNode(X86ISD::MUL_IMM, DL, VT, N->getOperand(0),
9439                           DAG.getConstant(MulAmt1, VT));
9440
9441    if (isPowerOf2_64(MulAmt2))
9442      NewMul = DAG.getNode(ISD::SHL, DL, VT, NewMul,
9443                           DAG.getConstant(Log2_64(MulAmt2), MVT::i8));
9444    else
9445      NewMul = DAG.getNode(X86ISD::MUL_IMM, DL, VT, NewMul,
9446                           DAG.getConstant(MulAmt2, VT));
9447
9448    // Do not add new nodes to DAG combiner worklist.
9449    DCI.CombineTo(N, NewMul, false);
9450  }
9451  return SDValue();
9452}
9453
9454static SDValue PerformSHLCombine(SDNode *N, SelectionDAG &DAG) {
9455  SDValue N0 = N->getOperand(0);
9456  SDValue N1 = N->getOperand(1);
9457  ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
9458  EVT VT = N0.getValueType();
9459
9460  // fold (shl (and (setcc_c), c1), c2) -> (and setcc_c, (c1 << c2))
9461  // since the result of setcc_c is all zero's or all ones.
9462  if (N1C && N0.getOpcode() == ISD::AND &&
9463      N0.getOperand(1).getOpcode() == ISD::Constant) {
9464    SDValue N00 = N0.getOperand(0);
9465    if (N00.getOpcode() == X86ISD::SETCC_CARRY ||
9466        ((N00.getOpcode() == ISD::ANY_EXTEND ||
9467          N00.getOpcode() == ISD::ZERO_EXTEND) &&
9468         N00.getOperand(0).getOpcode() == X86ISD::SETCC_CARRY)) {
9469      APInt Mask = cast<ConstantSDNode>(N0.getOperand(1))->getAPIntValue();
9470      APInt ShAmt = N1C->getAPIntValue();
9471      Mask = Mask.shl(ShAmt);
9472      if (Mask != 0)
9473        return DAG.getNode(ISD::AND, N->getDebugLoc(), VT,
9474                           N00, DAG.getConstant(Mask, VT));
9475    }
9476  }
9477
9478  return SDValue();
9479}
9480
9481/// PerformShiftCombine - Transforms vector shift nodes to use vector shifts
9482///                       when possible.
9483static SDValue PerformShiftCombine(SDNode* N, SelectionDAG &DAG,
9484                                   const X86Subtarget *Subtarget) {
9485  EVT VT = N->getValueType(0);
9486  if (!VT.isVector() && VT.isInteger() &&
9487      N->getOpcode() == ISD::SHL)
9488    return PerformSHLCombine(N, DAG);
9489
9490  // On X86 with SSE2 support, we can transform this to a vector shift if
9491  // all elements are shifted by the same amount.  We can't do this in legalize
9492  // because the a constant vector is typically transformed to a constant pool
9493  // so we have no knowledge of the shift amount.
9494  if (!Subtarget->hasSSE2())
9495    return SDValue();
9496
9497  if (VT != MVT::v2i64 && VT != MVT::v4i32 && VT != MVT::v8i16)
9498    return SDValue();
9499
9500  SDValue ShAmtOp = N->getOperand(1);
9501  EVT EltVT = VT.getVectorElementType();
9502  DebugLoc DL = N->getDebugLoc();
9503  SDValue BaseShAmt = SDValue();
9504  if (ShAmtOp.getOpcode() == ISD::BUILD_VECTOR) {
9505    unsigned NumElts = VT.getVectorNumElements();
9506    unsigned i = 0;
9507    for (; i != NumElts; ++i) {
9508      SDValue Arg = ShAmtOp.getOperand(i);
9509      if (Arg.getOpcode() == ISD::UNDEF) continue;
9510      BaseShAmt = Arg;
9511      break;
9512    }
9513    for (; i != NumElts; ++i) {
9514      SDValue Arg = ShAmtOp.getOperand(i);
9515      if (Arg.getOpcode() == ISD::UNDEF) continue;
9516      if (Arg != BaseShAmt) {
9517        return SDValue();
9518      }
9519    }
9520  } else if (ShAmtOp.getOpcode() == ISD::VECTOR_SHUFFLE &&
9521             cast<ShuffleVectorSDNode>(ShAmtOp)->isSplat()) {
9522    SDValue InVec = ShAmtOp.getOperand(0);
9523    if (InVec.getOpcode() == ISD::BUILD_VECTOR) {
9524      unsigned NumElts = InVec.getValueType().getVectorNumElements();
9525      unsigned i = 0;
9526      for (; i != NumElts; ++i) {
9527        SDValue Arg = InVec.getOperand(i);
9528        if (Arg.getOpcode() == ISD::UNDEF) continue;
9529        BaseShAmt = Arg;
9530        break;
9531      }
9532    } else if (InVec.getOpcode() == ISD::INSERT_VECTOR_ELT) {
9533       if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(InVec.getOperand(2))) {
9534         unsigned SplatIdx= cast<ShuffleVectorSDNode>(ShAmtOp)->getSplatIndex();
9535         if (C->getZExtValue() == SplatIdx)
9536           BaseShAmt = InVec.getOperand(1);
9537       }
9538    }
9539    if (BaseShAmt.getNode() == 0)
9540      BaseShAmt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, ShAmtOp,
9541                              DAG.getIntPtrConstant(0));
9542  } else
9543    return SDValue();
9544
9545  // The shift amount is an i32.
9546  if (EltVT.bitsGT(MVT::i32))
9547    BaseShAmt = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, BaseShAmt);
9548  else if (EltVT.bitsLT(MVT::i32))
9549    BaseShAmt = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i32, BaseShAmt);
9550
9551  // The shift amount is identical so we can do a vector shift.
9552  SDValue  ValOp = N->getOperand(0);
9553  switch (N->getOpcode()) {
9554  default:
9555    llvm_unreachable("Unknown shift opcode!");
9556    break;
9557  case ISD::SHL:
9558    if (VT == MVT::v2i64)
9559      return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, VT,
9560                         DAG.getConstant(Intrinsic::x86_sse2_pslli_q, MVT::i32),
9561                         ValOp, BaseShAmt);
9562    if (VT == MVT::v4i32)
9563      return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, VT,
9564                         DAG.getConstant(Intrinsic::x86_sse2_pslli_d, MVT::i32),
9565                         ValOp, BaseShAmt);
9566    if (VT == MVT::v8i16)
9567      return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, VT,
9568                         DAG.getConstant(Intrinsic::x86_sse2_pslli_w, MVT::i32),
9569                         ValOp, BaseShAmt);
9570    break;
9571  case ISD::SRA:
9572    if (VT == MVT::v4i32)
9573      return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, VT,
9574                         DAG.getConstant(Intrinsic::x86_sse2_psrai_d, MVT::i32),
9575                         ValOp, BaseShAmt);
9576    if (VT == MVT::v8i16)
9577      return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, VT,
9578                         DAG.getConstant(Intrinsic::x86_sse2_psrai_w, MVT::i32),
9579                         ValOp, BaseShAmt);
9580    break;
9581  case ISD::SRL:
9582    if (VT == MVT::v2i64)
9583      return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, VT,
9584                         DAG.getConstant(Intrinsic::x86_sse2_psrli_q, MVT::i32),
9585                         ValOp, BaseShAmt);
9586    if (VT == MVT::v4i32)
9587      return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, VT,
9588                         DAG.getConstant(Intrinsic::x86_sse2_psrli_d, MVT::i32),
9589                         ValOp, BaseShAmt);
9590    if (VT ==  MVT::v8i16)
9591      return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, VT,
9592                         DAG.getConstant(Intrinsic::x86_sse2_psrli_w, MVT::i32),
9593                         ValOp, BaseShAmt);
9594    break;
9595  }
9596  return SDValue();
9597}
9598
9599static SDValue PerformOrCombine(SDNode *N, SelectionDAG &DAG,
9600                                TargetLowering::DAGCombinerInfo &DCI,
9601                                const X86Subtarget *Subtarget) {
9602  if (DCI.isBeforeLegalizeOps())
9603    return SDValue();
9604
9605  EVT VT = N->getValueType(0);
9606  if (VT != MVT::i16 && VT != MVT::i32 && VT != MVT::i64)
9607    return SDValue();
9608
9609  // fold (or (x << c) | (y >> (64 - c))) ==> (shld64 x, y, c)
9610  SDValue N0 = N->getOperand(0);
9611  SDValue N1 = N->getOperand(1);
9612  if (N0.getOpcode() == ISD::SRL && N1.getOpcode() == ISD::SHL)
9613    std::swap(N0, N1);
9614  if (N0.getOpcode() != ISD::SHL || N1.getOpcode() != ISD::SRL)
9615    return SDValue();
9616  if (!N0.hasOneUse() || !N1.hasOneUse())
9617    return SDValue();
9618
9619  SDValue ShAmt0 = N0.getOperand(1);
9620  if (ShAmt0.getValueType() != MVT::i8)
9621    return SDValue();
9622  SDValue ShAmt1 = N1.getOperand(1);
9623  if (ShAmt1.getValueType() != MVT::i8)
9624    return SDValue();
9625  if (ShAmt0.getOpcode() == ISD::TRUNCATE)
9626    ShAmt0 = ShAmt0.getOperand(0);
9627  if (ShAmt1.getOpcode() == ISD::TRUNCATE)
9628    ShAmt1 = ShAmt1.getOperand(0);
9629
9630  DebugLoc DL = N->getDebugLoc();
9631  unsigned Opc = X86ISD::SHLD;
9632  SDValue Op0 = N0.getOperand(0);
9633  SDValue Op1 = N1.getOperand(0);
9634  if (ShAmt0.getOpcode() == ISD::SUB) {
9635    Opc = X86ISD::SHRD;
9636    std::swap(Op0, Op1);
9637    std::swap(ShAmt0, ShAmt1);
9638  }
9639
9640  unsigned Bits = VT.getSizeInBits();
9641  if (ShAmt1.getOpcode() == ISD::SUB) {
9642    SDValue Sum = ShAmt1.getOperand(0);
9643    if (ConstantSDNode *SumC = dyn_cast<ConstantSDNode>(Sum)) {
9644      SDValue ShAmt1Op1 = ShAmt1.getOperand(1);
9645      if (ShAmt1Op1.getNode()->getOpcode() == ISD::TRUNCATE)
9646        ShAmt1Op1 = ShAmt1Op1.getOperand(0);
9647      if (SumC->getSExtValue() == Bits && ShAmt1Op1 == ShAmt0)
9648        return DAG.getNode(Opc, DL, VT,
9649                           Op0, Op1,
9650                           DAG.getNode(ISD::TRUNCATE, DL,
9651                                       MVT::i8, ShAmt0));
9652    }
9653  } else if (ConstantSDNode *ShAmt1C = dyn_cast<ConstantSDNode>(ShAmt1)) {
9654    ConstantSDNode *ShAmt0C = dyn_cast<ConstantSDNode>(ShAmt0);
9655    if (ShAmt0C &&
9656        ShAmt0C->getSExtValue() + ShAmt1C->getSExtValue() == Bits)
9657      return DAG.getNode(Opc, DL, VT,
9658                         N0.getOperand(0), N1.getOperand(0),
9659                         DAG.getNode(ISD::TRUNCATE, DL,
9660                                       MVT::i8, ShAmt0));
9661  }
9662
9663  return SDValue();
9664}
9665
9666/// PerformSTORECombine - Do target-specific dag combines on STORE nodes.
9667static SDValue PerformSTORECombine(SDNode *N, SelectionDAG &DAG,
9668                                   const X86Subtarget *Subtarget) {
9669  // Turn load->store of MMX types into GPR load/stores.  This avoids clobbering
9670  // the FP state in cases where an emms may be missing.
9671  // A preferable solution to the general problem is to figure out the right
9672  // places to insert EMMS.  This qualifies as a quick hack.
9673
9674  // Similarly, turn load->store of i64 into double load/stores in 32-bit mode.
9675  StoreSDNode *St = cast<StoreSDNode>(N);
9676  EVT VT = St->getValue().getValueType();
9677  if (VT.getSizeInBits() != 64)
9678    return SDValue();
9679
9680  const Function *F = DAG.getMachineFunction().getFunction();
9681  bool NoImplicitFloatOps = F->hasFnAttr(Attribute::NoImplicitFloat);
9682  bool F64IsLegal = !UseSoftFloat && !NoImplicitFloatOps
9683    && Subtarget->hasSSE2();
9684  if ((VT.isVector() ||
9685       (VT == MVT::i64 && F64IsLegal && !Subtarget->is64Bit())) &&
9686      isa<LoadSDNode>(St->getValue()) &&
9687      !cast<LoadSDNode>(St->getValue())->isVolatile() &&
9688      St->getChain().hasOneUse() && !St->isVolatile()) {
9689    SDNode* LdVal = St->getValue().getNode();
9690    LoadSDNode *Ld = 0;
9691    int TokenFactorIndex = -1;
9692    SmallVector<SDValue, 8> Ops;
9693    SDNode* ChainVal = St->getChain().getNode();
9694    // Must be a store of a load.  We currently handle two cases:  the load
9695    // is a direct child, and it's under an intervening TokenFactor.  It is
9696    // possible to dig deeper under nested TokenFactors.
9697    if (ChainVal == LdVal)
9698      Ld = cast<LoadSDNode>(St->getChain());
9699    else if (St->getValue().hasOneUse() &&
9700             ChainVal->getOpcode() == ISD::TokenFactor) {
9701      for (unsigned i=0, e = ChainVal->getNumOperands(); i != e; ++i) {
9702        if (ChainVal->getOperand(i).getNode() == LdVal) {
9703          TokenFactorIndex = i;
9704          Ld = cast<LoadSDNode>(St->getValue());
9705        } else
9706          Ops.push_back(ChainVal->getOperand(i));
9707      }
9708    }
9709
9710    if (!Ld || !ISD::isNormalLoad(Ld))
9711      return SDValue();
9712
9713    // If this is not the MMX case, i.e. we are just turning i64 load/store
9714    // into f64 load/store, avoid the transformation if there are multiple
9715    // uses of the loaded value.
9716    if (!VT.isVector() && !Ld->hasNUsesOfValue(1, 0))
9717      return SDValue();
9718
9719    DebugLoc LdDL = Ld->getDebugLoc();
9720    DebugLoc StDL = N->getDebugLoc();
9721    // If we are a 64-bit capable x86, lower to a single movq load/store pair.
9722    // Otherwise, if it's legal to use f64 SSE instructions, use f64 load/store
9723    // pair instead.
9724    if (Subtarget->is64Bit() || F64IsLegal) {
9725      EVT LdVT = Subtarget->is64Bit() ? MVT::i64 : MVT::f64;
9726      SDValue NewLd = DAG.getLoad(LdVT, LdDL, Ld->getChain(),
9727                                  Ld->getBasePtr(), Ld->getSrcValue(),
9728                                  Ld->getSrcValueOffset(), Ld->isVolatile(),
9729                                  Ld->isNonTemporal(), Ld->getAlignment());
9730      SDValue NewChain = NewLd.getValue(1);
9731      if (TokenFactorIndex != -1) {
9732        Ops.push_back(NewChain);
9733        NewChain = DAG.getNode(ISD::TokenFactor, LdDL, MVT::Other, &Ops[0],
9734                               Ops.size());
9735      }
9736      return DAG.getStore(NewChain, StDL, NewLd, St->getBasePtr(),
9737                          St->getSrcValue(), St->getSrcValueOffset(),
9738                          St->isVolatile(), St->isNonTemporal(),
9739                          St->getAlignment());
9740    }
9741
9742    // Otherwise, lower to two pairs of 32-bit loads / stores.
9743    SDValue LoAddr = Ld->getBasePtr();
9744    SDValue HiAddr = DAG.getNode(ISD::ADD, LdDL, MVT::i32, LoAddr,
9745                                 DAG.getConstant(4, MVT::i32));
9746
9747    SDValue LoLd = DAG.getLoad(MVT::i32, LdDL, Ld->getChain(), LoAddr,
9748                               Ld->getSrcValue(), Ld->getSrcValueOffset(),
9749                               Ld->isVolatile(), Ld->isNonTemporal(),
9750                               Ld->getAlignment());
9751    SDValue HiLd = DAG.getLoad(MVT::i32, LdDL, Ld->getChain(), HiAddr,
9752                               Ld->getSrcValue(), Ld->getSrcValueOffset()+4,
9753                               Ld->isVolatile(), Ld->isNonTemporal(),
9754                               MinAlign(Ld->getAlignment(), 4));
9755
9756    SDValue NewChain = LoLd.getValue(1);
9757    if (TokenFactorIndex != -1) {
9758      Ops.push_back(LoLd);
9759      Ops.push_back(HiLd);
9760      NewChain = DAG.getNode(ISD::TokenFactor, LdDL, MVT::Other, &Ops[0],
9761                             Ops.size());
9762    }
9763
9764    LoAddr = St->getBasePtr();
9765    HiAddr = DAG.getNode(ISD::ADD, StDL, MVT::i32, LoAddr,
9766                         DAG.getConstant(4, MVT::i32));
9767
9768    SDValue LoSt = DAG.getStore(NewChain, StDL, LoLd, LoAddr,
9769                                St->getSrcValue(), St->getSrcValueOffset(),
9770                                St->isVolatile(), St->isNonTemporal(),
9771                                St->getAlignment());
9772    SDValue HiSt = DAG.getStore(NewChain, StDL, HiLd, HiAddr,
9773                                St->getSrcValue(),
9774                                St->getSrcValueOffset() + 4,
9775                                St->isVolatile(),
9776                                St->isNonTemporal(),
9777                                MinAlign(St->getAlignment(), 4));
9778    return DAG.getNode(ISD::TokenFactor, StDL, MVT::Other, LoSt, HiSt);
9779  }
9780  return SDValue();
9781}
9782
9783/// PerformFORCombine - Do target-specific dag combines on X86ISD::FOR and
9784/// X86ISD::FXOR nodes.
9785static SDValue PerformFORCombine(SDNode *N, SelectionDAG &DAG) {
9786  assert(N->getOpcode() == X86ISD::FOR || N->getOpcode() == X86ISD::FXOR);
9787  // F[X]OR(0.0, x) -> x
9788  // F[X]OR(x, 0.0) -> x
9789  if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(N->getOperand(0)))
9790    if (C->getValueAPF().isPosZero())
9791      return N->getOperand(1);
9792  if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(N->getOperand(1)))
9793    if (C->getValueAPF().isPosZero())
9794      return N->getOperand(0);
9795  return SDValue();
9796}
9797
9798/// PerformFANDCombine - Do target-specific dag combines on X86ISD::FAND nodes.
9799static SDValue PerformFANDCombine(SDNode *N, SelectionDAG &DAG) {
9800  // FAND(0.0, x) -> 0.0
9801  // FAND(x, 0.0) -> 0.0
9802  if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(N->getOperand(0)))
9803    if (C->getValueAPF().isPosZero())
9804      return N->getOperand(0);
9805  if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(N->getOperand(1)))
9806    if (C->getValueAPF().isPosZero())
9807      return N->getOperand(1);
9808  return SDValue();
9809}
9810
9811static SDValue PerformBTCombine(SDNode *N,
9812                                SelectionDAG &DAG,
9813                                TargetLowering::DAGCombinerInfo &DCI) {
9814  // BT ignores high bits in the bit index operand.
9815  SDValue Op1 = N->getOperand(1);
9816  if (Op1.hasOneUse()) {
9817    unsigned BitWidth = Op1.getValueSizeInBits();
9818    APInt DemandedMask = APInt::getLowBitsSet(BitWidth, Log2_32(BitWidth));
9819    APInt KnownZero, KnownOne;
9820    TargetLowering::TargetLoweringOpt TLO(DAG, !DCI.isBeforeLegalize(),
9821                                          !DCI.isBeforeLegalizeOps());
9822    const TargetLowering &TLI = DAG.getTargetLoweringInfo();
9823    if (TLO.ShrinkDemandedConstant(Op1, DemandedMask) ||
9824        TLI.SimplifyDemandedBits(Op1, DemandedMask, KnownZero, KnownOne, TLO))
9825      DCI.CommitTargetLoweringOpt(TLO);
9826  }
9827  return SDValue();
9828}
9829
9830static SDValue PerformVZEXT_MOVLCombine(SDNode *N, SelectionDAG &DAG) {
9831  SDValue Op = N->getOperand(0);
9832  if (Op.getOpcode() == ISD::BIT_CONVERT)
9833    Op = Op.getOperand(0);
9834  EVT VT = N->getValueType(0), OpVT = Op.getValueType();
9835  if (Op.getOpcode() == X86ISD::VZEXT_LOAD &&
9836      VT.getVectorElementType().getSizeInBits() ==
9837      OpVT.getVectorElementType().getSizeInBits()) {
9838    return DAG.getNode(ISD::BIT_CONVERT, N->getDebugLoc(), VT, Op);
9839  }
9840  return SDValue();
9841}
9842
9843// On X86 and X86-64, atomic operations are lowered to locked instructions.
9844// Locked instructions, in turn, have implicit fence semantics (all memory
9845// operations are flushed before issuing the locked instruction, and the
9846// are not buffered), so we can fold away the common pattern of
9847// fence-atomic-fence.
9848static SDValue PerformMEMBARRIERCombine(SDNode* N, SelectionDAG &DAG) {
9849  SDValue atomic = N->getOperand(0);
9850  switch (atomic.getOpcode()) {
9851    case ISD::ATOMIC_CMP_SWAP:
9852    case ISD::ATOMIC_SWAP:
9853    case ISD::ATOMIC_LOAD_ADD:
9854    case ISD::ATOMIC_LOAD_SUB:
9855    case ISD::ATOMIC_LOAD_AND:
9856    case ISD::ATOMIC_LOAD_OR:
9857    case ISD::ATOMIC_LOAD_XOR:
9858    case ISD::ATOMIC_LOAD_NAND:
9859    case ISD::ATOMIC_LOAD_MIN:
9860    case ISD::ATOMIC_LOAD_MAX:
9861    case ISD::ATOMIC_LOAD_UMIN:
9862    case ISD::ATOMIC_LOAD_UMAX:
9863      break;
9864    default:
9865      return SDValue();
9866  }
9867
9868  SDValue fence = atomic.getOperand(0);
9869  if (fence.getOpcode() != ISD::MEMBARRIER)
9870    return SDValue();
9871
9872  switch (atomic.getOpcode()) {
9873    case ISD::ATOMIC_CMP_SWAP:
9874      return DAG.UpdateNodeOperands(atomic, fence.getOperand(0),
9875                                    atomic.getOperand(1), atomic.getOperand(2),
9876                                    atomic.getOperand(3));
9877    case ISD::ATOMIC_SWAP:
9878    case ISD::ATOMIC_LOAD_ADD:
9879    case ISD::ATOMIC_LOAD_SUB:
9880    case ISD::ATOMIC_LOAD_AND:
9881    case ISD::ATOMIC_LOAD_OR:
9882    case ISD::ATOMIC_LOAD_XOR:
9883    case ISD::ATOMIC_LOAD_NAND:
9884    case ISD::ATOMIC_LOAD_MIN:
9885    case ISD::ATOMIC_LOAD_MAX:
9886    case ISD::ATOMIC_LOAD_UMIN:
9887    case ISD::ATOMIC_LOAD_UMAX:
9888      return DAG.UpdateNodeOperands(atomic, fence.getOperand(0),
9889                                    atomic.getOperand(1), atomic.getOperand(2));
9890    default:
9891      return SDValue();
9892  }
9893}
9894
9895static SDValue PerformZExtCombine(SDNode *N, SelectionDAG &DAG) {
9896  // (i32 zext (and (i8  x86isd::setcc_carry), 1)) ->
9897  //           (and (i32 x86isd::setcc_carry), 1)
9898  // This eliminates the zext. This transformation is necessary because
9899  // ISD::SETCC is always legalized to i8.
9900  DebugLoc dl = N->getDebugLoc();
9901  SDValue N0 = N->getOperand(0);
9902  EVT VT = N->getValueType(0);
9903  if (N0.getOpcode() == ISD::AND &&
9904      N0.hasOneUse() &&
9905      N0.getOperand(0).hasOneUse()) {
9906    SDValue N00 = N0.getOperand(0);
9907    if (N00.getOpcode() != X86ISD::SETCC_CARRY)
9908      return SDValue();
9909    ConstantSDNode *C = dyn_cast<ConstantSDNode>(N0.getOperand(1));
9910    if (!C || C->getZExtValue() != 1)
9911      return SDValue();
9912    return DAG.getNode(ISD::AND, dl, VT,
9913                       DAG.getNode(X86ISD::SETCC_CARRY, dl, VT,
9914                                   N00.getOperand(0), N00.getOperand(1)),
9915                       DAG.getConstant(1, VT));
9916  }
9917
9918  return SDValue();
9919}
9920
9921SDValue X86TargetLowering::PerformDAGCombine(SDNode *N,
9922                                             DAGCombinerInfo &DCI) const {
9923  SelectionDAG &DAG = DCI.DAG;
9924  switch (N->getOpcode()) {
9925  default: break;
9926  case ISD::VECTOR_SHUFFLE: return PerformShuffleCombine(N, DAG, *this);
9927  case ISD::EXTRACT_VECTOR_ELT:
9928                        return PerformEXTRACT_VECTOR_ELTCombine(N, DAG, *this);
9929  case ISD::SELECT:         return PerformSELECTCombine(N, DAG, Subtarget);
9930  case X86ISD::CMOV:        return PerformCMOVCombine(N, DAG, DCI);
9931  case ISD::MUL:            return PerformMulCombine(N, DAG, DCI);
9932  case ISD::SHL:
9933  case ISD::SRA:
9934  case ISD::SRL:            return PerformShiftCombine(N, DAG, Subtarget);
9935  case ISD::OR:             return PerformOrCombine(N, DAG, DCI, Subtarget);
9936  case ISD::STORE:          return PerformSTORECombine(N, DAG, Subtarget);
9937  case X86ISD::FXOR:
9938  case X86ISD::FOR:         return PerformFORCombine(N, DAG);
9939  case X86ISD::FAND:        return PerformFANDCombine(N, DAG);
9940  case X86ISD::BT:          return PerformBTCombine(N, DAG, DCI);
9941  case X86ISD::VZEXT_MOVL:  return PerformVZEXT_MOVLCombine(N, DAG);
9942  case ISD::MEMBARRIER:     return PerformMEMBARRIERCombine(N, DAG);
9943  case ISD::ZERO_EXTEND:    return PerformZExtCombine(N, DAG);
9944  }
9945
9946  return SDValue();
9947}
9948
9949/// isTypeDesirableForOp - Return true if the target has native support for
9950/// the specified value type and it is 'desirable' to use the type for the
9951/// given node type. e.g. On x86 i16 is legal, but undesirable since i16
9952/// instruction encodings are longer and some i16 instructions are slow.
9953bool X86TargetLowering::isTypeDesirableForOp(unsigned Opc, EVT VT) const {
9954  if (!isTypeLegal(VT))
9955    return false;
9956  if (VT != MVT::i16)
9957    return true;
9958
9959  switch (Opc) {
9960  default:
9961    return true;
9962  case ISD::LOAD:
9963  case ISD::SIGN_EXTEND:
9964  case ISD::ZERO_EXTEND:
9965  case ISD::ANY_EXTEND:
9966  case ISD::SHL:
9967  case ISD::SRL:
9968  case ISD::SUB:
9969  case ISD::ADD:
9970  case ISD::MUL:
9971  case ISD::AND:
9972  case ISD::OR:
9973  case ISD::XOR:
9974    return false;
9975  }
9976}
9977
9978static bool MayFoldLoad(SDValue Op) {
9979  return Op.hasOneUse() && ISD::isNormalLoad(Op.getNode());
9980}
9981
9982static bool MayFoldIntoStore(SDValue Op) {
9983  return Op.hasOneUse() && ISD::isNormalStore(*Op.getNode()->use_begin());
9984}
9985
9986/// IsDesirableToPromoteOp - This method query the target whether it is
9987/// beneficial for dag combiner to promote the specified node. If true, it
9988/// should return the desired promotion type by reference.
9989bool X86TargetLowering::IsDesirableToPromoteOp(SDValue Op, EVT &PVT) const {
9990  EVT VT = Op.getValueType();
9991  if (VT != MVT::i16)
9992    return false;
9993
9994  bool Promote = false;
9995  bool Commute = false;
9996  switch (Op.getOpcode()) {
9997  default: break;
9998  case ISD::LOAD: {
9999    LoadSDNode *LD = cast<LoadSDNode>(Op);
10000    // If the non-extending load has a single use and it's not live out, then it
10001    // might be folded.
10002    if (LD->getExtensionType() == ISD::NON_EXTLOAD /*&&
10003                                                     Op.hasOneUse()*/) {
10004      for (SDNode::use_iterator UI = Op.getNode()->use_begin(),
10005             UE = Op.getNode()->use_end(); UI != UE; ++UI) {
10006        // The only case where we'd want to promote LOAD (rather then it being
10007        // promoted as an operand is when it's only use is liveout.
10008        if (UI->getOpcode() != ISD::CopyToReg)
10009          return false;
10010      }
10011    }
10012    Promote = true;
10013    break;
10014  }
10015  case ISD::SIGN_EXTEND:
10016  case ISD::ZERO_EXTEND:
10017  case ISD::ANY_EXTEND:
10018    Promote = true;
10019    break;
10020  case ISD::SHL:
10021  case ISD::SRL: {
10022    SDValue N0 = Op.getOperand(0);
10023    // Look out for (store (shl (load), x)).
10024    if (MayFoldLoad(N0) && MayFoldIntoStore(Op))
10025      return false;
10026    Promote = true;
10027    break;
10028  }
10029  case ISD::ADD:
10030  case ISD::MUL:
10031  case ISD::AND:
10032  case ISD::OR:
10033  case ISD::XOR:
10034    Commute = true;
10035    // fallthrough
10036  case ISD::SUB: {
10037    SDValue N0 = Op.getOperand(0);
10038    SDValue N1 = Op.getOperand(1);
10039    if (!Commute && MayFoldLoad(N1))
10040      return false;
10041    // Avoid disabling potential load folding opportunities.
10042    if (MayFoldLoad(N0) && (!isa<ConstantSDNode>(N1) || MayFoldIntoStore(Op)))
10043      return false;
10044    if (MayFoldLoad(N1) && (!isa<ConstantSDNode>(N0) || MayFoldIntoStore(Op)))
10045      return false;
10046    Promote = true;
10047  }
10048  }
10049
10050  PVT = MVT::i32;
10051  return Promote;
10052}
10053
10054//===----------------------------------------------------------------------===//
10055//                           X86 Inline Assembly Support
10056//===----------------------------------------------------------------------===//
10057
10058static bool LowerToBSwap(CallInst *CI) {
10059  // FIXME: this should verify that we are targetting a 486 or better.  If not,
10060  // we will turn this bswap into something that will be lowered to logical ops
10061  // instead of emitting the bswap asm.  For now, we don't support 486 or lower
10062  // so don't worry about this.
10063
10064  // Verify this is a simple bswap.
10065  if (CI->getNumOperands() != 2 ||
10066      CI->getType() != CI->getOperand(1)->getType() ||
10067      !CI->getType()->isIntegerTy())
10068    return false;
10069
10070  const IntegerType *Ty = dyn_cast<IntegerType>(CI->getType());
10071  if (!Ty || Ty->getBitWidth() % 16 != 0)
10072    return false;
10073
10074  // Okay, we can do this xform, do so now.
10075  const Type *Tys[] = { Ty };
10076  Module *M = CI->getParent()->getParent()->getParent();
10077  Constant *Int = Intrinsic::getDeclaration(M, Intrinsic::bswap, Tys, 1);
10078
10079  Value *Op = CI->getOperand(1);
10080  Op = CallInst::Create(Int, Op, CI->getName(), CI);
10081
10082  CI->replaceAllUsesWith(Op);
10083  CI->eraseFromParent();
10084  return true;
10085}
10086
10087bool X86TargetLowering::ExpandInlineAsm(CallInst *CI) const {
10088  InlineAsm *IA = cast<InlineAsm>(CI->getCalledValue());
10089  std::vector<InlineAsm::ConstraintInfo> Constraints = IA->ParseConstraints();
10090
10091  std::string AsmStr = IA->getAsmString();
10092
10093  // TODO: should remove alternatives from the asmstring: "foo {a|b}" -> "foo a"
10094  SmallVector<StringRef, 4> AsmPieces;
10095  SplitString(AsmStr, AsmPieces, "\n");  // ; as separator?
10096
10097  switch (AsmPieces.size()) {
10098  default: return false;
10099  case 1:
10100    AsmStr = AsmPieces[0];
10101    AsmPieces.clear();
10102    SplitString(AsmStr, AsmPieces, " \t");  // Split with whitespace.
10103
10104    // bswap $0
10105    if (AsmPieces.size() == 2 &&
10106        (AsmPieces[0] == "bswap" ||
10107         AsmPieces[0] == "bswapq" ||
10108         AsmPieces[0] == "bswapl") &&
10109        (AsmPieces[1] == "$0" ||
10110         AsmPieces[1] == "${0:q}")) {
10111      // No need to check constraints, nothing other than the equivalent of
10112      // "=r,0" would be valid here.
10113      return LowerToBSwap(CI);
10114    }
10115    // rorw $$8, ${0:w}  -->  llvm.bswap.i16
10116    if (CI->getType()->isIntegerTy(16) &&
10117        AsmPieces.size() == 3 &&
10118        (AsmPieces[0] == "rorw" || AsmPieces[0] == "rolw") &&
10119        AsmPieces[1] == "$$8," &&
10120        AsmPieces[2] == "${0:w}" &&
10121        IA->getConstraintString().compare(0, 5, "=r,0,") == 0) {
10122      AsmPieces.clear();
10123      const std::string &Constraints = IA->getConstraintString();
10124      SplitString(StringRef(Constraints).substr(5), AsmPieces, ",");
10125      std::sort(AsmPieces.begin(), AsmPieces.end());
10126      if (AsmPieces.size() == 4 &&
10127          AsmPieces[0] == "~{cc}" &&
10128          AsmPieces[1] == "~{dirflag}" &&
10129          AsmPieces[2] == "~{flags}" &&
10130          AsmPieces[3] == "~{fpsr}") {
10131        return LowerToBSwap(CI);
10132      }
10133    }
10134    break;
10135  case 3:
10136    if (CI->getType()->isIntegerTy(64) &&
10137        Constraints.size() >= 2 &&
10138        Constraints[0].Codes.size() == 1 && Constraints[0].Codes[0] == "A" &&
10139        Constraints[1].Codes.size() == 1 && Constraints[1].Codes[0] == "0") {
10140      // bswap %eax / bswap %edx / xchgl %eax, %edx  -> llvm.bswap.i64
10141      SmallVector<StringRef, 4> Words;
10142      SplitString(AsmPieces[0], Words, " \t");
10143      if (Words.size() == 2 && Words[0] == "bswap" && Words[1] == "%eax") {
10144        Words.clear();
10145        SplitString(AsmPieces[1], Words, " \t");
10146        if (Words.size() == 2 && Words[0] == "bswap" && Words[1] == "%edx") {
10147          Words.clear();
10148          SplitString(AsmPieces[2], Words, " \t,");
10149          if (Words.size() == 3 && Words[0] == "xchgl" && Words[1] == "%eax" &&
10150              Words[2] == "%edx") {
10151            return LowerToBSwap(CI);
10152          }
10153        }
10154      }
10155    }
10156    break;
10157  }
10158  return false;
10159}
10160
10161
10162
10163/// getConstraintType - Given a constraint letter, return the type of
10164/// constraint it is for this target.
10165X86TargetLowering::ConstraintType
10166X86TargetLowering::getConstraintType(const std::string &Constraint) const {
10167  if (Constraint.size() == 1) {
10168    switch (Constraint[0]) {
10169    case 'A':
10170      return C_Register;
10171    case 'f':
10172    case 'r':
10173    case 'R':
10174    case 'l':
10175    case 'q':
10176    case 'Q':
10177    case 'x':
10178    case 'y':
10179    case 'Y':
10180      return C_RegisterClass;
10181    case 'e':
10182    case 'Z':
10183      return C_Other;
10184    default:
10185      break;
10186    }
10187  }
10188  return TargetLowering::getConstraintType(Constraint);
10189}
10190
10191/// LowerXConstraint - try to replace an X constraint, which matches anything,
10192/// with another that has more specific requirements based on the type of the
10193/// corresponding operand.
10194const char *X86TargetLowering::
10195LowerXConstraint(EVT ConstraintVT) const {
10196  // FP X constraints get lowered to SSE1/2 registers if available, otherwise
10197  // 'f' like normal targets.
10198  if (ConstraintVT.isFloatingPoint()) {
10199    if (Subtarget->hasSSE2())
10200      return "Y";
10201    if (Subtarget->hasSSE1())
10202      return "x";
10203  }
10204
10205  return TargetLowering::LowerXConstraint(ConstraintVT);
10206}
10207
10208/// LowerAsmOperandForConstraint - Lower the specified operand into the Ops
10209/// vector.  If it is invalid, don't add anything to Ops.
10210void X86TargetLowering::LowerAsmOperandForConstraint(SDValue Op,
10211                                                     char Constraint,
10212                                                     bool hasMemory,
10213                                                     std::vector<SDValue>&Ops,
10214                                                     SelectionDAG &DAG) const {
10215  SDValue Result(0, 0);
10216
10217  switch (Constraint) {
10218  default: break;
10219  case 'I':
10220    if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) {
10221      if (C->getZExtValue() <= 31) {
10222        Result = DAG.getTargetConstant(C->getZExtValue(), Op.getValueType());
10223        break;
10224      }
10225    }
10226    return;
10227  case 'J':
10228    if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) {
10229      if (C->getZExtValue() <= 63) {
10230        Result = DAG.getTargetConstant(C->getZExtValue(), Op.getValueType());
10231        break;
10232      }
10233    }
10234    return;
10235  case 'K':
10236    if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) {
10237      if ((int8_t)C->getSExtValue() == C->getSExtValue()) {
10238        Result = DAG.getTargetConstant(C->getZExtValue(), Op.getValueType());
10239        break;
10240      }
10241    }
10242    return;
10243  case 'N':
10244    if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) {
10245      if (C->getZExtValue() <= 255) {
10246        Result = DAG.getTargetConstant(C->getZExtValue(), Op.getValueType());
10247        break;
10248      }
10249    }
10250    return;
10251  case 'e': {
10252    // 32-bit signed value
10253    if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) {
10254      if (ConstantInt::isValueValidForType(Type::getInt32Ty(*DAG.getContext()),
10255                                           C->getSExtValue())) {
10256        // Widen to 64 bits here to get it sign extended.
10257        Result = DAG.getTargetConstant(C->getSExtValue(), MVT::i64);
10258        break;
10259      }
10260    // FIXME gcc accepts some relocatable values here too, but only in certain
10261    // memory models; it's complicated.
10262    }
10263    return;
10264  }
10265  case 'Z': {
10266    // 32-bit unsigned value
10267    if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) {
10268      if (ConstantInt::isValueValidForType(Type::getInt32Ty(*DAG.getContext()),
10269                                           C->getZExtValue())) {
10270        Result = DAG.getTargetConstant(C->getZExtValue(), Op.getValueType());
10271        break;
10272      }
10273    }
10274    // FIXME gcc accepts some relocatable values here too, but only in certain
10275    // memory models; it's complicated.
10276    return;
10277  }
10278  case 'i': {
10279    // Literal immediates are always ok.
10280    if (ConstantSDNode *CST = dyn_cast<ConstantSDNode>(Op)) {
10281      // Widen to 64 bits here to get it sign extended.
10282      Result = DAG.getTargetConstant(CST->getSExtValue(), MVT::i64);
10283      break;
10284    }
10285
10286    // If we are in non-pic codegen mode, we allow the address of a global (with
10287    // an optional displacement) to be used with 'i'.
10288    GlobalAddressSDNode *GA = 0;
10289    int64_t Offset = 0;
10290
10291    // Match either (GA), (GA+C), (GA+C1+C2), etc.
10292    while (1) {
10293      if ((GA = dyn_cast<GlobalAddressSDNode>(Op))) {
10294        Offset += GA->getOffset();
10295        break;
10296      } else if (Op.getOpcode() == ISD::ADD) {
10297        if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
10298          Offset += C->getZExtValue();
10299          Op = Op.getOperand(0);
10300          continue;
10301        }
10302      } else if (Op.getOpcode() == ISD::SUB) {
10303        if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
10304          Offset += -C->getZExtValue();
10305          Op = Op.getOperand(0);
10306          continue;
10307        }
10308      }
10309
10310      // Otherwise, this isn't something we can handle, reject it.
10311      return;
10312    }
10313
10314    const GlobalValue *GV = GA->getGlobal();
10315    // If we require an extra load to get this address, as in PIC mode, we
10316    // can't accept it.
10317    if (isGlobalStubReference(Subtarget->ClassifyGlobalReference(GV,
10318                                                        getTargetMachine())))
10319      return;
10320
10321    if (hasMemory)
10322      Op = LowerGlobalAddress(GV, Op.getDebugLoc(), Offset, DAG);
10323    else
10324      Op = DAG.getTargetGlobalAddress(GV, GA->getValueType(0), Offset);
10325    Result = Op;
10326    break;
10327  }
10328  }
10329
10330  if (Result.getNode()) {
10331    Ops.push_back(Result);
10332    return;
10333  }
10334  return TargetLowering::LowerAsmOperandForConstraint(Op, Constraint, hasMemory,
10335                                                      Ops, DAG);
10336}
10337
10338std::vector<unsigned> X86TargetLowering::
10339getRegClassForInlineAsmConstraint(const std::string &Constraint,
10340                                  EVT VT) const {
10341  if (Constraint.size() == 1) {
10342    // FIXME: not handling fp-stack yet!
10343    switch (Constraint[0]) {      // GCC X86 Constraint Letters
10344    default: break;  // Unknown constraint letter
10345    case 'q':   // GENERAL_REGS in 64-bit mode, Q_REGS in 32-bit mode.
10346      if (Subtarget->is64Bit()) {
10347        if (VT == MVT::i32)
10348          return make_vector<unsigned>(X86::EAX, X86::EDX, X86::ECX, X86::EBX,
10349                                       X86::ESI, X86::EDI, X86::R8D, X86::R9D,
10350                                       X86::R10D,X86::R11D,X86::R12D,
10351                                       X86::R13D,X86::R14D,X86::R15D,
10352                                       X86::EBP, X86::ESP, 0);
10353        else if (VT == MVT::i16)
10354          return make_vector<unsigned>(X86::AX,  X86::DX,  X86::CX, X86::BX,
10355                                       X86::SI,  X86::DI,  X86::R8W,X86::R9W,
10356                                       X86::R10W,X86::R11W,X86::R12W,
10357                                       X86::R13W,X86::R14W,X86::R15W,
10358                                       X86::BP,  X86::SP, 0);
10359        else if (VT == MVT::i8)
10360          return make_vector<unsigned>(X86::AL,  X86::DL,  X86::CL, X86::BL,
10361                                       X86::SIL, X86::DIL, X86::R8B,X86::R9B,
10362                                       X86::R10B,X86::R11B,X86::R12B,
10363                                       X86::R13B,X86::R14B,X86::R15B,
10364                                       X86::BPL, X86::SPL, 0);
10365
10366        else if (VT == MVT::i64)
10367          return make_vector<unsigned>(X86::RAX, X86::RDX, X86::RCX, X86::RBX,
10368                                       X86::RSI, X86::RDI, X86::R8,  X86::R9,
10369                                       X86::R10, X86::R11, X86::R12,
10370                                       X86::R13, X86::R14, X86::R15,
10371                                       X86::RBP, X86::RSP, 0);
10372
10373        break;
10374      }
10375      // 32-bit fallthrough
10376    case 'Q':   // Q_REGS
10377      if (VT == MVT::i32)
10378        return make_vector<unsigned>(X86::EAX, X86::EDX, X86::ECX, X86::EBX, 0);
10379      else if (VT == MVT::i16)
10380        return make_vector<unsigned>(X86::AX, X86::DX, X86::CX, X86::BX, 0);
10381      else if (VT == MVT::i8)
10382        return make_vector<unsigned>(X86::AL, X86::DL, X86::CL, X86::BL, 0);
10383      else if (VT == MVT::i64)
10384        return make_vector<unsigned>(X86::RAX, X86::RDX, X86::RCX, X86::RBX, 0);
10385      break;
10386    }
10387  }
10388
10389  return std::vector<unsigned>();
10390}
10391
10392std::pair<unsigned, const TargetRegisterClass*>
10393X86TargetLowering::getRegForInlineAsmConstraint(const std::string &Constraint,
10394                                                EVT VT) const {
10395  // First, see if this is a constraint that directly corresponds to an LLVM
10396  // register class.
10397  if (Constraint.size() == 1) {
10398    // GCC Constraint Letters
10399    switch (Constraint[0]) {
10400    default: break;
10401    case 'r':   // GENERAL_REGS
10402    case 'l':   // INDEX_REGS
10403      if (VT == MVT::i8)
10404        return std::make_pair(0U, X86::GR8RegisterClass);
10405      if (VT == MVT::i16)
10406        return std::make_pair(0U, X86::GR16RegisterClass);
10407      if (VT == MVT::i32 || !Subtarget->is64Bit())
10408        return std::make_pair(0U, X86::GR32RegisterClass);
10409      return std::make_pair(0U, X86::GR64RegisterClass);
10410    case 'R':   // LEGACY_REGS
10411      if (VT == MVT::i8)
10412        return std::make_pair(0U, X86::GR8_NOREXRegisterClass);
10413      if (VT == MVT::i16)
10414        return std::make_pair(0U, X86::GR16_NOREXRegisterClass);
10415      if (VT == MVT::i32 || !Subtarget->is64Bit())
10416        return std::make_pair(0U, X86::GR32_NOREXRegisterClass);
10417      return std::make_pair(0U, X86::GR64_NOREXRegisterClass);
10418    case 'f':  // FP Stack registers.
10419      // If SSE is enabled for this VT, use f80 to ensure the isel moves the
10420      // value to the correct fpstack register class.
10421      if (VT == MVT::f32 && !isScalarFPTypeInSSEReg(VT))
10422        return std::make_pair(0U, X86::RFP32RegisterClass);
10423      if (VT == MVT::f64 && !isScalarFPTypeInSSEReg(VT))
10424        return std::make_pair(0U, X86::RFP64RegisterClass);
10425      return std::make_pair(0U, X86::RFP80RegisterClass);
10426    case 'y':   // MMX_REGS if MMX allowed.
10427      if (!Subtarget->hasMMX()) break;
10428      return std::make_pair(0U, X86::VR64RegisterClass);
10429    case 'Y':   // SSE_REGS if SSE2 allowed
10430      if (!Subtarget->hasSSE2()) break;
10431      // FALL THROUGH.
10432    case 'x':   // SSE_REGS if SSE1 allowed
10433      if (!Subtarget->hasSSE1()) break;
10434
10435      switch (VT.getSimpleVT().SimpleTy) {
10436      default: break;
10437      // Scalar SSE types.
10438      case MVT::f32:
10439      case MVT::i32:
10440        return std::make_pair(0U, X86::FR32RegisterClass);
10441      case MVT::f64:
10442      case MVT::i64:
10443        return std::make_pair(0U, X86::FR64RegisterClass);
10444      // Vector types.
10445      case MVT::v16i8:
10446      case MVT::v8i16:
10447      case MVT::v4i32:
10448      case MVT::v2i64:
10449      case MVT::v4f32:
10450      case MVT::v2f64:
10451        return std::make_pair(0U, X86::VR128RegisterClass);
10452      }
10453      break;
10454    }
10455  }
10456
10457  // Use the default implementation in TargetLowering to convert the register
10458  // constraint into a member of a register class.
10459  std::pair<unsigned, const TargetRegisterClass*> Res;
10460  Res = TargetLowering::getRegForInlineAsmConstraint(Constraint, VT);
10461
10462  // Not found as a standard register?
10463  if (Res.second == 0) {
10464    // Map st(0) -> st(7) -> ST0
10465    if (Constraint.size() == 7 && Constraint[0] == '{' &&
10466        tolower(Constraint[1]) == 's' &&
10467        tolower(Constraint[2]) == 't' &&
10468        Constraint[3] == '(' &&
10469        (Constraint[4] >= '0' && Constraint[4] <= '7') &&
10470        Constraint[5] == ')' &&
10471        Constraint[6] == '}') {
10472
10473      Res.first = X86::ST0+Constraint[4]-'0';
10474      Res.second = X86::RFP80RegisterClass;
10475      return Res;
10476    }
10477
10478    // GCC allows "st(0)" to be called just plain "st".
10479    if (StringRef("{st}").equals_lower(Constraint)) {
10480      Res.first = X86::ST0;
10481      Res.second = X86::RFP80RegisterClass;
10482      return Res;
10483    }
10484
10485    // flags -> EFLAGS
10486    if (StringRef("{flags}").equals_lower(Constraint)) {
10487      Res.first = X86::EFLAGS;
10488      Res.second = X86::CCRRegisterClass;
10489      return Res;
10490    }
10491
10492    // 'A' means EAX + EDX.
10493    if (Constraint == "A") {
10494      Res.first = X86::EAX;
10495      Res.second = X86::GR32_ADRegisterClass;
10496      return Res;
10497    }
10498    return Res;
10499  }
10500
10501  // Otherwise, check to see if this is a register class of the wrong value
10502  // type.  For example, we want to map "{ax},i32" -> {eax}, we don't want it to
10503  // turn into {ax},{dx}.
10504  if (Res.second->hasType(VT))
10505    return Res;   // Correct type already, nothing to do.
10506
10507  // All of the single-register GCC register classes map their values onto
10508  // 16-bit register pieces "ax","dx","cx","bx","si","di","bp","sp".  If we
10509  // really want an 8-bit or 32-bit register, map to the appropriate register
10510  // class and return the appropriate register.
10511  if (Res.second == X86::GR16RegisterClass) {
10512    if (VT == MVT::i8) {
10513      unsigned DestReg = 0;
10514      switch (Res.first) {
10515      default: break;
10516      case X86::AX: DestReg = X86::AL; break;
10517      case X86::DX: DestReg = X86::DL; break;
10518      case X86::CX: DestReg = X86::CL; break;
10519      case X86::BX: DestReg = X86::BL; break;
10520      }
10521      if (DestReg) {
10522        Res.first = DestReg;
10523        Res.second = X86::GR8RegisterClass;
10524      }
10525    } else if (VT == MVT::i32) {
10526      unsigned DestReg = 0;
10527      switch (Res.first) {
10528      default: break;
10529      case X86::AX: DestReg = X86::EAX; break;
10530      case X86::DX: DestReg = X86::EDX; break;
10531      case X86::CX: DestReg = X86::ECX; break;
10532      case X86::BX: DestReg = X86::EBX; break;
10533      case X86::SI: DestReg = X86::ESI; break;
10534      case X86::DI: DestReg = X86::EDI; break;
10535      case X86::BP: DestReg = X86::EBP; break;
10536      case X86::SP: DestReg = X86::ESP; break;
10537      }
10538      if (DestReg) {
10539        Res.first = DestReg;
10540        Res.second = X86::GR32RegisterClass;
10541      }
10542    } else if (VT == MVT::i64) {
10543      unsigned DestReg = 0;
10544      switch (Res.first) {
10545      default: break;
10546      case X86::AX: DestReg = X86::RAX; break;
10547      case X86::DX: DestReg = X86::RDX; break;
10548      case X86::CX: DestReg = X86::RCX; break;
10549      case X86::BX: DestReg = X86::RBX; break;
10550      case X86::SI: DestReg = X86::RSI; break;
10551      case X86::DI: DestReg = X86::RDI; break;
10552      case X86::BP: DestReg = X86::RBP; break;
10553      case X86::SP: DestReg = X86::RSP; break;
10554      }
10555      if (DestReg) {
10556        Res.first = DestReg;
10557        Res.second = X86::GR64RegisterClass;
10558      }
10559    }
10560  } else if (Res.second == X86::FR32RegisterClass ||
10561             Res.second == X86::FR64RegisterClass ||
10562             Res.second == X86::VR128RegisterClass) {
10563    // Handle references to XMM physical registers that got mapped into the
10564    // wrong class.  This can happen with constraints like {xmm0} where the
10565    // target independent register mapper will just pick the first match it can
10566    // find, ignoring the required type.
10567    if (VT == MVT::f32)
10568      Res.second = X86::FR32RegisterClass;
10569    else if (VT == MVT::f64)
10570      Res.second = X86::FR64RegisterClass;
10571    else if (X86::VR128RegisterClass->hasType(VT))
10572      Res.second = X86::VR128RegisterClass;
10573  }
10574
10575  return Res;
10576}
10577