ARMISelDAGToDAG.cpp revision 40ff01a0305403541353ac83f9f498ab67ebe944 
1//===-- ARMISelDAGToDAG.cpp - A dag to dag inst selector for ARM ----------===//
2//
3//                     The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10// This file defines an instruction selector for the ARM target.
11//
12//===----------------------------------------------------------------------===//
13
14#define DEBUG_TYPE "arm-isel"
15#include "ARM.h"
16#include "ARMAddressingModes.h"
17#include "ARMTargetMachine.h"
18#include "llvm/CallingConv.h"
19#include "llvm/Constants.h"
20#include "llvm/DerivedTypes.h"
21#include "llvm/Function.h"
22#include "llvm/Intrinsics.h"
23#include "llvm/LLVMContext.h"
24#include "llvm/CodeGen/MachineFrameInfo.h"
25#include "llvm/CodeGen/MachineFunction.h"
26#include "llvm/CodeGen/MachineInstrBuilder.h"
27#include "llvm/CodeGen/SelectionDAG.h"
28#include "llvm/CodeGen/SelectionDAGISel.h"
29#include "llvm/Target/TargetLowering.h"
30#include "llvm/Target/TargetOptions.h"
31#include "llvm/Support/CommandLine.h"
32#include "llvm/Support/Compiler.h"
33#include "llvm/Support/Debug.h"
34#include "llvm/Support/ErrorHandling.h"
35#include "llvm/Support/raw_ostream.h"
36
37using namespace llvm;
38
39static cl::opt<bool>
40DisableShifterOp("disable-shifter-op", cl::Hidden,
41  cl::desc("Disable isel of shifter-op"),
42  cl::init(false));
43
44//===--------------------------------------------------------------------===//
45/// ARMDAGToDAGISel - ARM specific code to select ARM machine
46/// instructions for SelectionDAG operations.
47///
48namespace {
49class ARMDAGToDAGISel : public SelectionDAGISel {
50  ARMBaseTargetMachine &TM;
51
52  /// Subtarget - Keep a pointer to the ARMSubtarget around so that we can
53  /// make the right decision when generating code for different targets.
54  const ARMSubtarget *Subtarget;
55
56public:
57  explicit ARMDAGToDAGISel(ARMBaseTargetMachine &tm,
58                           CodeGenOpt::Level OptLevel)
59    : SelectionDAGISel(tm, OptLevel), TM(tm),
60    Subtarget(&TM.getSubtarget<ARMSubtarget>()) {
61  }
62
63  virtual const char *getPassName() const {
64    return "ARM Instruction Selection";
65  }
66
67  /// getI32Imm - Return a target constant of type i32 with the specified
68  /// value.
69  inline SDValue getI32Imm(unsigned Imm) {
70    return CurDAG->getTargetConstant(Imm, MVT::i32);
71  }
72
73  SDNode *Select(SDNode *N);
74
75  bool SelectShifterOperandReg(SDValue N, SDValue &A,
76                               SDValue &B, SDValue &C);
77  bool SelectAddrMode2(SDValue N, SDValue &Base,
78                       SDValue &Offset, SDValue &Opc);
79  bool SelectAddrMode2Offset(SDNode *Op, SDValue N,
80                             SDValue &Offset, SDValue &Opc);
81  bool SelectAddrMode3(SDValue N, SDValue &Base,
82                       SDValue &Offset, SDValue &Opc);
83  bool SelectAddrMode3Offset(SDNode *Op, SDValue N,
84                             SDValue &Offset, SDValue &Opc);
85  bool SelectAddrMode4(SDValue N, SDValue &Addr, SDValue &Mode);
86  bool SelectAddrMode5(SDValue N, SDValue &Base,
87                       SDValue &Offset);
88  bool SelectAddrMode6(SDValue N, SDValue &Addr, SDValue &Align);
89
90  bool SelectAddrModePC(SDValue N, SDValue &Offset,
91                        SDValue &Label);
92
93  bool SelectThumbAddrModeRR(SDValue N, SDValue &Base, SDValue &Offset);
94  bool SelectThumbAddrModeRI5(SDValue N, unsigned Scale,
95                              SDValue &Base, SDValue &OffImm,
96                              SDValue &Offset);
97  bool SelectThumbAddrModeS1(SDValue N, SDValue &Base,
98                             SDValue &OffImm, SDValue &Offset);
99  bool SelectThumbAddrModeS2(SDValue N, SDValue &Base,
100                             SDValue &OffImm, SDValue &Offset);
101  bool SelectThumbAddrModeS4(SDValue N, SDValue &Base,
102                             SDValue &OffImm, SDValue &Offset);
103  bool SelectThumbAddrModeSP(SDValue N, SDValue &Base, SDValue &OffImm);
104
105  bool SelectT2ShifterOperandReg(SDValue N,
106                                 SDValue &BaseReg, SDValue &Opc);
107  bool SelectT2AddrModeImm12(SDValue N, SDValue &Base, SDValue &OffImm);
108  bool SelectT2AddrModeImm8(SDValue N, SDValue &Base,
109                            SDValue &OffImm);
110  bool SelectT2AddrModeImm8Offset(SDNode *Op, SDValue N,
111                                 SDValue &OffImm);
112  bool SelectT2AddrModeSoReg(SDValue N, SDValue &Base,
113                             SDValue &OffReg, SDValue &ShImm);
114
115  inline bool Pred_so_imm(SDNode *inN) const {
116    ConstantSDNode *N = cast<ConstantSDNode>(inN);
117    return ARM_AM::getSOImmVal(N->getZExtValue()) != -1;
118  }
119
120  inline bool Pred_t2_so_imm(SDNode *inN) const {
121    ConstantSDNode *N = cast<ConstantSDNode>(inN);
122    return ARM_AM::getT2SOImmVal(N->getZExtValue()) != -1;
123  }
124
125  // Include the pieces autogenerated from the target description.
126#include "ARMGenDAGISel.inc"
127
128private:
129  /// SelectARMIndexedLoad - Indexed (pre/post inc/dec) load matching code for
130  /// ARM.
131  SDNode *SelectARMIndexedLoad(SDNode *N);
132  SDNode *SelectT2IndexedLoad(SDNode *N);
133
134  /// SelectVLD - Select NEON load intrinsics.  NumVecs should be
135  /// 1, 2, 3 or 4.  The opcode arrays specify the instructions used for
136  /// loads of D registers and even subregs and odd subregs of Q registers.
137  /// For NumVecs <= 2, QOpcodes1 is not used.
138  SDNode *SelectVLD(SDNode *N, unsigned NumVecs, unsigned *DOpcodes,
139                    unsigned *QOpcodes0, unsigned *QOpcodes1);
140
141  /// SelectVST - Select NEON store intrinsics.  NumVecs should
142  /// be 1, 2, 3 or 4.  The opcode arrays specify the instructions used for
143  /// stores of D registers and even subregs and odd subregs of Q registers.
144  /// For NumVecs <= 2, QOpcodes1 is not used.
145  SDNode *SelectVST(SDNode *N, unsigned NumVecs, unsigned *DOpcodes,
146                    unsigned *QOpcodes0, unsigned *QOpcodes1);
147
148  /// SelectVLDSTLane - Select NEON load/store lane intrinsics.  NumVecs should
149  /// be 2, 3 or 4.  The opcode arrays specify the instructions used for
150  /// load/store of D registers and Q registers.
151  SDNode *SelectVLDSTLane(SDNode *N, bool IsLoad, unsigned NumVecs,
152                          unsigned *DOpcodes, unsigned *QOpcodes);
153
154  /// SelectVTBL - Select NEON VTBL and VTBX intrinsics.  NumVecs should be 2,
155  /// 3 or 4.  These are custom-selected so that a REG_SEQUENCE can be
156  /// generated to force the table registers to be consecutive.
157  SDNode *SelectVTBL(SDNode *N, bool IsExt, unsigned NumVecs, unsigned Opc);
158
159  /// SelectV6T2BitfieldExtractOp - Select SBFX/UBFX instructions for ARM.
160  SDNode *SelectV6T2BitfieldExtractOp(SDNode *N, bool isSigned);
161
162  /// SelectCMOVOp - Select CMOV instructions for ARM.
163  SDNode *SelectCMOVOp(SDNode *N);
164  SDNode *SelectT2CMOVShiftOp(SDNode *N, SDValue FalseVal, SDValue TrueVal,
165                              ARMCC::CondCodes CCVal, SDValue CCR,
166                              SDValue InFlag);
167  SDNode *SelectARMCMOVShiftOp(SDNode *N, SDValue FalseVal, SDValue TrueVal,
168                               ARMCC::CondCodes CCVal, SDValue CCR,
169                               SDValue InFlag);
170  SDNode *SelectT2CMOVSoImmOp(SDNode *N, SDValue FalseVal, SDValue TrueVal,
171                              ARMCC::CondCodes CCVal, SDValue CCR,
172                              SDValue InFlag);
173  SDNode *SelectARMCMOVSoImmOp(SDNode *N, SDValue FalseVal, SDValue TrueVal,
174                               ARMCC::CondCodes CCVal, SDValue CCR,
175                               SDValue InFlag);
176
177  SDNode *SelectConcatVector(SDNode *N);
178
179  /// SelectInlineAsmMemoryOperand - Implement addressing mode selection for
180  /// inline asm expressions.
181  virtual bool SelectInlineAsmMemoryOperand(const SDValue &Op,
182                                            char ConstraintCode,
183                                            std::vector<SDValue> &OutOps);
184
185  // Form pairs of consecutive S, D, or Q registers.
186  SDNode *PairSRegs(EVT VT, SDValue V0, SDValue V1);
187  SDNode *PairDRegs(EVT VT, SDValue V0, SDValue V1);
188  SDNode *PairQRegs(EVT VT, SDValue V0, SDValue V1);
189
190  // Form sequences of 4 consecutive S, D, or Q registers.
191  SDNode *QuadSRegs(EVT VT, SDValue V0, SDValue V1, SDValue V2, SDValue V3);
192  SDNode *QuadDRegs(EVT VT, SDValue V0, SDValue V1, SDValue V2, SDValue V3);
193  SDNode *QuadQRegs(EVT VT, SDValue V0, SDValue V1, SDValue V2, SDValue V3);
194};
195}
196
197/// isInt32Immediate - This method tests to see if the node is a 32-bit constant
198/// operand. If so Imm will receive the 32-bit value.
199static bool isInt32Immediate(SDNode *N, unsigned &Imm) {
200  if (N->getOpcode() == ISD::Constant && N->getValueType(0) == MVT::i32) {
201    Imm = cast<ConstantSDNode>(N)->getZExtValue();
202    return true;
203  }
204  return false;
205}
206
207// isInt32Immediate - This method tests to see if a constant operand.
208// If so Imm will receive the 32 bit value.
209static bool isInt32Immediate(SDValue N, unsigned &Imm) {
210  return isInt32Immediate(N.getNode(), Imm);
211}
212
213// isOpcWithIntImmediate - This method tests to see if the node is a specific
214// opcode and that it has a immediate integer right operand.
215// If so Imm will receive the 32 bit value.
216static bool isOpcWithIntImmediate(SDNode *N, unsigned Opc, unsigned& Imm) {
217  return N->getOpcode() == Opc &&
218         isInt32Immediate(N->getOperand(1).getNode(), Imm);
219}
220
221
222bool ARMDAGToDAGISel::SelectShifterOperandReg(SDValue N,
223                                              SDValue &BaseReg,
224                                              SDValue &ShReg,
225                                              SDValue &Opc) {
226  if (DisableShifterOp)
227    return false;
228
229  ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(N);
230
231  // Don't match base register only case. That is matched to a separate
232  // lower complexity pattern with explicit register operand.
233  if (ShOpcVal == ARM_AM::no_shift) return false;
234
235  BaseReg = N.getOperand(0);
236  unsigned ShImmVal = 0;
237  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
238    ShReg = CurDAG->getRegister(0, MVT::i32);
239    ShImmVal = RHS->getZExtValue() & 31;
240  } else {
241    ShReg = N.getOperand(1);
242  }
243  Opc = CurDAG->getTargetConstant(ARM_AM::getSORegOpc(ShOpcVal, ShImmVal),
244                                  MVT::i32);
245  return true;
246}
247
248bool ARMDAGToDAGISel::SelectAddrMode2(SDValue N,
249                                      SDValue &Base, SDValue &Offset,
250                                      SDValue &Opc) {
251  if (N.getOpcode() == ISD::MUL) {
252    if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
253      // X * [3,5,9] -> X + X * [2,4,8] etc.
254      int RHSC = (int)RHS->getZExtValue();
255      if (RHSC & 1) {
256        RHSC = RHSC & ~1;
257        ARM_AM::AddrOpc AddSub = ARM_AM::add;
258        if (RHSC < 0) {
259          AddSub = ARM_AM::sub;
260          RHSC = - RHSC;
261        }
262        if (isPowerOf2_32(RHSC)) {
263          unsigned ShAmt = Log2_32(RHSC);
264          Base = Offset = N.getOperand(0);
265          Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, ShAmt,
266                                                            ARM_AM::lsl),
267                                          MVT::i32);
268          return true;
269        }
270      }
271    }
272  }
273
274  if (N.getOpcode() != ISD::ADD && N.getOpcode() != ISD::SUB) {
275    Base = N;
276    if (N.getOpcode() == ISD::FrameIndex) {
277      int FI = cast<FrameIndexSDNode>(N)->getIndex();
278      Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
279    } else if (N.getOpcode() == ARMISD::Wrapper &&
280               !(Subtarget->useMovt() &&
281                 N.getOperand(0).getOpcode() == ISD::TargetGlobalAddress)) {
282      Base = N.getOperand(0);
283    }
284    Offset = CurDAG->getRegister(0, MVT::i32);
285    Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(ARM_AM::add, 0,
286                                                      ARM_AM::no_shift),
287                                    MVT::i32);
288    return true;
289  }
290
291  // Match simple R +/- imm12 operands.
292  if (N.getOpcode() == ISD::ADD)
293    if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
294      int RHSC = (int)RHS->getZExtValue();
295      if ((RHSC >= 0 && RHSC < 0x1000) ||
296          (RHSC < 0 && RHSC > -0x1000)) { // 12 bits.
297        Base = N.getOperand(0);
298        if (Base.getOpcode() == ISD::FrameIndex) {
299          int FI = cast<FrameIndexSDNode>(Base)->getIndex();
300          Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
301        }
302        Offset = CurDAG->getRegister(0, MVT::i32);
303
304        ARM_AM::AddrOpc AddSub = ARM_AM::add;
305        if (RHSC < 0) {
306          AddSub = ARM_AM::sub;
307          RHSC = - RHSC;
308        }
309        Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, RHSC,
310                                                          ARM_AM::no_shift),
311                                        MVT::i32);
312        return true;
313      }
314    }
315
316  // Otherwise this is R +/- [possibly shifted] R.
317  ARM_AM::AddrOpc AddSub = N.getOpcode() == ISD::ADD ? ARM_AM::add:ARM_AM::sub;
318  ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(N.getOperand(1));
319  unsigned ShAmt = 0;
320
321  Base   = N.getOperand(0);
322  Offset = N.getOperand(1);
323
324  if (ShOpcVal != ARM_AM::no_shift) {
325    // Check to see if the RHS of the shift is a constant, if not, we can't fold
326    // it.
327    if (ConstantSDNode *Sh =
328           dyn_cast<ConstantSDNode>(N.getOperand(1).getOperand(1))) {
329      ShAmt = Sh->getZExtValue();
330      Offset = N.getOperand(1).getOperand(0);
331    } else {
332      ShOpcVal = ARM_AM::no_shift;
333    }
334  }
335
336  // Try matching (R shl C) + (R).
337  if (N.getOpcode() == ISD::ADD && ShOpcVal == ARM_AM::no_shift) {
338    ShOpcVal = ARM_AM::getShiftOpcForNode(N.getOperand(0));
339    if (ShOpcVal != ARM_AM::no_shift) {
340      // Check to see if the RHS of the shift is a constant, if not, we can't
341      // fold it.
342      if (ConstantSDNode *Sh =
343          dyn_cast<ConstantSDNode>(N.getOperand(0).getOperand(1))) {
344        ShAmt = Sh->getZExtValue();
345        Offset = N.getOperand(0).getOperand(0);
346        Base = N.getOperand(1);
347      } else {
348        ShOpcVal = ARM_AM::no_shift;
349      }
350    }
351  }
352
353  Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, ShAmt, ShOpcVal),
354                                  MVT::i32);
355  return true;
356}
357
358bool ARMDAGToDAGISel::SelectAddrMode2Offset(SDNode *Op, SDValue N,
359                                            SDValue &Offset, SDValue &Opc) {
360  unsigned Opcode = Op->getOpcode();
361  ISD::MemIndexedMode AM = (Opcode == ISD::LOAD)
362    ? cast<LoadSDNode>(Op)->getAddressingMode()
363    : cast<StoreSDNode>(Op)->getAddressingMode();
364  ARM_AM::AddrOpc AddSub = (AM == ISD::PRE_INC || AM == ISD::POST_INC)
365    ? ARM_AM::add : ARM_AM::sub;
366  if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(N)) {
367    int Val = (int)C->getZExtValue();
368    if (Val >= 0 && Val < 0x1000) { // 12 bits.
369      Offset = CurDAG->getRegister(0, MVT::i32);
370      Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, Val,
371                                                        ARM_AM::no_shift),
372                                      MVT::i32);
373      return true;
374    }
375  }
376
377  Offset = N;
378  ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(N);
379  unsigned ShAmt = 0;
380  if (ShOpcVal != ARM_AM::no_shift) {
381    // Check to see if the RHS of the shift is a constant, if not, we can't fold
382    // it.
383    if (ConstantSDNode *Sh = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
384      ShAmt = Sh->getZExtValue();
385      Offset = N.getOperand(0);
386    } else {
387      ShOpcVal = ARM_AM::no_shift;
388    }
389  }
390
391  Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, ShAmt, ShOpcVal),
392                                  MVT::i32);
393  return true;
394}
395
396
397bool ARMDAGToDAGISel::SelectAddrMode3(SDValue N,
398                                      SDValue &Base, SDValue &Offset,
399                                      SDValue &Opc) {
400  if (N.getOpcode() == ISD::SUB) {
401    // X - C  is canonicalize to X + -C, no need to handle it here.
402    Base = N.getOperand(0);
403    Offset = N.getOperand(1);
404    Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(ARM_AM::sub, 0),MVT::i32);
405    return true;
406  }
407
408  if (N.getOpcode() != ISD::ADD) {
409    Base = N;
410    if (N.getOpcode() == ISD::FrameIndex) {
411      int FI = cast<FrameIndexSDNode>(N)->getIndex();
412      Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
413    }
414    Offset = CurDAG->getRegister(0, MVT::i32);
415    Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(ARM_AM::add, 0),MVT::i32);
416    return true;
417  }
418
419  // If the RHS is +/- imm8, fold into addr mode.
420  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
421    int RHSC = (int)RHS->getZExtValue();
422    if ((RHSC >= 0 && RHSC < 256) ||
423        (RHSC < 0 && RHSC > -256)) { // note -256 itself isn't allowed.
424      Base = N.getOperand(0);
425      if (Base.getOpcode() == ISD::FrameIndex) {
426        int FI = cast<FrameIndexSDNode>(Base)->getIndex();
427        Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
428      }
429      Offset = CurDAG->getRegister(0, MVT::i32);
430
431      ARM_AM::AddrOpc AddSub = ARM_AM::add;
432      if (RHSC < 0) {
433        AddSub = ARM_AM::sub;
434        RHSC = - RHSC;
435      }
436      Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(AddSub, RHSC),MVT::i32);
437      return true;
438    }
439  }
440
441  Base = N.getOperand(0);
442  Offset = N.getOperand(1);
443  Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(ARM_AM::add, 0), MVT::i32);
444  return true;
445}
446
447bool ARMDAGToDAGISel::SelectAddrMode3Offset(SDNode *Op, SDValue N,
448                                            SDValue &Offset, SDValue &Opc) {
449  unsigned Opcode = Op->getOpcode();
450  ISD::MemIndexedMode AM = (Opcode == ISD::LOAD)
451    ? cast<LoadSDNode>(Op)->getAddressingMode()
452    : cast<StoreSDNode>(Op)->getAddressingMode();
453  ARM_AM::AddrOpc AddSub = (AM == ISD::PRE_INC || AM == ISD::POST_INC)
454    ? ARM_AM::add : ARM_AM::sub;
455  if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(N)) {
456    int Val = (int)C->getZExtValue();
457    if (Val >= 0 && Val < 256) {
458      Offset = CurDAG->getRegister(0, MVT::i32);
459      Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(AddSub, Val), MVT::i32);
460      return true;
461    }
462  }
463
464  Offset = N;
465  Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(AddSub, 0), MVT::i32);
466  return true;
467}
468
469bool ARMDAGToDAGISel::SelectAddrMode4(SDValue N, SDValue &Addr, SDValue &Mode) {
470  Addr = N;
471  Mode = CurDAG->getTargetConstant(ARM_AM::getAM4ModeImm(ARM_AM::ia), MVT::i32);
472  return true;
473}
474
475bool ARMDAGToDAGISel::SelectAddrMode5(SDValue N,
476                                      SDValue &Base, SDValue &Offset) {
477  if (N.getOpcode() != ISD::ADD) {
478    Base = N;
479    if (N.getOpcode() == ISD::FrameIndex) {
480      int FI = cast<FrameIndexSDNode>(N)->getIndex();
481      Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
482    } else if (N.getOpcode() == ARMISD::Wrapper &&
483               !(Subtarget->useMovt() &&
484                 N.getOperand(0).getOpcode() == ISD::TargetGlobalAddress)) {
485      Base = N.getOperand(0);
486    }
487    Offset = CurDAG->getTargetConstant(ARM_AM::getAM5Opc(ARM_AM::add, 0),
488                                       MVT::i32);
489    return true;
490  }
491
492  // If the RHS is +/- imm8, fold into addr mode.
493  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
494    int RHSC = (int)RHS->getZExtValue();
495    if ((RHSC & 3) == 0) {  // The constant is implicitly multiplied by 4.
496      RHSC >>= 2;
497      if ((RHSC >= 0 && RHSC < 256) ||
498          (RHSC < 0 && RHSC > -256)) { // note -256 itself isn't allowed.
499        Base = N.getOperand(0);
500        if (Base.getOpcode() == ISD::FrameIndex) {
501          int FI = cast<FrameIndexSDNode>(Base)->getIndex();
502          Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
503        }
504
505        ARM_AM::AddrOpc AddSub = ARM_AM::add;
506        if (RHSC < 0) {
507          AddSub = ARM_AM::sub;
508          RHSC = - RHSC;
509        }
510        Offset = CurDAG->getTargetConstant(ARM_AM::getAM5Opc(AddSub, RHSC),
511                                           MVT::i32);
512        return true;
513      }
514    }
515  }
516
517  Base = N;
518  Offset = CurDAG->getTargetConstant(ARM_AM::getAM5Opc(ARM_AM::add, 0),
519                                     MVT::i32);
520  return true;
521}
522
523bool ARMDAGToDAGISel::SelectAddrMode6(SDValue N, SDValue &Addr, SDValue &Align){
524  Addr = N;
525  // Default to no alignment.
526  Align = CurDAG->getTargetConstant(0, MVT::i32);
527  return true;
528}
529
530bool ARMDAGToDAGISel::SelectAddrModePC(SDValue N,
531                                       SDValue &Offset, SDValue &Label) {
532  if (N.getOpcode() == ARMISD::PIC_ADD && N.hasOneUse()) {
533    Offset = N.getOperand(0);
534    SDValue N1 = N.getOperand(1);
535    Label  = CurDAG->getTargetConstant(cast<ConstantSDNode>(N1)->getZExtValue(),
536                                       MVT::i32);
537    return true;
538  }
539  return false;
540}
541
542bool ARMDAGToDAGISel::SelectThumbAddrModeRR(SDValue N,
543                                            SDValue &Base, SDValue &Offset){
544  // FIXME dl should come from the parent load or store, not the address
545  if (N.getOpcode() != ISD::ADD) {
546    ConstantSDNode *NC = dyn_cast<ConstantSDNode>(N);
547    if (!NC || !NC->isNullValue())
548      return false;
549
550    Base = Offset = N;
551    return true;
552  }
553
554  Base = N.getOperand(0);
555  Offset = N.getOperand(1);
556  return true;
557}
558
559bool
560ARMDAGToDAGISel::SelectThumbAddrModeRI5(SDValue N,
561                                        unsigned Scale, SDValue &Base,
562                                        SDValue &OffImm, SDValue &Offset) {
563  if (Scale == 4) {
564    SDValue TmpBase, TmpOffImm;
565    if (SelectThumbAddrModeSP(N, TmpBase, TmpOffImm))
566      return false;  // We want to select tLDRspi / tSTRspi instead.
567    if (N.getOpcode() == ARMISD::Wrapper &&
568        N.getOperand(0).getOpcode() == ISD::TargetConstantPool)
569      return false;  // We want to select tLDRpci instead.
570  }
571
572  if (N.getOpcode() != ISD::ADD) {
573    if (N.getOpcode() == ARMISD::Wrapper &&
574        !(Subtarget->useMovt() &&
575          N.getOperand(0).getOpcode() == ISD::TargetGlobalAddress)) {
576      Base = N.getOperand(0);
577    } else
578      Base = N;
579
580    Offset = CurDAG->getRegister(0, MVT::i32);
581    OffImm = CurDAG->getTargetConstant(0, MVT::i32);
582    return true;
583  }
584
585  // Thumb does not have [sp, r] address mode.
586  RegisterSDNode *LHSR = dyn_cast<RegisterSDNode>(N.getOperand(0));
587  RegisterSDNode *RHSR = dyn_cast<RegisterSDNode>(N.getOperand(1));
588  if ((LHSR && LHSR->getReg() == ARM::SP) ||
589      (RHSR && RHSR->getReg() == ARM::SP)) {
590    Base = N;
591    Offset = CurDAG->getRegister(0, MVT::i32);
592    OffImm = CurDAG->getTargetConstant(0, MVT::i32);
593    return true;
594  }
595
596  // If the RHS is + imm5 * scale, fold into addr mode.
597  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
598    int RHSC = (int)RHS->getZExtValue();
599    if ((RHSC & (Scale-1)) == 0) {  // The constant is implicitly multiplied.
600      RHSC /= Scale;
601      if (RHSC >= 0 && RHSC < 32) {
602        Base = N.getOperand(0);
603        Offset = CurDAG->getRegister(0, MVT::i32);
604        OffImm = CurDAG->getTargetConstant(RHSC, MVT::i32);
605        return true;
606      }
607    }
608  }
609
610  Base = N.getOperand(0);
611  Offset = N.getOperand(1);
612  OffImm = CurDAG->getTargetConstant(0, MVT::i32);
613  return true;
614}
615
616bool ARMDAGToDAGISel::SelectThumbAddrModeS1(SDValue N,
617                                            SDValue &Base, SDValue &OffImm,
618                                            SDValue &Offset) {
619  return SelectThumbAddrModeRI5(N, 1, Base, OffImm, Offset);
620}
621
622bool ARMDAGToDAGISel::SelectThumbAddrModeS2(SDValue N,
623                                            SDValue &Base, SDValue &OffImm,
624                                            SDValue &Offset) {
625  return SelectThumbAddrModeRI5(N, 2, Base, OffImm, Offset);
626}
627
628bool ARMDAGToDAGISel::SelectThumbAddrModeS4(SDValue N,
629                                            SDValue &Base, SDValue &OffImm,
630                                            SDValue &Offset) {
631  return SelectThumbAddrModeRI5(N, 4, Base, OffImm, Offset);
632}
633
634bool ARMDAGToDAGISel::SelectThumbAddrModeSP(SDValue N,
635                                            SDValue &Base, SDValue &OffImm) {
636  if (N.getOpcode() == ISD::FrameIndex) {
637    int FI = cast<FrameIndexSDNode>(N)->getIndex();
638    Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
639    OffImm = CurDAG->getTargetConstant(0, MVT::i32);
640    return true;
641  }
642
643  if (N.getOpcode() != ISD::ADD)
644    return false;
645
646  RegisterSDNode *LHSR = dyn_cast<RegisterSDNode>(N.getOperand(0));
647  if (N.getOperand(0).getOpcode() == ISD::FrameIndex ||
648      (LHSR && LHSR->getReg() == ARM::SP)) {
649    // If the RHS is + imm8 * scale, fold into addr mode.
650    if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
651      int RHSC = (int)RHS->getZExtValue();
652      if ((RHSC & 3) == 0) {  // The constant is implicitly multiplied.
653        RHSC >>= 2;
654        if (RHSC >= 0 && RHSC < 256) {
655          Base = N.getOperand(0);
656          if (Base.getOpcode() == ISD::FrameIndex) {
657            int FI = cast<FrameIndexSDNode>(Base)->getIndex();
658            Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
659          }
660          OffImm = CurDAG->getTargetConstant(RHSC, MVT::i32);
661          return true;
662        }
663      }
664    }
665  }
666
667  return false;
668}
669
670bool ARMDAGToDAGISel::SelectT2ShifterOperandReg(SDValue N, SDValue &BaseReg,
671                                                SDValue &Opc) {
672  if (DisableShifterOp)
673    return false;
674
675  ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(N);
676
677  // Don't match base register only case. That is matched to a separate
678  // lower complexity pattern with explicit register operand.
679  if (ShOpcVal == ARM_AM::no_shift) return false;
680
681  BaseReg = N.getOperand(0);
682  unsigned ShImmVal = 0;
683  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
684    ShImmVal = RHS->getZExtValue() & 31;
685    Opc = getI32Imm(ARM_AM::getSORegOpc(ShOpcVal, ShImmVal));
686    return true;
687  }
688
689  return false;
690}
691
692bool ARMDAGToDAGISel::SelectT2AddrModeImm12(SDValue N,
693                                            SDValue &Base, SDValue &OffImm) {
694  // Match simple R + imm12 operands.
695
696  // Base only.
697  if (N.getOpcode() != ISD::ADD && N.getOpcode() != ISD::SUB) {
698    if (N.getOpcode() == ISD::FrameIndex) {
699      // Match frame index...
700      int FI = cast<FrameIndexSDNode>(N)->getIndex();
701      Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
702      OffImm  = CurDAG->getTargetConstant(0, MVT::i32);
703      return true;
704    } else if (N.getOpcode() == ARMISD::Wrapper &&
705               !(Subtarget->useMovt() &&
706                 N.getOperand(0).getOpcode() == ISD::TargetGlobalAddress)) {
707      Base = N.getOperand(0);
708      if (Base.getOpcode() == ISD::TargetConstantPool)
709        return false;  // We want to select t2LDRpci instead.
710    } else
711      Base = N;
712    OffImm  = CurDAG->getTargetConstant(0, MVT::i32);
713    return true;
714  }
715
716  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
717    if (SelectT2AddrModeImm8(N, Base, OffImm))
718      // Let t2LDRi8 handle (R - imm8).
719      return false;
720
721    int RHSC = (int)RHS->getZExtValue();
722    if (N.getOpcode() == ISD::SUB)
723      RHSC = -RHSC;
724
725    if (RHSC >= 0 && RHSC < 0x1000) { // 12 bits (unsigned)
726      Base   = N.getOperand(0);
727      if (Base.getOpcode() == ISD::FrameIndex) {
728        int FI = cast<FrameIndexSDNode>(Base)->getIndex();
729        Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
730      }
731      OffImm = CurDAG->getTargetConstant(RHSC, MVT::i32);
732      return true;
733    }
734  }
735
736  // Base only.
737  Base = N;
738  OffImm  = CurDAG->getTargetConstant(0, MVT::i32);
739  return true;
740}
741
742bool ARMDAGToDAGISel::SelectT2AddrModeImm8(SDValue N,
743                                           SDValue &Base, SDValue &OffImm) {
744  // Match simple R - imm8 operands.
745  if (N.getOpcode() == ISD::ADD || N.getOpcode() == ISD::SUB) {
746    if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
747      int RHSC = (int)RHS->getSExtValue();
748      if (N.getOpcode() == ISD::SUB)
749        RHSC = -RHSC;
750
751      if ((RHSC >= -255) && (RHSC < 0)) { // 8 bits (always negative)
752        Base = N.getOperand(0);
753        if (Base.getOpcode() == ISD::FrameIndex) {
754          int FI = cast<FrameIndexSDNode>(Base)->getIndex();
755          Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
756        }
757        OffImm = CurDAG->getTargetConstant(RHSC, MVT::i32);
758        return true;
759      }
760    }
761  }
762
763  return false;
764}
765
766bool ARMDAGToDAGISel::SelectT2AddrModeImm8Offset(SDNode *Op, SDValue N,
767                                                 SDValue &OffImm){
768  unsigned Opcode = Op->getOpcode();
769  ISD::MemIndexedMode AM = (Opcode == ISD::LOAD)
770    ? cast<LoadSDNode>(Op)->getAddressingMode()
771    : cast<StoreSDNode>(Op)->getAddressingMode();
772  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N)) {
773    int RHSC = (int)RHS->getZExtValue();
774    if (RHSC >= 0 && RHSC < 0x100) { // 8 bits.
775      OffImm = ((AM == ISD::PRE_INC) || (AM == ISD::POST_INC))
776        ? CurDAG->getTargetConstant(RHSC, MVT::i32)
777        : CurDAG->getTargetConstant(-RHSC, MVT::i32);
778      return true;
779    }
780  }
781
782  return false;
783}
784
785bool ARMDAGToDAGISel::SelectT2AddrModeSoReg(SDValue N,
786                                            SDValue &Base,
787                                            SDValue &OffReg, SDValue &ShImm) {
788  // (R - imm8) should be handled by t2LDRi8. The rest are handled by t2LDRi12.
789  if (N.getOpcode() != ISD::ADD)
790    return false;
791
792  // Leave (R + imm12) for t2LDRi12, (R - imm8) for t2LDRi8.
793  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
794    int RHSC = (int)RHS->getZExtValue();
795    if (RHSC >= 0 && RHSC < 0x1000) // 12 bits (unsigned)
796      return false;
797    else if (RHSC < 0 && RHSC >= -255) // 8 bits
798      return false;
799  }
800
801  // Look for (R + R) or (R + (R << [1,2,3])).
802  unsigned ShAmt = 0;
803  Base   = N.getOperand(0);
804  OffReg = N.getOperand(1);
805
806  // Swap if it is ((R << c) + R).
807  ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(OffReg);
808  if (ShOpcVal != ARM_AM::lsl) {
809    ShOpcVal = ARM_AM::getShiftOpcForNode(Base);
810    if (ShOpcVal == ARM_AM::lsl)
811      std::swap(Base, OffReg);
812  }
813
814  if (ShOpcVal == ARM_AM::lsl) {
815    // Check to see if the RHS of the shift is a constant, if not, we can't fold
816    // it.
817    if (ConstantSDNode *Sh = dyn_cast<ConstantSDNode>(OffReg.getOperand(1))) {
818      ShAmt = Sh->getZExtValue();
819      if (ShAmt >= 4) {
820        ShAmt = 0;
821        ShOpcVal = ARM_AM::no_shift;
822      } else
823        OffReg = OffReg.getOperand(0);
824    } else {
825      ShOpcVal = ARM_AM::no_shift;
826    }
827  }
828
829  ShImm = CurDAG->getTargetConstant(ShAmt, MVT::i32);
830
831  return true;
832}
833
834//===--------------------------------------------------------------------===//
835
836/// getAL - Returns a ARMCC::AL immediate node.
837static inline SDValue getAL(SelectionDAG *CurDAG) {
838  return CurDAG->getTargetConstant((uint64_t)ARMCC::AL, MVT::i32);
839}
840
841SDNode *ARMDAGToDAGISel::SelectARMIndexedLoad(SDNode *N) {
842  LoadSDNode *LD = cast<LoadSDNode>(N);
843  ISD::MemIndexedMode AM = LD->getAddressingMode();
844  if (AM == ISD::UNINDEXED)
845    return NULL;
846
847  EVT LoadedVT = LD->getMemoryVT();
848  SDValue Offset, AMOpc;
849  bool isPre = (AM == ISD::PRE_INC) || (AM == ISD::PRE_DEC);
850  unsigned Opcode = 0;
851  bool Match = false;
852  if (LoadedVT == MVT::i32 &&
853      SelectAddrMode2Offset(N, LD->getOffset(), Offset, AMOpc)) {
854    Opcode = isPre ? ARM::LDR_PRE : ARM::LDR_POST;
855    Match = true;
856  } else if (LoadedVT == MVT::i16 &&
857             SelectAddrMode3Offset(N, LD->getOffset(), Offset, AMOpc)) {
858    Match = true;
859    Opcode = (LD->getExtensionType() == ISD::SEXTLOAD)
860      ? (isPre ? ARM::LDRSH_PRE : ARM::LDRSH_POST)
861      : (isPre ? ARM::LDRH_PRE : ARM::LDRH_POST);
862  } else if (LoadedVT == MVT::i8 || LoadedVT == MVT::i1) {
863    if (LD->getExtensionType() == ISD::SEXTLOAD) {
864      if (SelectAddrMode3Offset(N, LD->getOffset(), Offset, AMOpc)) {
865        Match = true;
866        Opcode = isPre ? ARM::LDRSB_PRE : ARM::LDRSB_POST;
867      }
868    } else {
869      if (SelectAddrMode2Offset(N, LD->getOffset(), Offset, AMOpc)) {
870        Match = true;
871        Opcode = isPre ? ARM::LDRB_PRE : ARM::LDRB_POST;
872      }
873    }
874  }
875
876  if (Match) {
877    SDValue Chain = LD->getChain();
878    SDValue Base = LD->getBasePtr();
879    SDValue Ops[]= { Base, Offset, AMOpc, getAL(CurDAG),
880                     CurDAG->getRegister(0, MVT::i32), Chain };
881    return CurDAG->getMachineNode(Opcode, N->getDebugLoc(), MVT::i32, MVT::i32,
882                                  MVT::Other, Ops, 6);
883  }
884
885  return NULL;
886}
887
888SDNode *ARMDAGToDAGISel::SelectT2IndexedLoad(SDNode *N) {
889  LoadSDNode *LD = cast<LoadSDNode>(N);
890  ISD::MemIndexedMode AM = LD->getAddressingMode();
891  if (AM == ISD::UNINDEXED)
892    return NULL;
893
894  EVT LoadedVT = LD->getMemoryVT();
895  bool isSExtLd = LD->getExtensionType() == ISD::SEXTLOAD;
896  SDValue Offset;
897  bool isPre = (AM == ISD::PRE_INC) || (AM == ISD::PRE_DEC);
898  unsigned Opcode = 0;
899  bool Match = false;
900  if (SelectT2AddrModeImm8Offset(N, LD->getOffset(), Offset)) {
901    switch (LoadedVT.getSimpleVT().SimpleTy) {
902    case MVT::i32:
903      Opcode = isPre ? ARM::t2LDR_PRE : ARM::t2LDR_POST;
904      break;
905    case MVT::i16:
906      if (isSExtLd)
907        Opcode = isPre ? ARM::t2LDRSH_PRE : ARM::t2LDRSH_POST;
908      else
909        Opcode = isPre ? ARM::t2LDRH_PRE : ARM::t2LDRH_POST;
910      break;
911    case MVT::i8:
912    case MVT::i1:
913      if (isSExtLd)
914        Opcode = isPre ? ARM::t2LDRSB_PRE : ARM::t2LDRSB_POST;
915      else
916        Opcode = isPre ? ARM::t2LDRB_PRE : ARM::t2LDRB_POST;
917      break;
918    default:
919      return NULL;
920    }
921    Match = true;
922  }
923
924  if (Match) {
925    SDValue Chain = LD->getChain();
926    SDValue Base = LD->getBasePtr();
927    SDValue Ops[]= { Base, Offset, getAL(CurDAG),
928                     CurDAG->getRegister(0, MVT::i32), Chain };
929    return CurDAG->getMachineNode(Opcode, N->getDebugLoc(), MVT::i32, MVT::i32,
930                                  MVT::Other, Ops, 5);
931  }
932
933  return NULL;
934}
935
936/// PairSRegs - Form a D register from a pair of S registers.
937///
938SDNode *ARMDAGToDAGISel::PairSRegs(EVT VT, SDValue V0, SDValue V1) {
939  DebugLoc dl = V0.getNode()->getDebugLoc();
940  SDValue SubReg0 = CurDAG->getTargetConstant(ARM::ssub_0, MVT::i32);
941  SDValue SubReg1 = CurDAG->getTargetConstant(ARM::ssub_1, MVT::i32);
942  const SDValue Ops[] = { V0, SubReg0, V1, SubReg1 };
943  return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops, 4);
944}
945
946/// PairDRegs - Form a quad register from a pair of D registers.
947///
948SDNode *ARMDAGToDAGISel::PairDRegs(EVT VT, SDValue V0, SDValue V1) {
949  DebugLoc dl = V0.getNode()->getDebugLoc();
950  SDValue SubReg0 = CurDAG->getTargetConstant(ARM::dsub_0, MVT::i32);
951  SDValue SubReg1 = CurDAG->getTargetConstant(ARM::dsub_1, MVT::i32);
952  const SDValue Ops[] = { V0, SubReg0, V1, SubReg1 };
953  return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops, 4);
954}
955
956/// PairQRegs - Form 4 consecutive D registers from a pair of Q registers.
957///
958SDNode *ARMDAGToDAGISel::PairQRegs(EVT VT, SDValue V0, SDValue V1) {
959  DebugLoc dl = V0.getNode()->getDebugLoc();
960  SDValue SubReg0 = CurDAG->getTargetConstant(ARM::qsub_0, MVT::i32);
961  SDValue SubReg1 = CurDAG->getTargetConstant(ARM::qsub_1, MVT::i32);
962  const SDValue Ops[] = { V0, SubReg0, V1, SubReg1 };
963  return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops, 4);
964}
965
966/// QuadSRegs - Form 4 consecutive S registers.
967///
968SDNode *ARMDAGToDAGISel::QuadSRegs(EVT VT, SDValue V0, SDValue V1,
969                                   SDValue V2, SDValue V3) {
970  DebugLoc dl = V0.getNode()->getDebugLoc();
971  SDValue SubReg0 = CurDAG->getTargetConstant(ARM::ssub_0, MVT::i32);
972  SDValue SubReg1 = CurDAG->getTargetConstant(ARM::ssub_1, MVT::i32);
973  SDValue SubReg2 = CurDAG->getTargetConstant(ARM::ssub_2, MVT::i32);
974  SDValue SubReg3 = CurDAG->getTargetConstant(ARM::ssub_3, MVT::i32);
975  const SDValue Ops[] = { V0, SubReg0, V1, SubReg1, V2, SubReg2, V3, SubReg3 };
976  return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops, 8);
977}
978
979/// QuadDRegs - Form 4 consecutive D registers.
980///
981SDNode *ARMDAGToDAGISel::QuadDRegs(EVT VT, SDValue V0, SDValue V1,
982                                   SDValue V2, SDValue V3) {
983  DebugLoc dl = V0.getNode()->getDebugLoc();
984  SDValue SubReg0 = CurDAG->getTargetConstant(ARM::dsub_0, MVT::i32);
985  SDValue SubReg1 = CurDAG->getTargetConstant(ARM::dsub_1, MVT::i32);
986  SDValue SubReg2 = CurDAG->getTargetConstant(ARM::dsub_2, MVT::i32);
987  SDValue SubReg3 = CurDAG->getTargetConstant(ARM::dsub_3, MVT::i32);
988  const SDValue Ops[] = { V0, SubReg0, V1, SubReg1, V2, SubReg2, V3, SubReg3 };
989  return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops, 8);
990}
991
992/// QuadQRegs - Form 4 consecutive Q registers.
993///
994SDNode *ARMDAGToDAGISel::QuadQRegs(EVT VT, SDValue V0, SDValue V1,
995                                   SDValue V2, SDValue V3) {
996  DebugLoc dl = V0.getNode()->getDebugLoc();
997  SDValue SubReg0 = CurDAG->getTargetConstant(ARM::qsub_0, MVT::i32);
998  SDValue SubReg1 = CurDAG->getTargetConstant(ARM::qsub_1, MVT::i32);
999  SDValue SubReg2 = CurDAG->getTargetConstant(ARM::qsub_2, MVT::i32);
1000  SDValue SubReg3 = CurDAG->getTargetConstant(ARM::qsub_3, MVT::i32);
1001  const SDValue Ops[] = { V0, SubReg0, V1, SubReg1, V2, SubReg2, V3, SubReg3 };
1002  return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops, 8);
1003}
1004
1005SDNode *ARMDAGToDAGISel::SelectVLD(SDNode *N, unsigned NumVecs,
1006                                   unsigned *DOpcodes, unsigned *QOpcodes0,
1007                                   unsigned *QOpcodes1) {
1008  assert(NumVecs >= 1 && NumVecs <= 4 && "VLD NumVecs out-of-range");
1009  DebugLoc dl = N->getDebugLoc();
1010
1011  SDValue MemAddr, Align;
1012  if (!SelectAddrMode6(N->getOperand(2), MemAddr, Align))
1013    return NULL;
1014
1015  SDValue Chain = N->getOperand(0);
1016  EVT VT = N->getValueType(0);
1017  bool is64BitVector = VT.is64BitVector();
1018
1019  // Set the alignment.  The supported values depend on the number of
1020  // registers being loaded.
1021  unsigned NumRegs = NumVecs;
1022  if (!is64BitVector && NumVecs < 3)
1023    NumRegs *= 2;
1024  unsigned Alignment = cast<MemIntrinsicSDNode>(N)->getAlignment();
1025  if (Alignment >= 32 && NumRegs == 4)
1026    Alignment = 32;
1027  else if (Alignment >= 16 && (NumRegs == 2 || NumRegs == 4))
1028    Alignment = 16;
1029  else if (Alignment >= 8)
1030    Alignment = 8;
1031  else
1032    Alignment = 0;
1033  Align = CurDAG->getTargetConstant(Alignment, MVT::i32);
1034
1035  unsigned OpcodeIndex;
1036  switch (VT.getSimpleVT().SimpleTy) {
1037  default: llvm_unreachable("unhandled vld type");
1038    // Double-register operations:
1039  case MVT::v8i8:  OpcodeIndex = 0; break;
1040  case MVT::v4i16: OpcodeIndex = 1; break;
1041  case MVT::v2f32:
1042  case MVT::v2i32: OpcodeIndex = 2; break;
1043  case MVT::v1i64: OpcodeIndex = 3; break;
1044    // Quad-register operations:
1045  case MVT::v16i8: OpcodeIndex = 0; break;
1046  case MVT::v8i16: OpcodeIndex = 1; break;
1047  case MVT::v4f32:
1048  case MVT::v4i32: OpcodeIndex = 2; break;
1049  case MVT::v2i64: OpcodeIndex = 3;
1050    assert(NumVecs == 1 && "v2i64 type only supported for VLD1");
1051    break;
1052  }
1053
1054  EVT ResTy;
1055  if (NumVecs == 1)
1056    ResTy = VT;
1057  else {
1058    unsigned ResTyElts = (NumVecs == 3) ? 4 : NumVecs;
1059    if (!is64BitVector)
1060      ResTyElts *= 2;
1061    ResTy = EVT::getVectorVT(*CurDAG->getContext(), MVT::i64, ResTyElts);
1062  }
1063
1064  SDValue Pred = getAL(CurDAG);
1065  SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
1066  SDValue SuperReg;
1067  if (is64BitVector) {
1068    unsigned Opc = DOpcodes[OpcodeIndex];
1069    const SDValue Ops[] = { MemAddr, Align, Pred, Reg0, Chain };
1070    SDNode *VLd = CurDAG->getMachineNode(Opc, dl, ResTy, MVT::Other, Ops, 5);
1071    if (NumVecs == 1)
1072      return VLd;
1073
1074    SuperReg = SDValue(VLd, 0);
1075    assert(ARM::dsub_7 == ARM::dsub_0+7 && "Unexpected subreg numbering");
1076    for (unsigned Vec = 0; Vec < NumVecs; ++Vec) {
1077      SDValue D = CurDAG->getTargetExtractSubreg(ARM::dsub_0+Vec,
1078                                                 dl, VT, SuperReg);
1079      ReplaceUses(SDValue(N, Vec), D);
1080    }
1081    ReplaceUses(SDValue(N, NumVecs), SDValue(VLd, 1));
1082    return NULL;
1083  }
1084
1085  if (NumVecs <= 2) {
1086    // Quad registers are directly supported for VLD1 and VLD2,
1087    // loading pairs of D regs.
1088    unsigned Opc = QOpcodes0[OpcodeIndex];
1089    const SDValue Ops[] = { MemAddr, Align, Pred, Reg0, Chain };
1090    SDNode *VLd = CurDAG->getMachineNode(Opc, dl, ResTy, MVT::Other, Ops, 5);
1091    if (NumVecs == 1)
1092      return VLd;
1093
1094    SuperReg = SDValue(VLd, 0);
1095    Chain = SDValue(VLd, 1);
1096
1097  } else {
1098    // Otherwise, quad registers are loaded with two separate instructions,
1099    // where one loads the even registers and the other loads the odd registers.
1100    EVT AddrTy = MemAddr.getValueType();
1101
1102    // Load the even subregs.
1103    unsigned Opc = QOpcodes0[OpcodeIndex];
1104    SDValue ImplDef =
1105      SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, dl, ResTy), 0);
1106    const SDValue OpsA[] = { MemAddr, Align, Reg0, ImplDef, Pred, Reg0, Chain };
1107    SDNode *VLdA =
1108      CurDAG->getMachineNode(Opc, dl, ResTy, AddrTy, MVT::Other, OpsA, 7);
1109    Chain = SDValue(VLdA, 2);
1110
1111    // Load the odd subregs.
1112    Opc = QOpcodes1[OpcodeIndex];
1113    const SDValue OpsB[] = { SDValue(VLdA, 1), Align, Reg0, SDValue(VLdA, 0),
1114                             Pred, Reg0, Chain };
1115    SDNode *VLdB =
1116      CurDAG->getMachineNode(Opc, dl, ResTy, AddrTy, MVT::Other, OpsB, 7);
1117    SuperReg = SDValue(VLdB, 0);
1118    Chain = SDValue(VLdB, 2);
1119  }
1120
1121  // Extract out the Q registers.
1122  assert(ARM::qsub_3 == ARM::qsub_0+3 && "Unexpected subreg numbering");
1123  for (unsigned Vec = 0; Vec < NumVecs; ++Vec) {
1124    SDValue Q = CurDAG->getTargetExtractSubreg(ARM::qsub_0+Vec,
1125                                               dl, VT, SuperReg);
1126    ReplaceUses(SDValue(N, Vec), Q);
1127  }
1128  ReplaceUses(SDValue(N, NumVecs), Chain);
1129  return NULL;
1130}
1131
1132SDNode *ARMDAGToDAGISel::SelectVST(SDNode *N, unsigned NumVecs,
1133                                   unsigned *DOpcodes, unsigned *QOpcodes0,
1134                                   unsigned *QOpcodes1) {
1135  assert(NumVecs >= 1 && NumVecs <= 4 && "VST NumVecs out-of-range");
1136  DebugLoc dl = N->getDebugLoc();
1137
1138  SDValue MemAddr, Align;
1139  if (!SelectAddrMode6(N->getOperand(2), MemAddr, Align))
1140    return NULL;
1141
1142  SDValue Chain = N->getOperand(0);
1143  EVT VT = N->getOperand(3).getValueType();
1144  bool is64BitVector = VT.is64BitVector();
1145
1146  unsigned OpcodeIndex;
1147  switch (VT.getSimpleVT().SimpleTy) {
1148  default: llvm_unreachable("unhandled vst type");
1149    // Double-register operations:
1150  case MVT::v8i8:  OpcodeIndex = 0; break;
1151  case MVT::v4i16: OpcodeIndex = 1; break;
1152  case MVT::v2f32:
1153  case MVT::v2i32: OpcodeIndex = 2; break;
1154  case MVT::v1i64: OpcodeIndex = 3; break;
1155    // Quad-register operations:
1156  case MVT::v16i8: OpcodeIndex = 0; break;
1157  case MVT::v8i16: OpcodeIndex = 1; break;
1158  case MVT::v4f32:
1159  case MVT::v4i32: OpcodeIndex = 2; break;
1160  case MVT::v2i64: OpcodeIndex = 3;
1161    assert(NumVecs == 1 && "v2i64 type only supported for VST1");
1162    break;
1163  }
1164
1165  SDValue Pred = getAL(CurDAG);
1166  SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
1167
1168  SmallVector<SDValue, 7> Ops;
1169  Ops.push_back(MemAddr);
1170  Ops.push_back(Align);
1171
1172  if (is64BitVector) {
1173    if (NumVecs == 1) {
1174      Ops.push_back(N->getOperand(3));
1175    } else {
1176      SDValue RegSeq;
1177      SDValue V0 = N->getOperand(0+3);
1178      SDValue V1 = N->getOperand(1+3);
1179
1180      // Form a REG_SEQUENCE to force register allocation.
1181      if (NumVecs == 2)
1182        RegSeq = SDValue(PairDRegs(MVT::v2i64, V0, V1), 0);
1183      else {
1184        SDValue V2 = N->getOperand(2+3);
1185        // If it's a vld3, form a quad D-register and leave the last part as
1186        // an undef.
1187        SDValue V3 = (NumVecs == 3)
1188          ? SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF,dl,VT), 0)
1189          : N->getOperand(3+3);
1190        RegSeq = SDValue(QuadDRegs(MVT::v4i64, V0, V1, V2, V3), 0);
1191      }
1192      Ops.push_back(RegSeq);
1193    }
1194    Ops.push_back(Pred);
1195    Ops.push_back(Reg0); // predicate register
1196    Ops.push_back(Chain);
1197    unsigned Opc = DOpcodes[OpcodeIndex];
1198    return CurDAG->getMachineNode(Opc, dl, MVT::Other, Ops.data(), 6);
1199  }
1200
1201  if (NumVecs <= 2) {
1202    // Quad registers are directly supported for VST1 and VST2.
1203    unsigned Opc = QOpcodes0[OpcodeIndex];
1204    if (NumVecs == 1) {
1205      Ops.push_back(N->getOperand(3));
1206    } else {
1207      // Form a QQ register.
1208      SDValue Q0 = N->getOperand(3);
1209      SDValue Q1 = N->getOperand(4);
1210      Ops.push_back(SDValue(PairQRegs(MVT::v4i64, Q0, Q1), 0));
1211    }
1212    Ops.push_back(Pred);
1213    Ops.push_back(Reg0); // predicate register
1214    Ops.push_back(Chain);
1215    return CurDAG->getMachineNode(Opc, dl, MVT::Other, Ops.data(), 6);
1216  }
1217
1218  // Otherwise, quad registers are stored with two separate instructions,
1219  // where one stores the even registers and the other stores the odd registers.
1220
1221  // Form the QQQQ REG_SEQUENCE.
1222  SDValue V0 = N->getOperand(0+3);
1223  SDValue V1 = N->getOperand(1+3);
1224  SDValue V2 = N->getOperand(2+3);
1225  SDValue V3 = (NumVecs == 3)
1226    ? SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, dl, VT), 0)
1227    : N->getOperand(3+3);
1228  SDValue RegSeq = SDValue(QuadQRegs(MVT::v8i64, V0, V1, V2, V3), 0);
1229
1230  // Store the even D registers.
1231  Ops.push_back(Reg0); // post-access address offset
1232  Ops.push_back(RegSeq);
1233  Ops.push_back(Pred);
1234  Ops.push_back(Reg0); // predicate register
1235  Ops.push_back(Chain);
1236  unsigned Opc = QOpcodes0[OpcodeIndex];
1237  SDNode *VStA = CurDAG->getMachineNode(Opc, dl, MemAddr.getValueType(),
1238                                        MVT::Other, Ops.data(), 7);
1239  Chain = SDValue(VStA, 1);
1240
1241  // Store the odd D registers.
1242  Ops[0] = SDValue(VStA, 0); // MemAddr
1243  Ops[6] = Chain;
1244  Opc = QOpcodes1[OpcodeIndex];
1245  SDNode *VStB = CurDAG->getMachineNode(Opc, dl, MemAddr.getValueType(),
1246                                        MVT::Other, Ops.data(), 7);
1247  Chain = SDValue(VStB, 1);
1248  ReplaceUses(SDValue(N, 0), Chain);
1249  return NULL;
1250}
1251
1252SDNode *ARMDAGToDAGISel::SelectVLDSTLane(SDNode *N, bool IsLoad,
1253                                         unsigned NumVecs, unsigned *DOpcodes,
1254                                         unsigned *QOpcodes) {
1255  assert(NumVecs >=2 && NumVecs <= 4 && "VLDSTLane NumVecs out-of-range");
1256  DebugLoc dl = N->getDebugLoc();
1257
1258  SDValue MemAddr, Align;
1259  if (!SelectAddrMode6(N->getOperand(2), MemAddr, Align))
1260    return NULL;
1261
1262  SDValue Chain = N->getOperand(0);
1263  unsigned Lane =
1264    cast<ConstantSDNode>(N->getOperand(NumVecs+3))->getZExtValue();
1265  EVT VT = IsLoad ? N->getValueType(0) : N->getOperand(3).getValueType();
1266  bool is64BitVector = VT.is64BitVector();
1267
1268  unsigned OpcodeIndex;
1269  switch (VT.getSimpleVT().SimpleTy) {
1270  default: llvm_unreachable("unhandled vld/vst lane type");
1271    // Double-register operations:
1272  case MVT::v8i8:  OpcodeIndex = 0; break;
1273  case MVT::v4i16: OpcodeIndex = 1; break;
1274  case MVT::v2f32:
1275  case MVT::v2i32: OpcodeIndex = 2; break;
1276    // Quad-register operations:
1277  case MVT::v8i16: OpcodeIndex = 0; break;
1278  case MVT::v4f32:
1279  case MVT::v4i32: OpcodeIndex = 1; break;
1280  }
1281
1282  SDValue Pred = getAL(CurDAG);
1283  SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
1284
1285  SmallVector<SDValue, 7> Ops;
1286  Ops.push_back(MemAddr);
1287  Ops.push_back(Align);
1288
1289  unsigned Opc = (is64BitVector ? DOpcodes[OpcodeIndex] :
1290                                  QOpcodes[OpcodeIndex]);
1291
1292  SDValue SuperReg;
1293  SDValue V0 = N->getOperand(0+3);
1294  SDValue V1 = N->getOperand(1+3);
1295  if (NumVecs == 2) {
1296    if (is64BitVector)
1297      SuperReg = SDValue(PairDRegs(MVT::v2i64, V0, V1), 0);
1298    else
1299      SuperReg = SDValue(PairQRegs(MVT::v4i64, V0, V1), 0);
1300  } else {
1301    SDValue V2 = N->getOperand(2+3);
1302    SDValue V3 = (NumVecs == 3)
1303      ? SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF,dl,VT), 0)
1304      : N->getOperand(3+3);
1305    if (is64BitVector)
1306      SuperReg = SDValue(QuadDRegs(MVT::v4i64, V0, V1, V2, V3), 0);
1307    else
1308      SuperReg = SDValue(QuadQRegs(MVT::v8i64, V0, V1, V2, V3), 0);
1309  }
1310  Ops.push_back(SuperReg);
1311  Ops.push_back(getI32Imm(Lane));
1312  Ops.push_back(Pred);
1313  Ops.push_back(Reg0);
1314  Ops.push_back(Chain);
1315
1316  if (!IsLoad)
1317    return CurDAG->getMachineNode(Opc, dl, MVT::Other, Ops.data(), 7);
1318
1319  EVT ResTy;
1320  unsigned ResTyElts = (NumVecs == 3) ? 4 : NumVecs;
1321  if (!is64BitVector)
1322    ResTyElts *= 2;
1323  ResTy = EVT::getVectorVT(*CurDAG->getContext(), MVT::i64, ResTyElts);
1324
1325  SDNode *VLdLn = CurDAG->getMachineNode(Opc, dl, ResTy, MVT::Other,
1326                                         Ops.data(), 7);
1327  SuperReg = SDValue(VLdLn, 0);
1328  Chain = SDValue(VLdLn, 1);
1329
1330  // Extract the subregisters.
1331  assert(ARM::dsub_7 == ARM::dsub_0+7 && "Unexpected subreg numbering");
1332  assert(ARM::qsub_3 == ARM::qsub_0+3 && "Unexpected subreg numbering");
1333  unsigned SubIdx = is64BitVector ? ARM::dsub_0 : ARM::qsub_0;
1334  for (unsigned Vec = 0; Vec < NumVecs; ++Vec)
1335    ReplaceUses(SDValue(N, Vec),
1336                CurDAG->getTargetExtractSubreg(SubIdx+Vec, dl, VT, SuperReg));
1337  ReplaceUses(SDValue(N, NumVecs), Chain);
1338  return NULL;
1339}
1340
1341SDNode *ARMDAGToDAGISel::SelectVTBL(SDNode *N, bool IsExt, unsigned NumVecs,
1342                                    unsigned Opc) {
1343  assert(NumVecs >= 2 && NumVecs <= 4 && "VTBL NumVecs out-of-range");
1344  DebugLoc dl = N->getDebugLoc();
1345  EVT VT = N->getValueType(0);
1346  unsigned FirstTblReg = IsExt ? 2 : 1;
1347
1348  // Form a REG_SEQUENCE to force register allocation.
1349  SDValue RegSeq;
1350  SDValue V0 = N->getOperand(FirstTblReg + 0);
1351  SDValue V1 = N->getOperand(FirstTblReg + 1);
1352  if (NumVecs == 2)
1353    RegSeq = SDValue(PairDRegs(MVT::v16i8, V0, V1), 0);
1354  else {
1355    SDValue V2 = N->getOperand(FirstTblReg + 2);
1356    // If it's a vtbl3, form a quad D-register and leave the last part as
1357    // an undef.
1358    SDValue V3 = (NumVecs == 3)
1359      ? SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, dl, VT), 0)
1360      : N->getOperand(FirstTblReg + 3);
1361    RegSeq = SDValue(QuadDRegs(MVT::v4i64, V0, V1, V2, V3), 0);
1362  }
1363
1364  SmallVector<SDValue, 6> Ops;
1365  if (IsExt)
1366    Ops.push_back(N->getOperand(1));
1367  Ops.push_back(RegSeq);
1368  Ops.push_back(N->getOperand(FirstTblReg + NumVecs));
1369  Ops.push_back(getAL(CurDAG)); // predicate
1370  Ops.push_back(CurDAG->getRegister(0, MVT::i32)); // predicate register
1371  return CurDAG->getMachineNode(Opc, dl, VT, Ops.data(), Ops.size());
1372}
1373
1374SDNode *ARMDAGToDAGISel::SelectV6T2BitfieldExtractOp(SDNode *N,
1375                                                     bool isSigned) {
1376  if (!Subtarget->hasV6T2Ops())
1377    return NULL;
1378
1379  unsigned Opc = isSigned ? (Subtarget->isThumb() ? ARM::t2SBFX : ARM::SBFX)
1380    : (Subtarget->isThumb() ? ARM::t2UBFX : ARM::UBFX);
1381
1382
1383  // For unsigned extracts, check for a shift right and mask
1384  unsigned And_imm = 0;
1385  if (N->getOpcode() == ISD::AND) {
1386    if (isOpcWithIntImmediate(N, ISD::AND, And_imm)) {
1387
1388      // The immediate is a mask of the low bits iff imm & (imm+1) == 0
1389      if (And_imm & (And_imm + 1))
1390        return NULL;
1391
1392      unsigned Srl_imm = 0;
1393      if (isOpcWithIntImmediate(N->getOperand(0).getNode(), ISD::SRL,
1394                                Srl_imm)) {
1395        assert(Srl_imm > 0 && Srl_imm < 32 && "bad amount in shift node!");
1396
1397        unsigned Width = CountTrailingOnes_32(And_imm);
1398        unsigned LSB = Srl_imm;
1399        SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
1400        SDValue Ops[] = { N->getOperand(0).getOperand(0),
1401                          CurDAG->getTargetConstant(LSB, MVT::i32),
1402                          CurDAG->getTargetConstant(Width, MVT::i32),
1403          getAL(CurDAG), Reg0 };
1404        return CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops, 5);
1405      }
1406    }
1407    return NULL;
1408  }
1409
1410  // Otherwise, we're looking for a shift of a shift
1411  unsigned Shl_imm = 0;
1412  if (isOpcWithIntImmediate(N->getOperand(0).getNode(), ISD::SHL, Shl_imm)) {
1413    assert(Shl_imm > 0 && Shl_imm < 32 && "bad amount in shift node!");
1414    unsigned Srl_imm = 0;
1415    if (isInt32Immediate(N->getOperand(1), Srl_imm)) {
1416      assert(Srl_imm > 0 && Srl_imm < 32 && "bad amount in shift node!");
1417      unsigned Width = 32 - Srl_imm;
1418      int LSB = Srl_imm - Shl_imm;
1419      if (LSB < 0)
1420        return NULL;
1421      SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
1422      SDValue Ops[] = { N->getOperand(0).getOperand(0),
1423                        CurDAG->getTargetConstant(LSB, MVT::i32),
1424                        CurDAG->getTargetConstant(Width, MVT::i32),
1425                        getAL(CurDAG), Reg0 };
1426      return CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops, 5);
1427    }
1428  }
1429  return NULL;
1430}
1431
1432SDNode *ARMDAGToDAGISel::
1433SelectT2CMOVShiftOp(SDNode *N, SDValue FalseVal, SDValue TrueVal,
1434                    ARMCC::CondCodes CCVal, SDValue CCR, SDValue InFlag) {
1435  SDValue CPTmp0;
1436  SDValue CPTmp1;
1437  if (SelectT2ShifterOperandReg(TrueVal, CPTmp0, CPTmp1)) {
1438    unsigned SOVal = cast<ConstantSDNode>(CPTmp1)->getZExtValue();
1439    unsigned SOShOp = ARM_AM::getSORegShOp(SOVal);
1440    unsigned Opc = 0;
1441    switch (SOShOp) {
1442    case ARM_AM::lsl: Opc = ARM::t2MOVCClsl; break;
1443    case ARM_AM::lsr: Opc = ARM::t2MOVCClsr; break;
1444    case ARM_AM::asr: Opc = ARM::t2MOVCCasr; break;
1445    case ARM_AM::ror: Opc = ARM::t2MOVCCror; break;
1446    default:
1447      llvm_unreachable("Unknown so_reg opcode!");
1448      break;
1449    }
1450    SDValue SOShImm =
1451      CurDAG->getTargetConstant(ARM_AM::getSORegOffset(SOVal), MVT::i32);
1452    SDValue CC = CurDAG->getTargetConstant(CCVal, MVT::i32);
1453    SDValue Ops[] = { FalseVal, CPTmp0, SOShImm, CC, CCR, InFlag };
1454    return CurDAG->SelectNodeTo(N, Opc, MVT::i32,Ops, 6);
1455  }
1456  return 0;
1457}
1458
1459SDNode *ARMDAGToDAGISel::
1460SelectARMCMOVShiftOp(SDNode *N, SDValue FalseVal, SDValue TrueVal,
1461                     ARMCC::CondCodes CCVal, SDValue CCR, SDValue InFlag) {
1462  SDValue CPTmp0;
1463  SDValue CPTmp1;
1464  SDValue CPTmp2;
1465  if (SelectShifterOperandReg(TrueVal, CPTmp0, CPTmp1, CPTmp2)) {
1466    SDValue CC = CurDAG->getTargetConstant(CCVal, MVT::i32);
1467    SDValue Ops[] = { FalseVal, CPTmp0, CPTmp1, CPTmp2, CC, CCR, InFlag };
1468    return CurDAG->SelectNodeTo(N, ARM::MOVCCs, MVT::i32, Ops, 7);
1469  }
1470  return 0;
1471}
1472
1473SDNode *ARMDAGToDAGISel::
1474SelectT2CMOVSoImmOp(SDNode *N, SDValue FalseVal, SDValue TrueVal,
1475                    ARMCC::CondCodes CCVal, SDValue CCR, SDValue InFlag) {
1476  ConstantSDNode *T = dyn_cast<ConstantSDNode>(TrueVal);
1477  if (!T)
1478    return 0;
1479
1480  if (Pred_t2_so_imm(TrueVal.getNode())) {
1481    SDValue True = CurDAG->getTargetConstant(T->getZExtValue(), MVT::i32);
1482    SDValue CC = CurDAG->getTargetConstant(CCVal, MVT::i32);
1483    SDValue Ops[] = { FalseVal, True, CC, CCR, InFlag };
1484    return CurDAG->SelectNodeTo(N,
1485                                ARM::t2MOVCCi, MVT::i32, Ops, 5);
1486  }
1487  return 0;
1488}
1489
1490SDNode *ARMDAGToDAGISel::
1491SelectARMCMOVSoImmOp(SDNode *N, SDValue FalseVal, SDValue TrueVal,
1492                     ARMCC::CondCodes CCVal, SDValue CCR, SDValue InFlag) {
1493  ConstantSDNode *T = dyn_cast<ConstantSDNode>(TrueVal);
1494  if (!T)
1495    return 0;
1496
1497  if (Pred_so_imm(TrueVal.getNode())) {
1498    SDValue True = CurDAG->getTargetConstant(T->getZExtValue(), MVT::i32);
1499    SDValue CC = CurDAG->getTargetConstant(CCVal, MVT::i32);
1500    SDValue Ops[] = { FalseVal, True, CC, CCR, InFlag };
1501    return CurDAG->SelectNodeTo(N,
1502                                ARM::MOVCCi, MVT::i32, Ops, 5);
1503  }
1504  return 0;
1505}
1506
1507SDNode *ARMDAGToDAGISel::SelectCMOVOp(SDNode *N) {
1508  EVT VT = N->getValueType(0);
1509  SDValue FalseVal = N->getOperand(0);
1510  SDValue TrueVal  = N->getOperand(1);
1511  SDValue CC = N->getOperand(2);
1512  SDValue CCR = N->getOperand(3);
1513  SDValue InFlag = N->getOperand(4);
1514  assert(CC.getOpcode() == ISD::Constant);
1515  assert(CCR.getOpcode() == ISD::Register);
1516  ARMCC::CondCodes CCVal =
1517    (ARMCC::CondCodes)cast<ConstantSDNode>(CC)->getZExtValue();
1518
1519  if (!Subtarget->isThumb1Only() && VT == MVT::i32) {
1520    // Pattern: (ARMcmov:i32 GPR:i32:$false, so_reg:i32:$true, (imm:i32):$cc)
1521    // Emits: (MOVCCs:i32 GPR:i32:$false, so_reg:i32:$true, (imm:i32):$cc)
1522    // Pattern complexity = 18  cost = 1  size = 0
1523    SDValue CPTmp0;
1524    SDValue CPTmp1;
1525    SDValue CPTmp2;
1526    if (Subtarget->isThumb()) {
1527      SDNode *Res = SelectT2CMOVShiftOp(N, FalseVal, TrueVal,
1528                                        CCVal, CCR, InFlag);
1529      if (!Res)
1530        Res = SelectT2CMOVShiftOp(N, TrueVal, FalseVal,
1531                               ARMCC::getOppositeCondition(CCVal), CCR, InFlag);
1532      if (Res)
1533        return Res;
1534    } else {
1535      SDNode *Res = SelectARMCMOVShiftOp(N, FalseVal, TrueVal,
1536                                         CCVal, CCR, InFlag);
1537      if (!Res)
1538        Res = SelectARMCMOVShiftOp(N, TrueVal, FalseVal,
1539                               ARMCC::getOppositeCondition(CCVal), CCR, InFlag);
1540      if (Res)
1541        return Res;
1542    }
1543
1544    // Pattern: (ARMcmov:i32 GPR:i32:$false,
1545    //             (imm:i32)<<P:Pred_so_imm>>:$true,
1546    //             (imm:i32):$cc)
1547    // Emits: (MOVCCi:i32 GPR:i32:$false,
1548    //           (so_imm:i32 (imm:i32):$true), (imm:i32):$cc)
1549    // Pattern complexity = 10  cost = 1  size = 0
1550    if (Subtarget->isThumb()) {
1551      SDNode *Res = SelectT2CMOVSoImmOp(N, FalseVal, TrueVal,
1552                                        CCVal, CCR, InFlag);
1553      if (!Res)
1554        Res = SelectT2CMOVSoImmOp(N, TrueVal, FalseVal,
1555                               ARMCC::getOppositeCondition(CCVal), CCR, InFlag);
1556      if (Res)
1557        return Res;
1558    } else {
1559      SDNode *Res = SelectARMCMOVSoImmOp(N, FalseVal, TrueVal,
1560                                         CCVal, CCR, InFlag);
1561      if (!Res)
1562        Res = SelectARMCMOVSoImmOp(N, TrueVal, FalseVal,
1563                               ARMCC::getOppositeCondition(CCVal), CCR, InFlag);
1564      if (Res)
1565        return Res;
1566    }
1567  }
1568
1569  // Pattern: (ARMcmov:i32 GPR:i32:$false, GPR:i32:$true, (imm:i32):$cc)
1570  // Emits: (MOVCCr:i32 GPR:i32:$false, GPR:i32:$true, (imm:i32):$cc)
1571  // Pattern complexity = 6  cost = 1  size = 0
1572  //
1573  // Pattern: (ARMcmov:i32 GPR:i32:$false, GPR:i32:$true, (imm:i32):$cc)
1574  // Emits: (tMOVCCr:i32 GPR:i32:$false, GPR:i32:$true, (imm:i32):$cc)
1575  // Pattern complexity = 6  cost = 11  size = 0
1576  //
1577  // Also FCPYScc and FCPYDcc.
1578  SDValue Tmp2 = CurDAG->getTargetConstant(CCVal, MVT::i32);
1579  SDValue Ops[] = { FalseVal, TrueVal, Tmp2, CCR, InFlag };
1580  unsigned Opc = 0;
1581  switch (VT.getSimpleVT().SimpleTy) {
1582  default: assert(false && "Illegal conditional move type!");
1583    break;
1584  case MVT::i32:
1585    Opc = Subtarget->isThumb()
1586      ? (Subtarget->hasThumb2() ? ARM::t2MOVCCr : ARM::tMOVCCr_pseudo)
1587      : ARM::MOVCCr;
1588    break;
1589  case MVT::f32:
1590    Opc = ARM::VMOVScc;
1591    break;
1592  case MVT::f64:
1593    Opc = ARM::VMOVDcc;
1594    break;
1595  }
1596  return CurDAG->SelectNodeTo(N, Opc, VT, Ops, 5);
1597}
1598
1599SDNode *ARMDAGToDAGISel::SelectConcatVector(SDNode *N) {
1600  // The only time a CONCAT_VECTORS operation can have legal types is when
1601  // two 64-bit vectors are concatenated to a 128-bit vector.
1602  EVT VT = N->getValueType(0);
1603  if (!VT.is128BitVector() || N->getNumOperands() != 2)
1604    llvm_unreachable("unexpected CONCAT_VECTORS");
1605  DebugLoc dl = N->getDebugLoc();
1606  SDValue V0 = N->getOperand(0);
1607  SDValue V1 = N->getOperand(1);
1608  SDValue SubReg0 = CurDAG->getTargetConstant(ARM::dsub_0, MVT::i32);
1609  SDValue SubReg1 = CurDAG->getTargetConstant(ARM::dsub_1, MVT::i32);
1610  const SDValue Ops[] = { V0, SubReg0, V1, SubReg1 };
1611  return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops, 4);
1612}
1613
1614SDNode *ARMDAGToDAGISel::Select(SDNode *N) {
1615  DebugLoc dl = N->getDebugLoc();
1616
1617  if (N->isMachineOpcode())
1618    return NULL;   // Already selected.
1619
1620  switch (N->getOpcode()) {
1621  default: break;
1622  case ISD::Constant: {
1623    unsigned Val = cast<ConstantSDNode>(N)->getZExtValue();
1624    bool UseCP = true;
1625    if (Subtarget->hasThumb2())
1626      // Thumb2-aware targets have the MOVT instruction, so all immediates can
1627      // be done with MOV + MOVT, at worst.
1628      UseCP = 0;
1629    else {
1630      if (Subtarget->isThumb()) {
1631        UseCP = (Val > 255 &&                          // MOV
1632                 ~Val > 255 &&                         // MOV + MVN
1633                 !ARM_AM::isThumbImmShiftedVal(Val));  // MOV + LSL
1634      } else
1635        UseCP = (ARM_AM::getSOImmVal(Val) == -1 &&     // MOV
1636                 ARM_AM::getSOImmVal(~Val) == -1 &&    // MVN
1637                 !ARM_AM::isSOImmTwoPartVal(Val));     // two instrs.
1638    }
1639
1640    if (UseCP) {
1641      SDValue CPIdx =
1642        CurDAG->getTargetConstantPool(ConstantInt::get(
1643                                  Type::getInt32Ty(*CurDAG->getContext()), Val),
1644                                      TLI.getPointerTy());
1645
1646      SDNode *ResNode;
1647      if (Subtarget->isThumb1Only()) {
1648        SDValue Pred = getAL(CurDAG);
1649        SDValue PredReg = CurDAG->getRegister(0, MVT::i32);
1650        SDValue Ops[] = { CPIdx, Pred, PredReg, CurDAG->getEntryNode() };
1651        ResNode = CurDAG->getMachineNode(ARM::tLDRcp, dl, MVT::i32, MVT::Other,
1652                                         Ops, 4);
1653      } else {
1654        SDValue Ops[] = {
1655          CPIdx,
1656          CurDAG->getRegister(0, MVT::i32),
1657          CurDAG->getTargetConstant(0, MVT::i32),
1658          getAL(CurDAG),
1659          CurDAG->getRegister(0, MVT::i32),
1660          CurDAG->getEntryNode()
1661        };
1662        ResNode=CurDAG->getMachineNode(ARM::LDRcp, dl, MVT::i32, MVT::Other,
1663                                       Ops, 6);
1664      }
1665      ReplaceUses(SDValue(N, 0), SDValue(ResNode, 0));
1666      return NULL;
1667    }
1668
1669    // Other cases are autogenerated.
1670    break;
1671  }
1672  case ISD::FrameIndex: {
1673    // Selects to ADDri FI, 0 which in turn will become ADDri SP, imm.
1674    int FI = cast<FrameIndexSDNode>(N)->getIndex();
1675    SDValue TFI = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
1676    if (Subtarget->isThumb1Only()) {
1677      return CurDAG->SelectNodeTo(N, ARM::tADDrSPi, MVT::i32, TFI,
1678                                  CurDAG->getTargetConstant(0, MVT::i32));
1679    } else {
1680      unsigned Opc = ((Subtarget->isThumb() && Subtarget->hasThumb2()) ?
1681                      ARM::t2ADDri : ARM::ADDri);
1682      SDValue Ops[] = { TFI, CurDAG->getTargetConstant(0, MVT::i32),
1683                        getAL(CurDAG), CurDAG->getRegister(0, MVT::i32),
1684                        CurDAG->getRegister(0, MVT::i32) };
1685      return CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops, 5);
1686    }
1687  }
1688  case ISD::SRL:
1689    if (SDNode *I = SelectV6T2BitfieldExtractOp(N, false))
1690      return I;
1691    break;
1692  case ISD::SRA:
1693    if (SDNode *I = SelectV6T2BitfieldExtractOp(N, true))
1694      return I;
1695    break;
1696  case ISD::MUL:
1697    if (Subtarget->isThumb1Only())
1698      break;
1699    if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(N->getOperand(1))) {
1700      unsigned RHSV = C->getZExtValue();
1701      if (!RHSV) break;
1702      if (isPowerOf2_32(RHSV-1)) {  // 2^n+1?
1703        unsigned ShImm = Log2_32(RHSV-1);
1704        if (ShImm >= 32)
1705          break;
1706        SDValue V = N->getOperand(0);
1707        ShImm = ARM_AM::getSORegOpc(ARM_AM::lsl, ShImm);
1708        SDValue ShImmOp = CurDAG->getTargetConstant(ShImm, MVT::i32);
1709        SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
1710        if (Subtarget->isThumb()) {
1711          SDValue Ops[] = { V, V, ShImmOp, getAL(CurDAG), Reg0, Reg0 };
1712          return CurDAG->SelectNodeTo(N, ARM::t2ADDrs, MVT::i32, Ops, 6);
1713        } else {
1714          SDValue Ops[] = { V, V, Reg0, ShImmOp, getAL(CurDAG), Reg0, Reg0 };
1715          return CurDAG->SelectNodeTo(N, ARM::ADDrs, MVT::i32, Ops, 7);
1716        }
1717      }
1718      if (isPowerOf2_32(RHSV+1)) {  // 2^n-1?
1719        unsigned ShImm = Log2_32(RHSV+1);
1720        if (ShImm >= 32)
1721          break;
1722        SDValue V = N->getOperand(0);
1723        ShImm = ARM_AM::getSORegOpc(ARM_AM::lsl, ShImm);
1724        SDValue ShImmOp = CurDAG->getTargetConstant(ShImm, MVT::i32);
1725        SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
1726        if (Subtarget->isThumb()) {
1727          SDValue Ops[] = { V, V, ShImmOp, getAL(CurDAG), Reg0, Reg0 };
1728          return CurDAG->SelectNodeTo(N, ARM::t2RSBrs, MVT::i32, Ops, 6);
1729        } else {
1730          SDValue Ops[] = { V, V, Reg0, ShImmOp, getAL(CurDAG), Reg0, Reg0 };
1731          return CurDAG->SelectNodeTo(N, ARM::RSBrs, MVT::i32, Ops, 7);
1732        }
1733      }
1734    }
1735    break;
1736  case ISD::AND: {
1737    // Check for unsigned bitfield extract
1738    if (SDNode *I = SelectV6T2BitfieldExtractOp(N, false))
1739      return I;
1740
1741    // (and (or x, c2), c1) and top 16-bits of c1 and c2 match, lower 16-bits
1742    // of c1 are 0xffff, and lower 16-bit of c2 are 0. That is, the top 16-bits
1743    // are entirely contributed by c2 and lower 16-bits are entirely contributed
1744    // by x. That's equal to (or (and x, 0xffff), (and c1, 0xffff0000)).
1745    // Select it to: "movt x, ((c1 & 0xffff) >> 16)
1746    EVT VT = N->getValueType(0);
1747    if (VT != MVT::i32)
1748      break;
1749    unsigned Opc = (Subtarget->isThumb() && Subtarget->hasThumb2())
1750      ? ARM::t2MOVTi16
1751      : (Subtarget->hasV6T2Ops() ? ARM::MOVTi16 : 0);
1752    if (!Opc)
1753      break;
1754    SDValue N0 = N->getOperand(0), N1 = N->getOperand(1);
1755    ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
1756    if (!N1C)
1757      break;
1758    if (N0.getOpcode() == ISD::OR && N0.getNode()->hasOneUse()) {
1759      SDValue N2 = N0.getOperand(1);
1760      ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(N2);
1761      if (!N2C)
1762        break;
1763      unsigned N1CVal = N1C->getZExtValue();
1764      unsigned N2CVal = N2C->getZExtValue();
1765      if ((N1CVal & 0xffff0000U) == (N2CVal & 0xffff0000U) &&
1766          (N1CVal & 0xffffU) == 0xffffU &&
1767          (N2CVal & 0xffffU) == 0x0U) {
1768        SDValue Imm16 = CurDAG->getTargetConstant((N2CVal & 0xFFFF0000U) >> 16,
1769                                                  MVT::i32);
1770        SDValue Ops[] = { N0.getOperand(0), Imm16,
1771                          getAL(CurDAG), CurDAG->getRegister(0, MVT::i32) };
1772        return CurDAG->getMachineNode(Opc, dl, VT, Ops, 4);
1773      }
1774    }
1775    break;
1776  }
1777  case ARMISD::VMOVRRD:
1778    return CurDAG->getMachineNode(ARM::VMOVRRD, dl, MVT::i32, MVT::i32,
1779                                  N->getOperand(0), getAL(CurDAG),
1780                                  CurDAG->getRegister(0, MVT::i32));
1781  case ISD::UMUL_LOHI: {
1782    if (Subtarget->isThumb1Only())
1783      break;
1784    if (Subtarget->isThumb()) {
1785      SDValue Ops[] = { N->getOperand(0), N->getOperand(1),
1786                        getAL(CurDAG), CurDAG->getRegister(0, MVT::i32),
1787                        CurDAG->getRegister(0, MVT::i32) };
1788      return CurDAG->getMachineNode(ARM::t2UMULL, dl, MVT::i32, MVT::i32,Ops,4);
1789    } else {
1790      SDValue Ops[] = { N->getOperand(0), N->getOperand(1),
1791                        getAL(CurDAG), CurDAG->getRegister(0, MVT::i32),
1792                        CurDAG->getRegister(0, MVT::i32) };
1793      return CurDAG->getMachineNode(ARM::UMULL, dl, MVT::i32, MVT::i32, Ops, 5);
1794    }
1795  }
1796  case ISD::SMUL_LOHI: {
1797    if (Subtarget->isThumb1Only())
1798      break;
1799    if (Subtarget->isThumb()) {
1800      SDValue Ops[] = { N->getOperand(0), N->getOperand(1),
1801                        getAL(CurDAG), CurDAG->getRegister(0, MVT::i32) };
1802      return CurDAG->getMachineNode(ARM::t2SMULL, dl, MVT::i32, MVT::i32,Ops,4);
1803    } else {
1804      SDValue Ops[] = { N->getOperand(0), N->getOperand(1),
1805                        getAL(CurDAG), CurDAG->getRegister(0, MVT::i32),
1806                        CurDAG->getRegister(0, MVT::i32) };
1807      return CurDAG->getMachineNode(ARM::SMULL, dl, MVT::i32, MVT::i32, Ops, 5);
1808    }
1809  }
1810  case ISD::LOAD: {
1811    SDNode *ResNode = 0;
1812    if (Subtarget->isThumb() && Subtarget->hasThumb2())
1813      ResNode = SelectT2IndexedLoad(N);
1814    else
1815      ResNode = SelectARMIndexedLoad(N);
1816    if (ResNode)
1817      return ResNode;
1818    // Other cases are autogenerated.
1819    break;
1820  }
1821  case ARMISD::BRCOND: {
1822    // Pattern: (ARMbrcond:void (bb:Other):$dst, (imm:i32):$cc)
1823    // Emits: (Bcc:void (bb:Other):$dst, (imm:i32):$cc)
1824    // Pattern complexity = 6  cost = 1  size = 0
1825
1826    // Pattern: (ARMbrcond:void (bb:Other):$dst, (imm:i32):$cc)
1827    // Emits: (tBcc:void (bb:Other):$dst, (imm:i32):$cc)
1828    // Pattern complexity = 6  cost = 1  size = 0
1829
1830    // Pattern: (ARMbrcond:void (bb:Other):$dst, (imm:i32):$cc)
1831    // Emits: (t2Bcc:void (bb:Other):$dst, (imm:i32):$cc)
1832    // Pattern complexity = 6  cost = 1  size = 0
1833
1834    unsigned Opc = Subtarget->isThumb() ?
1835      ((Subtarget->hasThumb2()) ? ARM::t2Bcc : ARM::tBcc) : ARM::Bcc;
1836    SDValue Chain = N->getOperand(0);
1837    SDValue N1 = N->getOperand(1);
1838    SDValue N2 = N->getOperand(2);
1839    SDValue N3 = N->getOperand(3);
1840    SDValue InFlag = N->getOperand(4);
1841    assert(N1.getOpcode() == ISD::BasicBlock);
1842    assert(N2.getOpcode() == ISD::Constant);
1843    assert(N3.getOpcode() == ISD::Register);
1844
1845    SDValue Tmp2 = CurDAG->getTargetConstant(((unsigned)
1846                               cast<ConstantSDNode>(N2)->getZExtValue()),
1847                               MVT::i32);
1848    SDValue Ops[] = { N1, Tmp2, N3, Chain, InFlag };
1849    SDNode *ResNode = CurDAG->getMachineNode(Opc, dl, MVT::Other,
1850                                             MVT::Flag, Ops, 5);
1851    Chain = SDValue(ResNode, 0);
1852    if (N->getNumValues() == 2) {
1853      InFlag = SDValue(ResNode, 1);
1854      ReplaceUses(SDValue(N, 1), InFlag);
1855    }
1856    ReplaceUses(SDValue(N, 0),
1857                SDValue(Chain.getNode(), Chain.getResNo()));
1858    return NULL;
1859  }
1860  case ARMISD::CMOV:
1861    return SelectCMOVOp(N);
1862  case ARMISD::CNEG: {
1863    EVT VT = N->getValueType(0);
1864    SDValue N0 = N->getOperand(0);
1865    SDValue N1 = N->getOperand(1);
1866    SDValue N2 = N->getOperand(2);
1867    SDValue N3 = N->getOperand(3);
1868    SDValue InFlag = N->getOperand(4);
1869    assert(N2.getOpcode() == ISD::Constant);
1870    assert(N3.getOpcode() == ISD::Register);
1871
1872    SDValue Tmp2 = CurDAG->getTargetConstant(((unsigned)
1873                               cast<ConstantSDNode>(N2)->getZExtValue()),
1874                               MVT::i32);
1875    SDValue Ops[] = { N0, N1, Tmp2, N3, InFlag };
1876    unsigned Opc = 0;
1877    switch (VT.getSimpleVT().SimpleTy) {
1878    default: assert(false && "Illegal conditional move type!");
1879      break;
1880    case MVT::f32:
1881      Opc = ARM::VNEGScc;
1882      break;
1883    case MVT::f64:
1884      Opc = ARM::VNEGDcc;
1885      break;
1886    }
1887    return CurDAG->SelectNodeTo(N, Opc, VT, Ops, 5);
1888  }
1889
1890  case ARMISD::VZIP: {
1891    unsigned Opc = 0;
1892    EVT VT = N->getValueType(0);
1893    switch (VT.getSimpleVT().SimpleTy) {
1894    default: return NULL;
1895    case MVT::v8i8:  Opc = ARM::VZIPd8; break;
1896    case MVT::v4i16: Opc = ARM::VZIPd16; break;
1897    case MVT::v2f32:
1898    case MVT::v2i32: Opc = ARM::VZIPd32; break;
1899    case MVT::v16i8: Opc = ARM::VZIPq8; break;
1900    case MVT::v8i16: Opc = ARM::VZIPq16; break;
1901    case MVT::v4f32:
1902    case MVT::v4i32: Opc = ARM::VZIPq32; break;
1903    }
1904    SDValue Pred = getAL(CurDAG);
1905    SDValue PredReg = CurDAG->getRegister(0, MVT::i32);
1906    SDValue Ops[] = { N->getOperand(0), N->getOperand(1), Pred, PredReg };
1907    return CurDAG->getMachineNode(Opc, dl, VT, VT, Ops, 4);
1908  }
1909  case ARMISD::VUZP: {
1910    unsigned Opc = 0;
1911    EVT VT = N->getValueType(0);
1912    switch (VT.getSimpleVT().SimpleTy) {
1913    default: return NULL;
1914    case MVT::v8i8:  Opc = ARM::VUZPd8; break;
1915    case MVT::v4i16: Opc = ARM::VUZPd16; break;
1916    case MVT::v2f32:
1917    case MVT::v2i32: Opc = ARM::VUZPd32; break;
1918    case MVT::v16i8: Opc = ARM::VUZPq8; break;
1919    case MVT::v8i16: Opc = ARM::VUZPq16; break;
1920    case MVT::v4f32:
1921    case MVT::v4i32: Opc = ARM::VUZPq32; break;
1922    }
1923    SDValue Pred = getAL(CurDAG);
1924    SDValue PredReg = CurDAG->getRegister(0, MVT::i32);
1925    SDValue Ops[] = { N->getOperand(0), N->getOperand(1), Pred, PredReg };
1926    return CurDAG->getMachineNode(Opc, dl, VT, VT, Ops, 4);
1927  }
1928  case ARMISD::VTRN: {
1929    unsigned Opc = 0;
1930    EVT VT = N->getValueType(0);
1931    switch (VT.getSimpleVT().SimpleTy) {
1932    default: return NULL;
1933    case MVT::v8i8:  Opc = ARM::VTRNd8; break;
1934    case MVT::v4i16: Opc = ARM::VTRNd16; break;
1935    case MVT::v2f32:
1936    case MVT::v2i32: Opc = ARM::VTRNd32; break;
1937    case MVT::v16i8: Opc = ARM::VTRNq8; break;
1938    case MVT::v8i16: Opc = ARM::VTRNq16; break;
1939    case MVT::v4f32:
1940    case MVT::v4i32: Opc = ARM::VTRNq32; break;
1941    }
1942    SDValue Pred = getAL(CurDAG);
1943    SDValue PredReg = CurDAG->getRegister(0, MVT::i32);
1944    SDValue Ops[] = { N->getOperand(0), N->getOperand(1), Pred, PredReg };
1945    return CurDAG->getMachineNode(Opc, dl, VT, VT, Ops, 4);
1946  }
1947  case ARMISD::BUILD_VECTOR: {
1948    EVT VecVT = N->getValueType(0);
1949    EVT EltVT = VecVT.getVectorElementType();
1950    unsigned NumElts = VecVT.getVectorNumElements();
1951    if (EltVT.getSimpleVT() == MVT::f64) {
1952      assert(NumElts == 2 && "unexpected type for BUILD_VECTOR");
1953      return PairDRegs(VecVT, N->getOperand(0), N->getOperand(1));
1954    }
1955    assert(EltVT.getSimpleVT() == MVT::f32 &&
1956           "unexpected type for BUILD_VECTOR");
1957    if (NumElts == 2)
1958      return PairSRegs(VecVT, N->getOperand(0), N->getOperand(1));
1959    assert(NumElts == 4 && "unexpected type for BUILD_VECTOR");
1960    return QuadSRegs(VecVT, N->getOperand(0), N->getOperand(1),
1961                     N->getOperand(2), N->getOperand(3));
1962  }
1963
1964  case ISD::INTRINSIC_VOID:
1965  case ISD::INTRINSIC_W_CHAIN: {
1966    unsigned IntNo = cast<ConstantSDNode>(N->getOperand(1))->getZExtValue();
1967    switch (IntNo) {
1968    default:
1969      break;
1970
1971    case Intrinsic::arm_neon_vld1: {
1972      unsigned DOpcodes[] = { ARM::VLD1d8, ARM::VLD1d16,
1973                              ARM::VLD1d32, ARM::VLD1d64 };
1974      unsigned QOpcodes[] = { ARM::VLD1q8Pseudo, ARM::VLD1q16Pseudo,
1975                              ARM::VLD1q32Pseudo, ARM::VLD1q64Pseudo };
1976      return SelectVLD(N, 1, DOpcodes, QOpcodes, 0);
1977    }
1978
1979    case Intrinsic::arm_neon_vld2: {
1980      unsigned DOpcodes[] = { ARM::VLD2d8Pseudo, ARM::VLD2d16Pseudo,
1981                              ARM::VLD2d32Pseudo, ARM::VLD1q64Pseudo };
1982      unsigned QOpcodes[] = { ARM::VLD2q8Pseudo, ARM::VLD2q16Pseudo,
1983                              ARM::VLD2q32Pseudo };
1984      return SelectVLD(N, 2, DOpcodes, QOpcodes, 0);
1985    }
1986
1987    case Intrinsic::arm_neon_vld3: {
1988      unsigned DOpcodes[] = { ARM::VLD3d8Pseudo, ARM::VLD3d16Pseudo,
1989                              ARM::VLD3d32Pseudo, ARM::VLD1d64TPseudo };
1990      unsigned QOpcodes0[] = { ARM::VLD3q8Pseudo_UPD,
1991                               ARM::VLD3q16Pseudo_UPD,
1992                               ARM::VLD3q32Pseudo_UPD };
1993      unsigned QOpcodes1[] = { ARM::VLD3q8oddPseudo_UPD,
1994                               ARM::VLD3q16oddPseudo_UPD,
1995                               ARM::VLD3q32oddPseudo_UPD };
1996      return SelectVLD(N, 3, DOpcodes, QOpcodes0, QOpcodes1);
1997    }
1998
1999    case Intrinsic::arm_neon_vld4: {
2000      unsigned DOpcodes[] = { ARM::VLD4d8Pseudo, ARM::VLD4d16Pseudo,
2001                              ARM::VLD4d32Pseudo, ARM::VLD1d64QPseudo };
2002      unsigned QOpcodes0[] = { ARM::VLD4q8Pseudo_UPD,
2003                               ARM::VLD4q16Pseudo_UPD,
2004                               ARM::VLD4q32Pseudo_UPD };
2005      unsigned QOpcodes1[] = { ARM::VLD4q8oddPseudo_UPD,
2006                               ARM::VLD4q16oddPseudo_UPD,
2007                               ARM::VLD4q32oddPseudo_UPD };
2008      return SelectVLD(N, 4, DOpcodes, QOpcodes0, QOpcodes1);
2009    }
2010
2011    case Intrinsic::arm_neon_vld2lane: {
2012      unsigned DOpcodes[] = { ARM::VLD2LNd8Pseudo, ARM::VLD2LNd16Pseudo,
2013                              ARM::VLD2LNd32Pseudo };
2014      unsigned QOpcodes[] = { ARM::VLD2LNq16Pseudo, ARM::VLD2LNq32Pseudo };
2015      return SelectVLDSTLane(N, true, 2, DOpcodes, QOpcodes);
2016    }
2017
2018    case Intrinsic::arm_neon_vld3lane: {
2019      unsigned DOpcodes[] = { ARM::VLD3LNd8Pseudo, ARM::VLD3LNd16Pseudo,
2020                              ARM::VLD3LNd32Pseudo };
2021      unsigned QOpcodes[] = { ARM::VLD3LNq16Pseudo, ARM::VLD3LNq32Pseudo };
2022      return SelectVLDSTLane(N, true, 3, DOpcodes, QOpcodes);
2023    }
2024
2025    case Intrinsic::arm_neon_vld4lane: {
2026      unsigned DOpcodes[] = { ARM::VLD4LNd8Pseudo, ARM::VLD4LNd16Pseudo,
2027                              ARM::VLD4LNd32Pseudo };
2028      unsigned QOpcodes[] = { ARM::VLD4LNq16Pseudo, ARM::VLD4LNq32Pseudo };
2029      return SelectVLDSTLane(N, true, 4, DOpcodes, QOpcodes);
2030    }
2031
2032    case Intrinsic::arm_neon_vst1: {
2033      unsigned DOpcodes[] = { ARM::VST1d8, ARM::VST1d16,
2034                              ARM::VST1d32, ARM::VST1d64 };
2035      unsigned QOpcodes[] = { ARM::VST1q8Pseudo, ARM::VST1q16Pseudo,
2036                              ARM::VST1q32Pseudo, ARM::VST1q64Pseudo };
2037      return SelectVST(N, 1, DOpcodes, QOpcodes, 0);
2038    }
2039
2040    case Intrinsic::arm_neon_vst2: {
2041      unsigned DOpcodes[] = { ARM::VST2d8Pseudo, ARM::VST2d16Pseudo,
2042                              ARM::VST2d32Pseudo, ARM::VST1q64Pseudo };
2043      unsigned QOpcodes[] = { ARM::VST2q8Pseudo, ARM::VST2q16Pseudo,
2044                              ARM::VST2q32Pseudo };
2045      return SelectVST(N, 2, DOpcodes, QOpcodes, 0);
2046    }
2047
2048    case Intrinsic::arm_neon_vst3: {
2049      unsigned DOpcodes[] = { ARM::VST3d8Pseudo, ARM::VST3d16Pseudo,
2050                              ARM::VST3d32Pseudo, ARM::VST1d64TPseudo };
2051      unsigned QOpcodes0[] = { ARM::VST3q8Pseudo_UPD,
2052                               ARM::VST3q16Pseudo_UPD,
2053                               ARM::VST3q32Pseudo_UPD };
2054      unsigned QOpcodes1[] = { ARM::VST3q8oddPseudo_UPD,
2055                               ARM::VST3q16oddPseudo_UPD,
2056                               ARM::VST3q32oddPseudo_UPD };
2057      return SelectVST(N, 3, DOpcodes, QOpcodes0, QOpcodes1);
2058    }
2059
2060    case Intrinsic::arm_neon_vst4: {
2061      unsigned DOpcodes[] = { ARM::VST4d8Pseudo, ARM::VST4d16Pseudo,
2062                              ARM::VST4d32Pseudo, ARM::VST1d64QPseudo };
2063      unsigned QOpcodes0[] = { ARM::VST4q8Pseudo_UPD,
2064                               ARM::VST4q16Pseudo_UPD,
2065                               ARM::VST4q32Pseudo_UPD };
2066      unsigned QOpcodes1[] = { ARM::VST4q8oddPseudo_UPD,
2067                               ARM::VST4q16oddPseudo_UPD,
2068                               ARM::VST4q32oddPseudo_UPD };
2069      return SelectVST(N, 4, DOpcodes, QOpcodes0, QOpcodes1);
2070    }
2071
2072    case Intrinsic::arm_neon_vst2lane: {
2073      unsigned DOpcodes[] = { ARM::VST2LNd8Pseudo, ARM::VST2LNd16Pseudo,
2074                              ARM::VST2LNd32Pseudo };
2075      unsigned QOpcodes[] = { ARM::VST2LNq16Pseudo, ARM::VST2LNq32Pseudo };
2076      return SelectVLDSTLane(N, false, 2, DOpcodes, QOpcodes);
2077    }
2078
2079    case Intrinsic::arm_neon_vst3lane: {
2080      unsigned DOpcodes[] = { ARM::VST3LNd8Pseudo, ARM::VST3LNd16Pseudo,
2081                              ARM::VST3LNd32Pseudo };
2082      unsigned QOpcodes[] = { ARM::VST3LNq16Pseudo, ARM::VST3LNq32Pseudo };
2083      return SelectVLDSTLane(N, false, 3, DOpcodes, QOpcodes);
2084    }
2085
2086    case Intrinsic::arm_neon_vst4lane: {
2087      unsigned DOpcodes[] = { ARM::VST4LNd8Pseudo, ARM::VST4LNd16Pseudo,
2088                              ARM::VST4LNd32Pseudo };
2089      unsigned QOpcodes[] = { ARM::VST4LNq16Pseudo, ARM::VST4LNq32Pseudo };
2090      return SelectVLDSTLane(N, false, 4, DOpcodes, QOpcodes);
2091    }
2092    }
2093    break;
2094  }
2095
2096  case ISD::INTRINSIC_WO_CHAIN: {
2097    unsigned IntNo = cast<ConstantSDNode>(N->getOperand(0))->getZExtValue();
2098    switch (IntNo) {
2099    default:
2100      break;
2101
2102    case Intrinsic::arm_neon_vtbl2:
2103      return SelectVTBL(N, false, 2, ARM::VTBL2Pseudo);
2104    case Intrinsic::arm_neon_vtbl3:
2105      return SelectVTBL(N, false, 3, ARM::VTBL3Pseudo);
2106    case Intrinsic::arm_neon_vtbl4:
2107      return SelectVTBL(N, false, 4, ARM::VTBL4Pseudo);
2108
2109    case Intrinsic::arm_neon_vtbx2:
2110      return SelectVTBL(N, true, 2, ARM::VTBX2Pseudo);
2111    case Intrinsic::arm_neon_vtbx3:
2112      return SelectVTBL(N, true, 3, ARM::VTBX3Pseudo);
2113    case Intrinsic::arm_neon_vtbx4:
2114      return SelectVTBL(N, true, 4, ARM::VTBX4Pseudo);
2115    }
2116    break;
2117  }
2118
2119  case ISD::CONCAT_VECTORS:
2120    return SelectConcatVector(N);
2121  }
2122
2123  return SelectCode(N);
2124}
2125
2126bool ARMDAGToDAGISel::
2127SelectInlineAsmMemoryOperand(const SDValue &Op, char ConstraintCode,
2128                             std::vector<SDValue> &OutOps) {
2129  assert(ConstraintCode == 'm' && "unexpected asm memory constraint");
2130  // Require the address to be in a register.  That is safe for all ARM
2131  // variants and it is hard to do anything much smarter without knowing
2132  // how the operand is used.
2133  OutOps.push_back(Op);
2134  return false;
2135}
2136
2137/// createARMISelDag - This pass converts a legalized DAG into a
2138/// ARM-specific DAG, ready for instruction scheduling.
2139///
2140FunctionPass *llvm::createARMISelDag(ARMBaseTargetMachine &TM,
2141                                     CodeGenOpt::Level OptLevel) {
2142  return new ARMDAGToDAGISel(TM, OptLevel);
2143}
2144