ARMISelDAGToDAG.cpp revision 85a5c32287005113b019b2bb105f66a70ccd8c73 
1//===-- ARMISelDAGToDAG.cpp - A dag to dag inst selector for ARM ----------===//
2//
3//                     The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10// This file defines an instruction selector for the ARM target.
11//
12//===----------------------------------------------------------------------===//
13
14#define DEBUG_TYPE "arm-isel"
15#include "ARM.h"
16#include "ARMAddressingModes.h"
17#include "ARMTargetMachine.h"
18#include "llvm/CallingConv.h"
19#include "llvm/Constants.h"
20#include "llvm/DerivedTypes.h"
21#include "llvm/Function.h"
22#include "llvm/Intrinsics.h"
23#include "llvm/LLVMContext.h"
24#include "llvm/CodeGen/MachineFrameInfo.h"
25#include "llvm/CodeGen/MachineFunction.h"
26#include "llvm/CodeGen/MachineInstrBuilder.h"
27#include "llvm/CodeGen/SelectionDAG.h"
28#include "llvm/CodeGen/SelectionDAGISel.h"
29#include "llvm/Target/TargetLowering.h"
30#include "llvm/Target/TargetOptions.h"
31#include "llvm/Support/CommandLine.h"
32#include "llvm/Support/Compiler.h"
33#include "llvm/Support/Debug.h"
34#include "llvm/Support/ErrorHandling.h"
35#include "llvm/Support/raw_ostream.h"
36
37using namespace llvm;
38
39static cl::opt<bool>
40DisableShifterOp("disable-shifter-op", cl::Hidden,
41  cl::desc("Disable isel of shifter-op"),
42  cl::init(false));
43
44//===--------------------------------------------------------------------===//
45/// ARMDAGToDAGISel - ARM specific code to select ARM machine
46/// instructions for SelectionDAG operations.
47///
48namespace {
49class ARMDAGToDAGISel : public SelectionDAGISel {
50  ARMBaseTargetMachine &TM;
51
52  /// Subtarget - Keep a pointer to the ARMSubtarget around so that we can
53  /// make the right decision when generating code for different targets.
54  const ARMSubtarget *Subtarget;
55
56public:
57  explicit ARMDAGToDAGISel(ARMBaseTargetMachine &tm,
58                           CodeGenOpt::Level OptLevel)
59    : SelectionDAGISel(tm, OptLevel), TM(tm),
60    Subtarget(&TM.getSubtarget<ARMSubtarget>()) {
61  }
62
63  virtual const char *getPassName() const {
64    return "ARM Instruction Selection";
65  }
66
67  /// getI32Imm - Return a target constant of type i32 with the specified
68  /// value.
69  inline SDValue getI32Imm(unsigned Imm) {
70    return CurDAG->getTargetConstant(Imm, MVT::i32);
71  }
72
73  SDNode *Select(SDNode *N);
74
75  bool SelectShifterOperandReg(SDValue N, SDValue &A,
76                               SDValue &B, SDValue &C);
77  bool SelectAddrMode2(SDValue N, SDValue &Base,
78                       SDValue &Offset, SDValue &Opc);
79  bool SelectAddrMode2Offset(SDNode *Op, SDValue N,
80                             SDValue &Offset, SDValue &Opc);
81  bool SelectAddrMode3(SDValue N, SDValue &Base,
82                       SDValue &Offset, SDValue &Opc);
83  bool SelectAddrMode3Offset(SDNode *Op, SDValue N,
84                             SDValue &Offset, SDValue &Opc);
85  bool SelectAddrMode4(SDValue N, SDValue &Addr, SDValue &Mode);
86  bool SelectAddrMode5(SDValue N, SDValue &Base,
87                       SDValue &Offset);
88  bool SelectAddrMode6(SDValue N, SDValue &Addr, SDValue &Align);
89
90  bool SelectAddrModePC(SDValue N, SDValue &Offset,
91                        SDValue &Label);
92
93  bool SelectThumbAddrModeRR(SDValue N, SDValue &Base, SDValue &Offset);
94  bool SelectThumbAddrModeRI5(SDValue N, unsigned Scale,
95                              SDValue &Base, SDValue &OffImm,
96                              SDValue &Offset);
97  bool SelectThumbAddrModeS1(SDValue N, SDValue &Base,
98                             SDValue &OffImm, SDValue &Offset);
99  bool SelectThumbAddrModeS2(SDValue N, SDValue &Base,
100                             SDValue &OffImm, SDValue &Offset);
101  bool SelectThumbAddrModeS4(SDValue N, SDValue &Base,
102                             SDValue &OffImm, SDValue &Offset);
103  bool SelectThumbAddrModeSP(SDValue N, SDValue &Base, SDValue &OffImm);
104
105  bool SelectT2ShifterOperandReg(SDValue N,
106                                 SDValue &BaseReg, SDValue &Opc);
107  bool SelectT2AddrModeImm12(SDValue N, SDValue &Base, SDValue &OffImm);
108  bool SelectT2AddrModeImm8(SDValue N, SDValue &Base,
109                            SDValue &OffImm);
110  bool SelectT2AddrModeImm8Offset(SDNode *Op, SDValue N,
111                                 SDValue &OffImm);
112  bool SelectT2AddrModeSoReg(SDValue N, SDValue &Base,
113                             SDValue &OffReg, SDValue &ShImm);
114
115  inline bool Pred_so_imm(SDNode *inN) const {
116    ConstantSDNode *N = cast<ConstantSDNode>(inN);
117    return ARM_AM::getSOImmVal(N->getZExtValue()) != -1;
118  }
119
120  inline bool Pred_t2_so_imm(SDNode *inN) const {
121    ConstantSDNode *N = cast<ConstantSDNode>(inN);
122    return ARM_AM::getT2SOImmVal(N->getZExtValue()) != -1;
123  }
124
125  // Include the pieces autogenerated from the target description.
126#include "ARMGenDAGISel.inc"
127
128private:
129  /// SelectARMIndexedLoad - Indexed (pre/post inc/dec) load matching code for
130  /// ARM.
131  SDNode *SelectARMIndexedLoad(SDNode *N);
132  SDNode *SelectT2IndexedLoad(SDNode *N);
133
134  /// SelectVLD - Select NEON load intrinsics.  NumVecs should be
135  /// 1, 2, 3 or 4.  The opcode arrays specify the instructions used for
136  /// loads of D registers and even subregs and odd subregs of Q registers.
137  /// For NumVecs <= 2, QOpcodes1 is not used.
138  SDNode *SelectVLD(SDNode *N, unsigned NumVecs, unsigned *DOpcodes,
139                    unsigned *QOpcodes0, unsigned *QOpcodes1);
140
141  /// SelectVST - Select NEON store intrinsics.  NumVecs should
142  /// be 1, 2, 3 or 4.  The opcode arrays specify the instructions used for
143  /// stores of D registers and even subregs and odd subregs of Q registers.
144  /// For NumVecs <= 2, QOpcodes1 is not used.
145  SDNode *SelectVST(SDNode *N, unsigned NumVecs, unsigned *DOpcodes,
146                    unsigned *QOpcodes0, unsigned *QOpcodes1);
147
148  /// SelectVLDSTLane - Select NEON load/store lane intrinsics.  NumVecs should
149  /// be 2, 3 or 4.  The opcode arrays specify the instructions used for
150  /// load/store of D registers and Q registers.
151  SDNode *SelectVLDSTLane(SDNode *N, bool IsLoad, unsigned NumVecs,
152                          unsigned *DOpcodes, unsigned *QOpcodes);
153
154  /// SelectVTBL - Select NEON VTBL and VTBX intrinsics.  NumVecs should be 2,
155  /// 3 or 4.  These are custom-selected so that a REG_SEQUENCE can be
156  /// generated to force the table registers to be consecutive.
157  SDNode *SelectVTBL(SDNode *N, bool IsExt, unsigned NumVecs, unsigned Opc);
158
159  /// SelectV6T2BitfieldExtractOp - Select SBFX/UBFX instructions for ARM.
160  SDNode *SelectV6T2BitfieldExtractOp(SDNode *N, bool isSigned);
161
162  /// SelectCMOVOp - Select CMOV instructions for ARM.
163  SDNode *SelectCMOVOp(SDNode *N);
164  SDNode *SelectT2CMOVShiftOp(SDNode *N, SDValue FalseVal, SDValue TrueVal,
165                              ARMCC::CondCodes CCVal, SDValue CCR,
166                              SDValue InFlag);
167  SDNode *SelectARMCMOVShiftOp(SDNode *N, SDValue FalseVal, SDValue TrueVal,
168                               ARMCC::CondCodes CCVal, SDValue CCR,
169                               SDValue InFlag);
170  SDNode *SelectT2CMOVSoImmOp(SDNode *N, SDValue FalseVal, SDValue TrueVal,
171                              ARMCC::CondCodes CCVal, SDValue CCR,
172                              SDValue InFlag);
173  SDNode *SelectARMCMOVSoImmOp(SDNode *N, SDValue FalseVal, SDValue TrueVal,
174                               ARMCC::CondCodes CCVal, SDValue CCR,
175                               SDValue InFlag);
176
177  SDNode *SelectConcatVector(SDNode *N);
178
179  /// SelectInlineAsmMemoryOperand - Implement addressing mode selection for
180  /// inline asm expressions.
181  virtual bool SelectInlineAsmMemoryOperand(const SDValue &Op,
182                                            char ConstraintCode,
183                                            std::vector<SDValue> &OutOps);
184
185  // Form pairs of consecutive S, D, or Q registers.
186  SDNode *PairSRegs(EVT VT, SDValue V0, SDValue V1);
187  SDNode *PairDRegs(EVT VT, SDValue V0, SDValue V1);
188  SDNode *PairQRegs(EVT VT, SDValue V0, SDValue V1);
189
190  // Form sequences of 4 consecutive S, D, or Q registers.
191  SDNode *QuadSRegs(EVT VT, SDValue V0, SDValue V1, SDValue V2, SDValue V3);
192  SDNode *QuadDRegs(EVT VT, SDValue V0, SDValue V1, SDValue V2, SDValue V3);
193  SDNode *QuadQRegs(EVT VT, SDValue V0, SDValue V1, SDValue V2, SDValue V3);
194};
195}
196
197/// isInt32Immediate - This method tests to see if the node is a 32-bit constant
198/// operand. If so Imm will receive the 32-bit value.
199static bool isInt32Immediate(SDNode *N, unsigned &Imm) {
200  if (N->getOpcode() == ISD::Constant && N->getValueType(0) == MVT::i32) {
201    Imm = cast<ConstantSDNode>(N)->getZExtValue();
202    return true;
203  }
204  return false;
205}
206
207// isInt32Immediate - This method tests to see if a constant operand.
208// If so Imm will receive the 32 bit value.
209static bool isInt32Immediate(SDValue N, unsigned &Imm) {
210  return isInt32Immediate(N.getNode(), Imm);
211}
212
213// isOpcWithIntImmediate - This method tests to see if the node is a specific
214// opcode and that it has a immediate integer right operand.
215// If so Imm will receive the 32 bit value.
216static bool isOpcWithIntImmediate(SDNode *N, unsigned Opc, unsigned& Imm) {
217  return N->getOpcode() == Opc &&
218         isInt32Immediate(N->getOperand(1).getNode(), Imm);
219}
220
221
222bool ARMDAGToDAGISel::SelectShifterOperandReg(SDValue N,
223                                              SDValue &BaseReg,
224                                              SDValue &ShReg,
225                                              SDValue &Opc) {
226  if (DisableShifterOp)
227    return false;
228
229  ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(N);
230
231  // Don't match base register only case. That is matched to a separate
232  // lower complexity pattern with explicit register operand.
233  if (ShOpcVal == ARM_AM::no_shift) return false;
234
235  BaseReg = N.getOperand(0);
236  unsigned ShImmVal = 0;
237  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
238    ShReg = CurDAG->getRegister(0, MVT::i32);
239    ShImmVal = RHS->getZExtValue() & 31;
240  } else {
241    ShReg = N.getOperand(1);
242  }
243  Opc = CurDAG->getTargetConstant(ARM_AM::getSORegOpc(ShOpcVal, ShImmVal),
244                                  MVT::i32);
245  return true;
246}
247
248bool ARMDAGToDAGISel::SelectAddrMode2(SDValue N,
249                                      SDValue &Base, SDValue &Offset,
250                                      SDValue &Opc) {
251  if (N.getOpcode() == ISD::MUL) {
252    if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
253      // X * [3,5,9] -> X + X * [2,4,8] etc.
254      int RHSC = (int)RHS->getZExtValue();
255      if (RHSC & 1) {
256        RHSC = RHSC & ~1;
257        ARM_AM::AddrOpc AddSub = ARM_AM::add;
258        if (RHSC < 0) {
259          AddSub = ARM_AM::sub;
260          RHSC = - RHSC;
261        }
262        if (isPowerOf2_32(RHSC)) {
263          unsigned ShAmt = Log2_32(RHSC);
264          Base = Offset = N.getOperand(0);
265          Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, ShAmt,
266                                                            ARM_AM::lsl),
267                                          MVT::i32);
268          return true;
269        }
270      }
271    }
272  }
273
274  if (N.getOpcode() != ISD::ADD && N.getOpcode() != ISD::SUB) {
275    Base = N;
276    if (N.getOpcode() == ISD::FrameIndex) {
277      int FI = cast<FrameIndexSDNode>(N)->getIndex();
278      Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
279    } else if (N.getOpcode() == ARMISD::Wrapper &&
280               !(Subtarget->useMovt() &&
281                 N.getOperand(0).getOpcode() == ISD::TargetGlobalAddress)) {
282      Base = N.getOperand(0);
283    }
284    Offset = CurDAG->getRegister(0, MVT::i32);
285    Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(ARM_AM::add, 0,
286                                                      ARM_AM::no_shift),
287                                    MVT::i32);
288    return true;
289  }
290
291  // Match simple R +/- imm12 operands.
292  if (N.getOpcode() == ISD::ADD)
293    if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
294      int RHSC = (int)RHS->getZExtValue();
295      if ((RHSC >= 0 && RHSC < 0x1000) ||
296          (RHSC < 0 && RHSC > -0x1000)) { // 12 bits.
297        Base = N.getOperand(0);
298        if (Base.getOpcode() == ISD::FrameIndex) {
299          int FI = cast<FrameIndexSDNode>(Base)->getIndex();
300          Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
301        }
302        Offset = CurDAG->getRegister(0, MVT::i32);
303
304        ARM_AM::AddrOpc AddSub = ARM_AM::add;
305        if (RHSC < 0) {
306          AddSub = ARM_AM::sub;
307          RHSC = - RHSC;
308        }
309        Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, RHSC,
310                                                          ARM_AM::no_shift),
311                                        MVT::i32);
312        return true;
313      }
314    }
315
316  // Otherwise this is R +/- [possibly shifted] R.
317  ARM_AM::AddrOpc AddSub = N.getOpcode() == ISD::ADD ? ARM_AM::add:ARM_AM::sub;
318  ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(N.getOperand(1));
319  unsigned ShAmt = 0;
320
321  Base   = N.getOperand(0);
322  Offset = N.getOperand(1);
323
324  if (ShOpcVal != ARM_AM::no_shift) {
325    // Check to see if the RHS of the shift is a constant, if not, we can't fold
326    // it.
327    if (ConstantSDNode *Sh =
328           dyn_cast<ConstantSDNode>(N.getOperand(1).getOperand(1))) {
329      ShAmt = Sh->getZExtValue();
330      Offset = N.getOperand(1).getOperand(0);
331    } else {
332      ShOpcVal = ARM_AM::no_shift;
333    }
334  }
335
336  // Try matching (R shl C) + (R).
337  if (N.getOpcode() == ISD::ADD && ShOpcVal == ARM_AM::no_shift) {
338    ShOpcVal = ARM_AM::getShiftOpcForNode(N.getOperand(0));
339    if (ShOpcVal != ARM_AM::no_shift) {
340      // Check to see if the RHS of the shift is a constant, if not, we can't
341      // fold it.
342      if (ConstantSDNode *Sh =
343          dyn_cast<ConstantSDNode>(N.getOperand(0).getOperand(1))) {
344        ShAmt = Sh->getZExtValue();
345        Offset = N.getOperand(0).getOperand(0);
346        Base = N.getOperand(1);
347      } else {
348        ShOpcVal = ARM_AM::no_shift;
349      }
350    }
351  }
352
353  Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, ShAmt, ShOpcVal),
354                                  MVT::i32);
355  return true;
356}
357
358bool ARMDAGToDAGISel::SelectAddrMode2Offset(SDNode *Op, SDValue N,
359                                            SDValue &Offset, SDValue &Opc) {
360  unsigned Opcode = Op->getOpcode();
361  ISD::MemIndexedMode AM = (Opcode == ISD::LOAD)
362    ? cast<LoadSDNode>(Op)->getAddressingMode()
363    : cast<StoreSDNode>(Op)->getAddressingMode();
364  ARM_AM::AddrOpc AddSub = (AM == ISD::PRE_INC || AM == ISD::POST_INC)
365    ? ARM_AM::add : ARM_AM::sub;
366  if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(N)) {
367    int Val = (int)C->getZExtValue();
368    if (Val >= 0 && Val < 0x1000) { // 12 bits.
369      Offset = CurDAG->getRegister(0, MVT::i32);
370      Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, Val,
371                                                        ARM_AM::no_shift),
372                                      MVT::i32);
373      return true;
374    }
375  }
376
377  Offset = N;
378  ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(N);
379  unsigned ShAmt = 0;
380  if (ShOpcVal != ARM_AM::no_shift) {
381    // Check to see if the RHS of the shift is a constant, if not, we can't fold
382    // it.
383    if (ConstantSDNode *Sh = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
384      ShAmt = Sh->getZExtValue();
385      Offset = N.getOperand(0);
386    } else {
387      ShOpcVal = ARM_AM::no_shift;
388    }
389  }
390
391  Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, ShAmt, ShOpcVal),
392                                  MVT::i32);
393  return true;
394}
395
396
397bool ARMDAGToDAGISel::SelectAddrMode3(SDValue N,
398                                      SDValue &Base, SDValue &Offset,
399                                      SDValue &Opc) {
400  if (N.getOpcode() == ISD::SUB) {
401    // X - C  is canonicalize to X + -C, no need to handle it here.
402    Base = N.getOperand(0);
403    Offset = N.getOperand(1);
404    Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(ARM_AM::sub, 0),MVT::i32);
405    return true;
406  }
407
408  if (N.getOpcode() != ISD::ADD) {
409    Base = N;
410    if (N.getOpcode() == ISD::FrameIndex) {
411      int FI = cast<FrameIndexSDNode>(N)->getIndex();
412      Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
413    }
414    Offset = CurDAG->getRegister(0, MVT::i32);
415    Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(ARM_AM::add, 0),MVT::i32);
416    return true;
417  }
418
419  // If the RHS is +/- imm8, fold into addr mode.
420  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
421    int RHSC = (int)RHS->getZExtValue();
422    if ((RHSC >= 0 && RHSC < 256) ||
423        (RHSC < 0 && RHSC > -256)) { // note -256 itself isn't allowed.
424      Base = N.getOperand(0);
425      if (Base.getOpcode() == ISD::FrameIndex) {
426        int FI = cast<FrameIndexSDNode>(Base)->getIndex();
427        Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
428      }
429      Offset = CurDAG->getRegister(0, MVT::i32);
430
431      ARM_AM::AddrOpc AddSub = ARM_AM::add;
432      if (RHSC < 0) {
433        AddSub = ARM_AM::sub;
434        RHSC = - RHSC;
435      }
436      Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(AddSub, RHSC),MVT::i32);
437      return true;
438    }
439  }
440
441  Base = N.getOperand(0);
442  Offset = N.getOperand(1);
443  Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(ARM_AM::add, 0), MVT::i32);
444  return true;
445}
446
447bool ARMDAGToDAGISel::SelectAddrMode3Offset(SDNode *Op, SDValue N,
448                                            SDValue &Offset, SDValue &Opc) {
449  unsigned Opcode = Op->getOpcode();
450  ISD::MemIndexedMode AM = (Opcode == ISD::LOAD)
451    ? cast<LoadSDNode>(Op)->getAddressingMode()
452    : cast<StoreSDNode>(Op)->getAddressingMode();
453  ARM_AM::AddrOpc AddSub = (AM == ISD::PRE_INC || AM == ISD::POST_INC)
454    ? ARM_AM::add : ARM_AM::sub;
455  if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(N)) {
456    int Val = (int)C->getZExtValue();
457    if (Val >= 0 && Val < 256) {
458      Offset = CurDAG->getRegister(0, MVT::i32);
459      Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(AddSub, Val), MVT::i32);
460      return true;
461    }
462  }
463
464  Offset = N;
465  Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(AddSub, 0), MVT::i32);
466  return true;
467}
468
469bool ARMDAGToDAGISel::SelectAddrMode4(SDValue N, SDValue &Addr, SDValue &Mode) {
470  Addr = N;
471  Mode = CurDAG->getTargetConstant(ARM_AM::getAM4ModeImm(ARM_AM::ia), MVT::i32);
472  return true;
473}
474
475bool ARMDAGToDAGISel::SelectAddrMode5(SDValue N,
476                                      SDValue &Base, SDValue &Offset) {
477  if (N.getOpcode() != ISD::ADD) {
478    Base = N;
479    if (N.getOpcode() == ISD::FrameIndex) {
480      int FI = cast<FrameIndexSDNode>(N)->getIndex();
481      Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
482    } else if (N.getOpcode() == ARMISD::Wrapper &&
483               !(Subtarget->useMovt() &&
484                 N.getOperand(0).getOpcode() == ISD::TargetGlobalAddress)) {
485      Base = N.getOperand(0);
486    }
487    Offset = CurDAG->getTargetConstant(ARM_AM::getAM5Opc(ARM_AM::add, 0),
488                                       MVT::i32);
489    return true;
490  }
491
492  // If the RHS is +/- imm8, fold into addr mode.
493  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
494    int RHSC = (int)RHS->getZExtValue();
495    if ((RHSC & 3) == 0) {  // The constant is implicitly multiplied by 4.
496      RHSC >>= 2;
497      if ((RHSC >= 0 && RHSC < 256) ||
498          (RHSC < 0 && RHSC > -256)) { // note -256 itself isn't allowed.
499        Base = N.getOperand(0);
500        if (Base.getOpcode() == ISD::FrameIndex) {
501          int FI = cast<FrameIndexSDNode>(Base)->getIndex();
502          Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
503        }
504
505        ARM_AM::AddrOpc AddSub = ARM_AM::add;
506        if (RHSC < 0) {
507          AddSub = ARM_AM::sub;
508          RHSC = - RHSC;
509        }
510        Offset = CurDAG->getTargetConstant(ARM_AM::getAM5Opc(AddSub, RHSC),
511                                           MVT::i32);
512        return true;
513      }
514    }
515  }
516
517  Base = N;
518  Offset = CurDAG->getTargetConstant(ARM_AM::getAM5Opc(ARM_AM::add, 0),
519                                     MVT::i32);
520  return true;
521}
522
523bool ARMDAGToDAGISel::SelectAddrMode6(SDValue N, SDValue &Addr, SDValue &Align){
524  Addr = N;
525  // Default to no alignment.
526  Align = CurDAG->getTargetConstant(0, MVT::i32);
527  return true;
528}
529
530bool ARMDAGToDAGISel::SelectAddrModePC(SDValue N,
531                                       SDValue &Offset, SDValue &Label) {
532  if (N.getOpcode() == ARMISD::PIC_ADD && N.hasOneUse()) {
533    Offset = N.getOperand(0);
534    SDValue N1 = N.getOperand(1);
535    Label  = CurDAG->getTargetConstant(cast<ConstantSDNode>(N1)->getZExtValue(),
536                                       MVT::i32);
537    return true;
538  }
539  return false;
540}
541
542bool ARMDAGToDAGISel::SelectThumbAddrModeRR(SDValue N,
543                                            SDValue &Base, SDValue &Offset){
544  // FIXME dl should come from the parent load or store, not the address
545  if (N.getOpcode() != ISD::ADD) {
546    ConstantSDNode *NC = dyn_cast<ConstantSDNode>(N);
547    if (!NC || !NC->isNullValue())
548      return false;
549
550    Base = Offset = N;
551    return true;
552  }
553
554  Base = N.getOperand(0);
555  Offset = N.getOperand(1);
556  return true;
557}
558
559bool
560ARMDAGToDAGISel::SelectThumbAddrModeRI5(SDValue N,
561                                        unsigned Scale, SDValue &Base,
562                                        SDValue &OffImm, SDValue &Offset) {
563  if (Scale == 4) {
564    SDValue TmpBase, TmpOffImm;
565    if (SelectThumbAddrModeSP(N, TmpBase, TmpOffImm))
566      return false;  // We want to select tLDRspi / tSTRspi instead.
567    if (N.getOpcode() == ARMISD::Wrapper &&
568        N.getOperand(0).getOpcode() == ISD::TargetConstantPool)
569      return false;  // We want to select tLDRpci instead.
570  }
571
572  if (N.getOpcode() != ISD::ADD) {
573    if (N.getOpcode() == ARMISD::Wrapper &&
574        !(Subtarget->useMovt() &&
575          N.getOperand(0).getOpcode() == ISD::TargetGlobalAddress)) {
576      Base = N.getOperand(0);
577    } else
578      Base = N;
579
580    Offset = CurDAG->getRegister(0, MVT::i32);
581    OffImm = CurDAG->getTargetConstant(0, MVT::i32);
582    return true;
583  }
584
585  // Thumb does not have [sp, r] address mode.
586  RegisterSDNode *LHSR = dyn_cast<RegisterSDNode>(N.getOperand(0));
587  RegisterSDNode *RHSR = dyn_cast<RegisterSDNode>(N.getOperand(1));
588  if ((LHSR && LHSR->getReg() == ARM::SP) ||
589      (RHSR && RHSR->getReg() == ARM::SP)) {
590    Base = N;
591    Offset = CurDAG->getRegister(0, MVT::i32);
592    OffImm = CurDAG->getTargetConstant(0, MVT::i32);
593    return true;
594  }
595
596  // If the RHS is + imm5 * scale, fold into addr mode.
597  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
598    int RHSC = (int)RHS->getZExtValue();
599    if ((RHSC & (Scale-1)) == 0) {  // The constant is implicitly multiplied.
600      RHSC /= Scale;
601      if (RHSC >= 0 && RHSC < 32) {
602        Base = N.getOperand(0);
603        Offset = CurDAG->getRegister(0, MVT::i32);
604        OffImm = CurDAG->getTargetConstant(RHSC, MVT::i32);
605        return true;
606      }
607    }
608  }
609
610  Base = N.getOperand(0);
611  Offset = N.getOperand(1);
612  OffImm = CurDAG->getTargetConstant(0, MVT::i32);
613  return true;
614}
615
616bool ARMDAGToDAGISel::SelectThumbAddrModeS1(SDValue N,
617                                            SDValue &Base, SDValue &OffImm,
618                                            SDValue &Offset) {
619  return SelectThumbAddrModeRI5(N, 1, Base, OffImm, Offset);
620}
621
622bool ARMDAGToDAGISel::SelectThumbAddrModeS2(SDValue N,
623                                            SDValue &Base, SDValue &OffImm,
624                                            SDValue &Offset) {
625  return SelectThumbAddrModeRI5(N, 2, Base, OffImm, Offset);
626}
627
628bool ARMDAGToDAGISel::SelectThumbAddrModeS4(SDValue N,
629                                            SDValue &Base, SDValue &OffImm,
630                                            SDValue &Offset) {
631  return SelectThumbAddrModeRI5(N, 4, Base, OffImm, Offset);
632}
633
634bool ARMDAGToDAGISel::SelectThumbAddrModeSP(SDValue N,
635                                            SDValue &Base, SDValue &OffImm) {
636  if (N.getOpcode() == ISD::FrameIndex) {
637    int FI = cast<FrameIndexSDNode>(N)->getIndex();
638    Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
639    OffImm = CurDAG->getTargetConstant(0, MVT::i32);
640    return true;
641  }
642
643  if (N.getOpcode() != ISD::ADD)
644    return false;
645
646  RegisterSDNode *LHSR = dyn_cast<RegisterSDNode>(N.getOperand(0));
647  if (N.getOperand(0).getOpcode() == ISD::FrameIndex ||
648      (LHSR && LHSR->getReg() == ARM::SP)) {
649    // If the RHS is + imm8 * scale, fold into addr mode.
650    if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
651      int RHSC = (int)RHS->getZExtValue();
652      if ((RHSC & 3) == 0) {  // The constant is implicitly multiplied.
653        RHSC >>= 2;
654        if (RHSC >= 0 && RHSC < 256) {
655          Base = N.getOperand(0);
656          if (Base.getOpcode() == ISD::FrameIndex) {
657            int FI = cast<FrameIndexSDNode>(Base)->getIndex();
658            Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
659          }
660          OffImm = CurDAG->getTargetConstant(RHSC, MVT::i32);
661          return true;
662        }
663      }
664    }
665  }
666
667  return false;
668}
669
670bool ARMDAGToDAGISel::SelectT2ShifterOperandReg(SDValue N, SDValue &BaseReg,
671                                                SDValue &Opc) {
672  if (DisableShifterOp)
673    return false;
674
675  ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(N);
676
677  // Don't match base register only case. That is matched to a separate
678  // lower complexity pattern with explicit register operand.
679  if (ShOpcVal == ARM_AM::no_shift) return false;
680
681  BaseReg = N.getOperand(0);
682  unsigned ShImmVal = 0;
683  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
684    ShImmVal = RHS->getZExtValue() & 31;
685    Opc = getI32Imm(ARM_AM::getSORegOpc(ShOpcVal, ShImmVal));
686    return true;
687  }
688
689  return false;
690}
691
692bool ARMDAGToDAGISel::SelectT2AddrModeImm12(SDValue N,
693                                            SDValue &Base, SDValue &OffImm) {
694  // Match simple R + imm12 operands.
695
696  // Base only.
697  if (N.getOpcode() != ISD::ADD && N.getOpcode() != ISD::SUB) {
698    if (N.getOpcode() == ISD::FrameIndex) {
699      // Match frame index...
700      int FI = cast<FrameIndexSDNode>(N)->getIndex();
701      Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
702      OffImm  = CurDAG->getTargetConstant(0, MVT::i32);
703      return true;
704    } else if (N.getOpcode() == ARMISD::Wrapper &&
705               !(Subtarget->useMovt() &&
706                 N.getOperand(0).getOpcode() == ISD::TargetGlobalAddress)) {
707      Base = N.getOperand(0);
708      if (Base.getOpcode() == ISD::TargetConstantPool)
709        return false;  // We want to select t2LDRpci instead.
710    } else
711      Base = N;
712    OffImm  = CurDAG->getTargetConstant(0, MVT::i32);
713    return true;
714  }
715
716  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
717    if (SelectT2AddrModeImm8(N, Base, OffImm))
718      // Let t2LDRi8 handle (R - imm8).
719      return false;
720
721    int RHSC = (int)RHS->getZExtValue();
722    if (N.getOpcode() == ISD::SUB)
723      RHSC = -RHSC;
724
725    if (RHSC >= 0 && RHSC < 0x1000) { // 12 bits (unsigned)
726      Base   = N.getOperand(0);
727      if (Base.getOpcode() == ISD::FrameIndex) {
728        int FI = cast<FrameIndexSDNode>(Base)->getIndex();
729        Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
730      }
731      OffImm = CurDAG->getTargetConstant(RHSC, MVT::i32);
732      return true;
733    }
734  }
735
736  // Base only.
737  Base = N;
738  OffImm  = CurDAG->getTargetConstant(0, MVT::i32);
739  return true;
740}
741
742bool ARMDAGToDAGISel::SelectT2AddrModeImm8(SDValue N,
743                                           SDValue &Base, SDValue &OffImm) {
744  // Match simple R - imm8 operands.
745  if (N.getOpcode() == ISD::ADD || N.getOpcode() == ISD::SUB) {
746    if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
747      int RHSC = (int)RHS->getSExtValue();
748      if (N.getOpcode() == ISD::SUB)
749        RHSC = -RHSC;
750
751      if ((RHSC >= -255) && (RHSC < 0)) { // 8 bits (always negative)
752        Base = N.getOperand(0);
753        if (Base.getOpcode() == ISD::FrameIndex) {
754          int FI = cast<FrameIndexSDNode>(Base)->getIndex();
755          Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
756        }
757        OffImm = CurDAG->getTargetConstant(RHSC, MVT::i32);
758        return true;
759      }
760    }
761  }
762
763  return false;
764}
765
766bool ARMDAGToDAGISel::SelectT2AddrModeImm8Offset(SDNode *Op, SDValue N,
767                                                 SDValue &OffImm){
768  unsigned Opcode = Op->getOpcode();
769  ISD::MemIndexedMode AM = (Opcode == ISD::LOAD)
770    ? cast<LoadSDNode>(Op)->getAddressingMode()
771    : cast<StoreSDNode>(Op)->getAddressingMode();
772  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N)) {
773    int RHSC = (int)RHS->getZExtValue();
774    if (RHSC >= 0 && RHSC < 0x100) { // 8 bits.
775      OffImm = ((AM == ISD::PRE_INC) || (AM == ISD::POST_INC))
776        ? CurDAG->getTargetConstant(RHSC, MVT::i32)
777        : CurDAG->getTargetConstant(-RHSC, MVT::i32);
778      return true;
779    }
780  }
781
782  return false;
783}
784
785bool ARMDAGToDAGISel::SelectT2AddrModeSoReg(SDValue N,
786                                            SDValue &Base,
787                                            SDValue &OffReg, SDValue &ShImm) {
788  // (R - imm8) should be handled by t2LDRi8. The rest are handled by t2LDRi12.
789  if (N.getOpcode() != ISD::ADD)
790    return false;
791
792  // Leave (R + imm12) for t2LDRi12, (R - imm8) for t2LDRi8.
793  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
794    int RHSC = (int)RHS->getZExtValue();
795    if (RHSC >= 0 && RHSC < 0x1000) // 12 bits (unsigned)
796      return false;
797    else if (RHSC < 0 && RHSC >= -255) // 8 bits
798      return false;
799  }
800
801  // Look for (R + R) or (R + (R << [1,2,3])).
802  unsigned ShAmt = 0;
803  Base   = N.getOperand(0);
804  OffReg = N.getOperand(1);
805
806  // Swap if it is ((R << c) + R).
807  ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(OffReg);
808  if (ShOpcVal != ARM_AM::lsl) {
809    ShOpcVal = ARM_AM::getShiftOpcForNode(Base);
810    if (ShOpcVal == ARM_AM::lsl)
811      std::swap(Base, OffReg);
812  }
813
814  if (ShOpcVal == ARM_AM::lsl) {
815    // Check to see if the RHS of the shift is a constant, if not, we can't fold
816    // it.
817    if (ConstantSDNode *Sh = dyn_cast<ConstantSDNode>(OffReg.getOperand(1))) {
818      ShAmt = Sh->getZExtValue();
819      if (ShAmt >= 4) {
820        ShAmt = 0;
821        ShOpcVal = ARM_AM::no_shift;
822      } else
823        OffReg = OffReg.getOperand(0);
824    } else {
825      ShOpcVal = ARM_AM::no_shift;
826    }
827  }
828
829  ShImm = CurDAG->getTargetConstant(ShAmt, MVT::i32);
830
831  return true;
832}
833
834//===--------------------------------------------------------------------===//
835
836/// getAL - Returns a ARMCC::AL immediate node.
837static inline SDValue getAL(SelectionDAG *CurDAG) {
838  return CurDAG->getTargetConstant((uint64_t)ARMCC::AL, MVT::i32);
839}
840
841SDNode *ARMDAGToDAGISel::SelectARMIndexedLoad(SDNode *N) {
842  LoadSDNode *LD = cast<LoadSDNode>(N);
843  ISD::MemIndexedMode AM = LD->getAddressingMode();
844  if (AM == ISD::UNINDEXED)
845    return NULL;
846
847  EVT LoadedVT = LD->getMemoryVT();
848  SDValue Offset, AMOpc;
849  bool isPre = (AM == ISD::PRE_INC) || (AM == ISD::PRE_DEC);
850  unsigned Opcode = 0;
851  bool Match = false;
852  if (LoadedVT == MVT::i32 &&
853      SelectAddrMode2Offset(N, LD->getOffset(), Offset, AMOpc)) {
854    Opcode = isPre ? ARM::LDR_PRE : ARM::LDR_POST;
855    Match = true;
856  } else if (LoadedVT == MVT::i16 &&
857             SelectAddrMode3Offset(N, LD->getOffset(), Offset, AMOpc)) {
858    Match = true;
859    Opcode = (LD->getExtensionType() == ISD::SEXTLOAD)
860      ? (isPre ? ARM::LDRSH_PRE : ARM::LDRSH_POST)
861      : (isPre ? ARM::LDRH_PRE : ARM::LDRH_POST);
862  } else if (LoadedVT == MVT::i8 || LoadedVT == MVT::i1) {
863    if (LD->getExtensionType() == ISD::SEXTLOAD) {
864      if (SelectAddrMode3Offset(N, LD->getOffset(), Offset, AMOpc)) {
865        Match = true;
866        Opcode = isPre ? ARM::LDRSB_PRE : ARM::LDRSB_POST;
867      }
868    } else {
869      if (SelectAddrMode2Offset(N, LD->getOffset(), Offset, AMOpc)) {
870        Match = true;
871        Opcode = isPre ? ARM::LDRB_PRE : ARM::LDRB_POST;
872      }
873    }
874  }
875
876  if (Match) {
877    SDValue Chain = LD->getChain();
878    SDValue Base = LD->getBasePtr();
879    SDValue Ops[]= { Base, Offset, AMOpc, getAL(CurDAG),
880                     CurDAG->getRegister(0, MVT::i32), Chain };
881    return CurDAG->getMachineNode(Opcode, N->getDebugLoc(), MVT::i32, MVT::i32,
882                                  MVT::Other, Ops, 6);
883  }
884
885  return NULL;
886}
887
888SDNode *ARMDAGToDAGISel::SelectT2IndexedLoad(SDNode *N) {
889  LoadSDNode *LD = cast<LoadSDNode>(N);
890  ISD::MemIndexedMode AM = LD->getAddressingMode();
891  if (AM == ISD::UNINDEXED)
892    return NULL;
893
894  EVT LoadedVT = LD->getMemoryVT();
895  bool isSExtLd = LD->getExtensionType() == ISD::SEXTLOAD;
896  SDValue Offset;
897  bool isPre = (AM == ISD::PRE_INC) || (AM == ISD::PRE_DEC);
898  unsigned Opcode = 0;
899  bool Match = false;
900  if (SelectT2AddrModeImm8Offset(N, LD->getOffset(), Offset)) {
901    switch (LoadedVT.getSimpleVT().SimpleTy) {
902    case MVT::i32:
903      Opcode = isPre ? ARM::t2LDR_PRE : ARM::t2LDR_POST;
904      break;
905    case MVT::i16:
906      if (isSExtLd)
907        Opcode = isPre ? ARM::t2LDRSH_PRE : ARM::t2LDRSH_POST;
908      else
909        Opcode = isPre ? ARM::t2LDRH_PRE : ARM::t2LDRH_POST;
910      break;
911    case MVT::i8:
912    case MVT::i1:
913      if (isSExtLd)
914        Opcode = isPre ? ARM::t2LDRSB_PRE : ARM::t2LDRSB_POST;
915      else
916        Opcode = isPre ? ARM::t2LDRB_PRE : ARM::t2LDRB_POST;
917      break;
918    default:
919      return NULL;
920    }
921    Match = true;
922  }
923
924  if (Match) {
925    SDValue Chain = LD->getChain();
926    SDValue Base = LD->getBasePtr();
927    SDValue Ops[]= { Base, Offset, getAL(CurDAG),
928                     CurDAG->getRegister(0, MVT::i32), Chain };
929    return CurDAG->getMachineNode(Opcode, N->getDebugLoc(), MVT::i32, MVT::i32,
930                                  MVT::Other, Ops, 5);
931  }
932
933  return NULL;
934}
935
936/// PairSRegs - Form a D register from a pair of S registers.
937///
938SDNode *ARMDAGToDAGISel::PairSRegs(EVT VT, SDValue V0, SDValue V1) {
939  DebugLoc dl = V0.getNode()->getDebugLoc();
940  SDValue SubReg0 = CurDAG->getTargetConstant(ARM::ssub_0, MVT::i32);
941  SDValue SubReg1 = CurDAG->getTargetConstant(ARM::ssub_1, MVT::i32);
942  const SDValue Ops[] = { V0, SubReg0, V1, SubReg1 };
943  return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops, 4);
944}
945
946/// PairDRegs - Form a quad register from a pair of D registers.
947///
948SDNode *ARMDAGToDAGISel::PairDRegs(EVT VT, SDValue V0, SDValue V1) {
949  DebugLoc dl = V0.getNode()->getDebugLoc();
950  SDValue SubReg0 = CurDAG->getTargetConstant(ARM::dsub_0, MVT::i32);
951  SDValue SubReg1 = CurDAG->getTargetConstant(ARM::dsub_1, MVT::i32);
952  const SDValue Ops[] = { V0, SubReg0, V1, SubReg1 };
953  return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops, 4);
954}
955
956/// PairQRegs - Form 4 consecutive D registers from a pair of Q registers.
957///
958SDNode *ARMDAGToDAGISel::PairQRegs(EVT VT, SDValue V0, SDValue V1) {
959  DebugLoc dl = V0.getNode()->getDebugLoc();
960  SDValue SubReg0 = CurDAG->getTargetConstant(ARM::qsub_0, MVT::i32);
961  SDValue SubReg1 = CurDAG->getTargetConstant(ARM::qsub_1, MVT::i32);
962  const SDValue Ops[] = { V0, SubReg0, V1, SubReg1 };
963  return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops, 4);
964}
965
966/// QuadSRegs - Form 4 consecutive S registers.
967///
968SDNode *ARMDAGToDAGISel::QuadSRegs(EVT VT, SDValue V0, SDValue V1,
969                                   SDValue V2, SDValue V3) {
970  DebugLoc dl = V0.getNode()->getDebugLoc();
971  SDValue SubReg0 = CurDAG->getTargetConstant(ARM::ssub_0, MVT::i32);
972  SDValue SubReg1 = CurDAG->getTargetConstant(ARM::ssub_1, MVT::i32);
973  SDValue SubReg2 = CurDAG->getTargetConstant(ARM::ssub_2, MVT::i32);
974  SDValue SubReg3 = CurDAG->getTargetConstant(ARM::ssub_3, MVT::i32);
975  const SDValue Ops[] = { V0, SubReg0, V1, SubReg1, V2, SubReg2, V3, SubReg3 };
976  return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops, 8);
977}
978
979/// QuadDRegs - Form 4 consecutive D registers.
980///
981SDNode *ARMDAGToDAGISel::QuadDRegs(EVT VT, SDValue V0, SDValue V1,
982                                   SDValue V2, SDValue V3) {
983  DebugLoc dl = V0.getNode()->getDebugLoc();
984  SDValue SubReg0 = CurDAG->getTargetConstant(ARM::dsub_0, MVT::i32);
985  SDValue SubReg1 = CurDAG->getTargetConstant(ARM::dsub_1, MVT::i32);
986  SDValue SubReg2 = CurDAG->getTargetConstant(ARM::dsub_2, MVT::i32);
987  SDValue SubReg3 = CurDAG->getTargetConstant(ARM::dsub_3, MVT::i32);
988  const SDValue Ops[] = { V0, SubReg0, V1, SubReg1, V2, SubReg2, V3, SubReg3 };
989  return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops, 8);
990}
991
992/// QuadQRegs - Form 4 consecutive Q registers.
993///
994SDNode *ARMDAGToDAGISel::QuadQRegs(EVT VT, SDValue V0, SDValue V1,
995                                   SDValue V2, SDValue V3) {
996  DebugLoc dl = V0.getNode()->getDebugLoc();
997  SDValue SubReg0 = CurDAG->getTargetConstant(ARM::qsub_0, MVT::i32);
998  SDValue SubReg1 = CurDAG->getTargetConstant(ARM::qsub_1, MVT::i32);
999  SDValue SubReg2 = CurDAG->getTargetConstant(ARM::qsub_2, MVT::i32);
1000  SDValue SubReg3 = CurDAG->getTargetConstant(ARM::qsub_3, MVT::i32);
1001  const SDValue Ops[] = { V0, SubReg0, V1, SubReg1, V2, SubReg2, V3, SubReg3 };
1002  return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops, 8);
1003}
1004
1005/// GetVLDSTAlign - Get the alignment (in bytes) for the alignment operand
1006/// of a NEON VLD or VST instruction.  The supported values depend on the
1007/// number of registers being loaded.
1008static unsigned GetVLDSTAlign(SDNode *N, unsigned NumVecs, bool is64BitVector) {
1009  unsigned NumRegs = NumVecs;
1010  if (!is64BitVector && NumVecs < 3)
1011    NumRegs *= 2;
1012
1013  unsigned Alignment = cast<MemIntrinsicSDNode>(N)->getAlignment();
1014  if (Alignment >= 32 && NumRegs == 4)
1015    return 32;
1016  if (Alignment >= 16 && (NumRegs == 2 || NumRegs == 4))
1017    return 16;
1018  if (Alignment >= 8)
1019    return 8;
1020  return 0;
1021}
1022
1023SDNode *ARMDAGToDAGISel::SelectVLD(SDNode *N, unsigned NumVecs,
1024                                   unsigned *DOpcodes, unsigned *QOpcodes0,
1025                                   unsigned *QOpcodes1) {
1026  assert(NumVecs >= 1 && NumVecs <= 4 && "VLD NumVecs out-of-range");
1027  DebugLoc dl = N->getDebugLoc();
1028
1029  SDValue MemAddr, Align;
1030  if (!SelectAddrMode6(N->getOperand(2), MemAddr, Align))
1031    return NULL;
1032
1033  SDValue Chain = N->getOperand(0);
1034  EVT VT = N->getValueType(0);
1035  bool is64BitVector = VT.is64BitVector();
1036
1037  unsigned Alignment = GetVLDSTAlign(N, NumVecs, is64BitVector);
1038  Align = CurDAG->getTargetConstant(Alignment, MVT::i32);
1039
1040  unsigned OpcodeIndex;
1041  switch (VT.getSimpleVT().SimpleTy) {
1042  default: llvm_unreachable("unhandled vld type");
1043    // Double-register operations:
1044  case MVT::v8i8:  OpcodeIndex = 0; break;
1045  case MVT::v4i16: OpcodeIndex = 1; break;
1046  case MVT::v2f32:
1047  case MVT::v2i32: OpcodeIndex = 2; break;
1048  case MVT::v1i64: OpcodeIndex = 3; break;
1049    // Quad-register operations:
1050  case MVT::v16i8: OpcodeIndex = 0; break;
1051  case MVT::v8i16: OpcodeIndex = 1; break;
1052  case MVT::v4f32:
1053  case MVT::v4i32: OpcodeIndex = 2; break;
1054  case MVT::v2i64: OpcodeIndex = 3;
1055    assert(NumVecs == 1 && "v2i64 type only supported for VLD1");
1056    break;
1057  }
1058
1059  EVT ResTy;
1060  if (NumVecs == 1)
1061    ResTy = VT;
1062  else {
1063    unsigned ResTyElts = (NumVecs == 3) ? 4 : NumVecs;
1064    if (!is64BitVector)
1065      ResTyElts *= 2;
1066    ResTy = EVT::getVectorVT(*CurDAG->getContext(), MVT::i64, ResTyElts);
1067  }
1068
1069  SDValue Pred = getAL(CurDAG);
1070  SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
1071  SDValue SuperReg;
1072  if (is64BitVector) {
1073    unsigned Opc = DOpcodes[OpcodeIndex];
1074    const SDValue Ops[] = { MemAddr, Align, Pred, Reg0, Chain };
1075    SDNode *VLd = CurDAG->getMachineNode(Opc, dl, ResTy, MVT::Other, Ops, 5);
1076    if (NumVecs == 1)
1077      return VLd;
1078
1079    SuperReg = SDValue(VLd, 0);
1080    assert(ARM::dsub_7 == ARM::dsub_0+7 && "Unexpected subreg numbering");
1081    for (unsigned Vec = 0; Vec < NumVecs; ++Vec) {
1082      SDValue D = CurDAG->getTargetExtractSubreg(ARM::dsub_0+Vec,
1083                                                 dl, VT, SuperReg);
1084      ReplaceUses(SDValue(N, Vec), D);
1085    }
1086    ReplaceUses(SDValue(N, NumVecs), SDValue(VLd, 1));
1087    return NULL;
1088  }
1089
1090  if (NumVecs <= 2) {
1091    // Quad registers are directly supported for VLD1 and VLD2,
1092    // loading pairs of D regs.
1093    unsigned Opc = QOpcodes0[OpcodeIndex];
1094    const SDValue Ops[] = { MemAddr, Align, Pred, Reg0, Chain };
1095    SDNode *VLd = CurDAG->getMachineNode(Opc, dl, ResTy, MVT::Other, Ops, 5);
1096    if (NumVecs == 1)
1097      return VLd;
1098
1099    SuperReg = SDValue(VLd, 0);
1100    Chain = SDValue(VLd, 1);
1101
1102  } else {
1103    // Otherwise, quad registers are loaded with two separate instructions,
1104    // where one loads the even registers and the other loads the odd registers.
1105    EVT AddrTy = MemAddr.getValueType();
1106
1107    // Load the even subregs.
1108    unsigned Opc = QOpcodes0[OpcodeIndex];
1109    SDValue ImplDef =
1110      SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, dl, ResTy), 0);
1111    const SDValue OpsA[] = { MemAddr, Align, Reg0, ImplDef, Pred, Reg0, Chain };
1112    SDNode *VLdA =
1113      CurDAG->getMachineNode(Opc, dl, ResTy, AddrTy, MVT::Other, OpsA, 7);
1114    Chain = SDValue(VLdA, 2);
1115
1116    // Load the odd subregs.
1117    Opc = QOpcodes1[OpcodeIndex];
1118    const SDValue OpsB[] = { SDValue(VLdA, 1), Align, Reg0, SDValue(VLdA, 0),
1119                             Pred, Reg0, Chain };
1120    SDNode *VLdB =
1121      CurDAG->getMachineNode(Opc, dl, ResTy, AddrTy, MVT::Other, OpsB, 7);
1122    SuperReg = SDValue(VLdB, 0);
1123    Chain = SDValue(VLdB, 2);
1124  }
1125
1126  // Extract out the Q registers.
1127  assert(ARM::qsub_3 == ARM::qsub_0+3 && "Unexpected subreg numbering");
1128  for (unsigned Vec = 0; Vec < NumVecs; ++Vec) {
1129    SDValue Q = CurDAG->getTargetExtractSubreg(ARM::qsub_0+Vec,
1130                                               dl, VT, SuperReg);
1131    ReplaceUses(SDValue(N, Vec), Q);
1132  }
1133  ReplaceUses(SDValue(N, NumVecs), Chain);
1134  return NULL;
1135}
1136
1137SDNode *ARMDAGToDAGISel::SelectVST(SDNode *N, unsigned NumVecs,
1138                                   unsigned *DOpcodes, unsigned *QOpcodes0,
1139                                   unsigned *QOpcodes1) {
1140  assert(NumVecs >= 1 && NumVecs <= 4 && "VST NumVecs out-of-range");
1141  DebugLoc dl = N->getDebugLoc();
1142
1143  SDValue MemAddr, Align;
1144  if (!SelectAddrMode6(N->getOperand(2), MemAddr, Align))
1145    return NULL;
1146
1147  SDValue Chain = N->getOperand(0);
1148  EVT VT = N->getOperand(3).getValueType();
1149  bool is64BitVector = VT.is64BitVector();
1150
1151  unsigned Alignment = GetVLDSTAlign(N, NumVecs, is64BitVector);
1152  Align = CurDAG->getTargetConstant(Alignment, MVT::i32);
1153
1154  unsigned OpcodeIndex;
1155  switch (VT.getSimpleVT().SimpleTy) {
1156  default: llvm_unreachable("unhandled vst type");
1157    // Double-register operations:
1158  case MVT::v8i8:  OpcodeIndex = 0; break;
1159  case MVT::v4i16: OpcodeIndex = 1; break;
1160  case MVT::v2f32:
1161  case MVT::v2i32: OpcodeIndex = 2; break;
1162  case MVT::v1i64: OpcodeIndex = 3; break;
1163    // Quad-register operations:
1164  case MVT::v16i8: OpcodeIndex = 0; break;
1165  case MVT::v8i16: OpcodeIndex = 1; break;
1166  case MVT::v4f32:
1167  case MVT::v4i32: OpcodeIndex = 2; break;
1168  case MVT::v2i64: OpcodeIndex = 3;
1169    assert(NumVecs == 1 && "v2i64 type only supported for VST1");
1170    break;
1171  }
1172
1173  SDValue Pred = getAL(CurDAG);
1174  SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
1175
1176  SmallVector<SDValue, 7> Ops;
1177  Ops.push_back(MemAddr);
1178  Ops.push_back(Align);
1179
1180  if (is64BitVector) {
1181    if (NumVecs == 1) {
1182      Ops.push_back(N->getOperand(3));
1183    } else {
1184      SDValue RegSeq;
1185      SDValue V0 = N->getOperand(0+3);
1186      SDValue V1 = N->getOperand(1+3);
1187
1188      // Form a REG_SEQUENCE to force register allocation.
1189      if (NumVecs == 2)
1190        RegSeq = SDValue(PairDRegs(MVT::v2i64, V0, V1), 0);
1191      else {
1192        SDValue V2 = N->getOperand(2+3);
1193        // If it's a vld3, form a quad D-register and leave the last part as
1194        // an undef.
1195        SDValue V3 = (NumVecs == 3)
1196          ? SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF,dl,VT), 0)
1197          : N->getOperand(3+3);
1198        RegSeq = SDValue(QuadDRegs(MVT::v4i64, V0, V1, V2, V3), 0);
1199      }
1200      Ops.push_back(RegSeq);
1201    }
1202    Ops.push_back(Pred);
1203    Ops.push_back(Reg0); // predicate register
1204    Ops.push_back(Chain);
1205    unsigned Opc = DOpcodes[OpcodeIndex];
1206    return CurDAG->getMachineNode(Opc, dl, MVT::Other, Ops.data(), 6);
1207  }
1208
1209  if (NumVecs <= 2) {
1210    // Quad registers are directly supported for VST1 and VST2.
1211    unsigned Opc = QOpcodes0[OpcodeIndex];
1212    if (NumVecs == 1) {
1213      Ops.push_back(N->getOperand(3));
1214    } else {
1215      // Form a QQ register.
1216      SDValue Q0 = N->getOperand(3);
1217      SDValue Q1 = N->getOperand(4);
1218      Ops.push_back(SDValue(PairQRegs(MVT::v4i64, Q0, Q1), 0));
1219    }
1220    Ops.push_back(Pred);
1221    Ops.push_back(Reg0); // predicate register
1222    Ops.push_back(Chain);
1223    return CurDAG->getMachineNode(Opc, dl, MVT::Other, Ops.data(), 6);
1224  }
1225
1226  // Otherwise, quad registers are stored with two separate instructions,
1227  // where one stores the even registers and the other stores the odd registers.
1228
1229  // Form the QQQQ REG_SEQUENCE.
1230  SDValue V0 = N->getOperand(0+3);
1231  SDValue V1 = N->getOperand(1+3);
1232  SDValue V2 = N->getOperand(2+3);
1233  SDValue V3 = (NumVecs == 3)
1234    ? SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, dl, VT), 0)
1235    : N->getOperand(3+3);
1236  SDValue RegSeq = SDValue(QuadQRegs(MVT::v8i64, V0, V1, V2, V3), 0);
1237
1238  // Store the even D registers.
1239  Ops.push_back(Reg0); // post-access address offset
1240  Ops.push_back(RegSeq);
1241  Ops.push_back(Pred);
1242  Ops.push_back(Reg0); // predicate register
1243  Ops.push_back(Chain);
1244  unsigned Opc = QOpcodes0[OpcodeIndex];
1245  SDNode *VStA = CurDAG->getMachineNode(Opc, dl, MemAddr.getValueType(),
1246                                        MVT::Other, Ops.data(), 7);
1247  Chain = SDValue(VStA, 1);
1248
1249  // Store the odd D registers.
1250  Ops[0] = SDValue(VStA, 0); // MemAddr
1251  Ops[6] = Chain;
1252  Opc = QOpcodes1[OpcodeIndex];
1253  SDNode *VStB = CurDAG->getMachineNode(Opc, dl, MemAddr.getValueType(),
1254                                        MVT::Other, Ops.data(), 7);
1255  Chain = SDValue(VStB, 1);
1256  ReplaceUses(SDValue(N, 0), Chain);
1257  return NULL;
1258}
1259
1260SDNode *ARMDAGToDAGISel::SelectVLDSTLane(SDNode *N, bool IsLoad,
1261                                         unsigned NumVecs, unsigned *DOpcodes,
1262                                         unsigned *QOpcodes) {
1263  assert(NumVecs >=2 && NumVecs <= 4 && "VLDSTLane NumVecs out-of-range");
1264  DebugLoc dl = N->getDebugLoc();
1265
1266  SDValue MemAddr, Align;
1267  if (!SelectAddrMode6(N->getOperand(2), MemAddr, Align))
1268    return NULL;
1269
1270  SDValue Chain = N->getOperand(0);
1271  unsigned Lane =
1272    cast<ConstantSDNode>(N->getOperand(NumVecs+3))->getZExtValue();
1273  EVT VT = IsLoad ? N->getValueType(0) : N->getOperand(3).getValueType();
1274  bool is64BitVector = VT.is64BitVector();
1275
1276  unsigned OpcodeIndex;
1277  switch (VT.getSimpleVT().SimpleTy) {
1278  default: llvm_unreachable("unhandled vld/vst lane type");
1279    // Double-register operations:
1280  case MVT::v8i8:  OpcodeIndex = 0; break;
1281  case MVT::v4i16: OpcodeIndex = 1; break;
1282  case MVT::v2f32:
1283  case MVT::v2i32: OpcodeIndex = 2; break;
1284    // Quad-register operations:
1285  case MVT::v8i16: OpcodeIndex = 0; break;
1286  case MVT::v4f32:
1287  case MVT::v4i32: OpcodeIndex = 1; break;
1288  }
1289
1290  SDValue Pred = getAL(CurDAG);
1291  SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
1292
1293  SmallVector<SDValue, 7> Ops;
1294  Ops.push_back(MemAddr);
1295  Ops.push_back(Align);
1296
1297  unsigned Opc = (is64BitVector ? DOpcodes[OpcodeIndex] :
1298                                  QOpcodes[OpcodeIndex]);
1299
1300  SDValue SuperReg;
1301  SDValue V0 = N->getOperand(0+3);
1302  SDValue V1 = N->getOperand(1+3);
1303  if (NumVecs == 2) {
1304    if (is64BitVector)
1305      SuperReg = SDValue(PairDRegs(MVT::v2i64, V0, V1), 0);
1306    else
1307      SuperReg = SDValue(PairQRegs(MVT::v4i64, V0, V1), 0);
1308  } else {
1309    SDValue V2 = N->getOperand(2+3);
1310    SDValue V3 = (NumVecs == 3)
1311      ? SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF,dl,VT), 0)
1312      : N->getOperand(3+3);
1313    if (is64BitVector)
1314      SuperReg = SDValue(QuadDRegs(MVT::v4i64, V0, V1, V2, V3), 0);
1315    else
1316      SuperReg = SDValue(QuadQRegs(MVT::v8i64, V0, V1, V2, V3), 0);
1317  }
1318  Ops.push_back(SuperReg);
1319  Ops.push_back(getI32Imm(Lane));
1320  Ops.push_back(Pred);
1321  Ops.push_back(Reg0);
1322  Ops.push_back(Chain);
1323
1324  if (!IsLoad)
1325    return CurDAG->getMachineNode(Opc, dl, MVT::Other, Ops.data(), 7);
1326
1327  EVT ResTy;
1328  unsigned ResTyElts = (NumVecs == 3) ? 4 : NumVecs;
1329  if (!is64BitVector)
1330    ResTyElts *= 2;
1331  ResTy = EVT::getVectorVT(*CurDAG->getContext(), MVT::i64, ResTyElts);
1332
1333  SDNode *VLdLn = CurDAG->getMachineNode(Opc, dl, ResTy, MVT::Other,
1334                                         Ops.data(), 7);
1335  SuperReg = SDValue(VLdLn, 0);
1336  Chain = SDValue(VLdLn, 1);
1337
1338  // Extract the subregisters.
1339  assert(ARM::dsub_7 == ARM::dsub_0+7 && "Unexpected subreg numbering");
1340  assert(ARM::qsub_3 == ARM::qsub_0+3 && "Unexpected subreg numbering");
1341  unsigned SubIdx = is64BitVector ? ARM::dsub_0 : ARM::qsub_0;
1342  for (unsigned Vec = 0; Vec < NumVecs; ++Vec)
1343    ReplaceUses(SDValue(N, Vec),
1344                CurDAG->getTargetExtractSubreg(SubIdx+Vec, dl, VT, SuperReg));
1345  ReplaceUses(SDValue(N, NumVecs), Chain);
1346  return NULL;
1347}
1348
1349SDNode *ARMDAGToDAGISel::SelectVTBL(SDNode *N, bool IsExt, unsigned NumVecs,
1350                                    unsigned Opc) {
1351  assert(NumVecs >= 2 && NumVecs <= 4 && "VTBL NumVecs out-of-range");
1352  DebugLoc dl = N->getDebugLoc();
1353  EVT VT = N->getValueType(0);
1354  unsigned FirstTblReg = IsExt ? 2 : 1;
1355
1356  // Form a REG_SEQUENCE to force register allocation.
1357  SDValue RegSeq;
1358  SDValue V0 = N->getOperand(FirstTblReg + 0);
1359  SDValue V1 = N->getOperand(FirstTblReg + 1);
1360  if (NumVecs == 2)
1361    RegSeq = SDValue(PairDRegs(MVT::v16i8, V0, V1), 0);
1362  else {
1363    SDValue V2 = N->getOperand(FirstTblReg + 2);
1364    // If it's a vtbl3, form a quad D-register and leave the last part as
1365    // an undef.
1366    SDValue V3 = (NumVecs == 3)
1367      ? SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, dl, VT), 0)
1368      : N->getOperand(FirstTblReg + 3);
1369    RegSeq = SDValue(QuadDRegs(MVT::v4i64, V0, V1, V2, V3), 0);
1370  }
1371
1372  SmallVector<SDValue, 6> Ops;
1373  if (IsExt)
1374    Ops.push_back(N->getOperand(1));
1375  Ops.push_back(RegSeq);
1376  Ops.push_back(N->getOperand(FirstTblReg + NumVecs));
1377  Ops.push_back(getAL(CurDAG)); // predicate
1378  Ops.push_back(CurDAG->getRegister(0, MVT::i32)); // predicate register
1379  return CurDAG->getMachineNode(Opc, dl, VT, Ops.data(), Ops.size());
1380}
1381
1382SDNode *ARMDAGToDAGISel::SelectV6T2BitfieldExtractOp(SDNode *N,
1383                                                     bool isSigned) {
1384  if (!Subtarget->hasV6T2Ops())
1385    return NULL;
1386
1387  unsigned Opc = isSigned ? (Subtarget->isThumb() ? ARM::t2SBFX : ARM::SBFX)
1388    : (Subtarget->isThumb() ? ARM::t2UBFX : ARM::UBFX);
1389
1390
1391  // For unsigned extracts, check for a shift right and mask
1392  unsigned And_imm = 0;
1393  if (N->getOpcode() == ISD::AND) {
1394    if (isOpcWithIntImmediate(N, ISD::AND, And_imm)) {
1395
1396      // The immediate is a mask of the low bits iff imm & (imm+1) == 0
1397      if (And_imm & (And_imm + 1))
1398        return NULL;
1399
1400      unsigned Srl_imm = 0;
1401      if (isOpcWithIntImmediate(N->getOperand(0).getNode(), ISD::SRL,
1402                                Srl_imm)) {
1403        assert(Srl_imm > 0 && Srl_imm < 32 && "bad amount in shift node!");
1404
1405        unsigned Width = CountTrailingOnes_32(And_imm);
1406        unsigned LSB = Srl_imm;
1407        SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
1408        SDValue Ops[] = { N->getOperand(0).getOperand(0),
1409                          CurDAG->getTargetConstant(LSB, MVT::i32),
1410                          CurDAG->getTargetConstant(Width, MVT::i32),
1411          getAL(CurDAG), Reg0 };
1412        return CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops, 5);
1413      }
1414    }
1415    return NULL;
1416  }
1417
1418  // Otherwise, we're looking for a shift of a shift
1419  unsigned Shl_imm = 0;
1420  if (isOpcWithIntImmediate(N->getOperand(0).getNode(), ISD::SHL, Shl_imm)) {
1421    assert(Shl_imm > 0 && Shl_imm < 32 && "bad amount in shift node!");
1422    unsigned Srl_imm = 0;
1423    if (isInt32Immediate(N->getOperand(1), Srl_imm)) {
1424      assert(Srl_imm > 0 && Srl_imm < 32 && "bad amount in shift node!");
1425      unsigned Width = 32 - Srl_imm;
1426      int LSB = Srl_imm - Shl_imm;
1427      if (LSB < 0)
1428        return NULL;
1429      SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
1430      SDValue Ops[] = { N->getOperand(0).getOperand(0),
1431                        CurDAG->getTargetConstant(LSB, MVT::i32),
1432                        CurDAG->getTargetConstant(Width, MVT::i32),
1433                        getAL(CurDAG), Reg0 };
1434      return CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops, 5);
1435    }
1436  }
1437  return NULL;
1438}
1439
1440SDNode *ARMDAGToDAGISel::
1441SelectT2CMOVShiftOp(SDNode *N, SDValue FalseVal, SDValue TrueVal,
1442                    ARMCC::CondCodes CCVal, SDValue CCR, SDValue InFlag) {
1443  SDValue CPTmp0;
1444  SDValue CPTmp1;
1445  if (SelectT2ShifterOperandReg(TrueVal, CPTmp0, CPTmp1)) {
1446    unsigned SOVal = cast<ConstantSDNode>(CPTmp1)->getZExtValue();
1447    unsigned SOShOp = ARM_AM::getSORegShOp(SOVal);
1448    unsigned Opc = 0;
1449    switch (SOShOp) {
1450    case ARM_AM::lsl: Opc = ARM::t2MOVCClsl; break;
1451    case ARM_AM::lsr: Opc = ARM::t2MOVCClsr; break;
1452    case ARM_AM::asr: Opc = ARM::t2MOVCCasr; break;
1453    case ARM_AM::ror: Opc = ARM::t2MOVCCror; break;
1454    default:
1455      llvm_unreachable("Unknown so_reg opcode!");
1456      break;
1457    }
1458    SDValue SOShImm =
1459      CurDAG->getTargetConstant(ARM_AM::getSORegOffset(SOVal), MVT::i32);
1460    SDValue CC = CurDAG->getTargetConstant(CCVal, MVT::i32);
1461    SDValue Ops[] = { FalseVal, CPTmp0, SOShImm, CC, CCR, InFlag };
1462    return CurDAG->SelectNodeTo(N, Opc, MVT::i32,Ops, 6);
1463  }
1464  return 0;
1465}
1466
1467SDNode *ARMDAGToDAGISel::
1468SelectARMCMOVShiftOp(SDNode *N, SDValue FalseVal, SDValue TrueVal,
1469                     ARMCC::CondCodes CCVal, SDValue CCR, SDValue InFlag) {
1470  SDValue CPTmp0;
1471  SDValue CPTmp1;
1472  SDValue CPTmp2;
1473  if (SelectShifterOperandReg(TrueVal, CPTmp0, CPTmp1, CPTmp2)) {
1474    SDValue CC = CurDAG->getTargetConstant(CCVal, MVT::i32);
1475    SDValue Ops[] = { FalseVal, CPTmp0, CPTmp1, CPTmp2, CC, CCR, InFlag };
1476    return CurDAG->SelectNodeTo(N, ARM::MOVCCs, MVT::i32, Ops, 7);
1477  }
1478  return 0;
1479}
1480
1481SDNode *ARMDAGToDAGISel::
1482SelectT2CMOVSoImmOp(SDNode *N, SDValue FalseVal, SDValue TrueVal,
1483                    ARMCC::CondCodes CCVal, SDValue CCR, SDValue InFlag) {
1484  ConstantSDNode *T = dyn_cast<ConstantSDNode>(TrueVal);
1485  if (!T)
1486    return 0;
1487
1488  if (Pred_t2_so_imm(TrueVal.getNode())) {
1489    SDValue True = CurDAG->getTargetConstant(T->getZExtValue(), MVT::i32);
1490    SDValue CC = CurDAG->getTargetConstant(CCVal, MVT::i32);
1491    SDValue Ops[] = { FalseVal, True, CC, CCR, InFlag };
1492    return CurDAG->SelectNodeTo(N,
1493                                ARM::t2MOVCCi, MVT::i32, Ops, 5);
1494  }
1495  return 0;
1496}
1497
1498SDNode *ARMDAGToDAGISel::
1499SelectARMCMOVSoImmOp(SDNode *N, SDValue FalseVal, SDValue TrueVal,
1500                     ARMCC::CondCodes CCVal, SDValue CCR, SDValue InFlag) {
1501  ConstantSDNode *T = dyn_cast<ConstantSDNode>(TrueVal);
1502  if (!T)
1503    return 0;
1504
1505  if (Pred_so_imm(TrueVal.getNode())) {
1506    SDValue True = CurDAG->getTargetConstant(T->getZExtValue(), MVT::i32);
1507    SDValue CC = CurDAG->getTargetConstant(CCVal, MVT::i32);
1508    SDValue Ops[] = { FalseVal, True, CC, CCR, InFlag };
1509    return CurDAG->SelectNodeTo(N,
1510                                ARM::MOVCCi, MVT::i32, Ops, 5);
1511  }
1512  return 0;
1513}
1514
1515SDNode *ARMDAGToDAGISel::SelectCMOVOp(SDNode *N) {
1516  EVT VT = N->getValueType(0);
1517  SDValue FalseVal = N->getOperand(0);
1518  SDValue TrueVal  = N->getOperand(1);
1519  SDValue CC = N->getOperand(2);
1520  SDValue CCR = N->getOperand(3);
1521  SDValue InFlag = N->getOperand(4);
1522  assert(CC.getOpcode() == ISD::Constant);
1523  assert(CCR.getOpcode() == ISD::Register);
1524  ARMCC::CondCodes CCVal =
1525    (ARMCC::CondCodes)cast<ConstantSDNode>(CC)->getZExtValue();
1526
1527  if (!Subtarget->isThumb1Only() && VT == MVT::i32) {
1528    // Pattern: (ARMcmov:i32 GPR:i32:$false, so_reg:i32:$true, (imm:i32):$cc)
1529    // Emits: (MOVCCs:i32 GPR:i32:$false, so_reg:i32:$true, (imm:i32):$cc)
1530    // Pattern complexity = 18  cost = 1  size = 0
1531    SDValue CPTmp0;
1532    SDValue CPTmp1;
1533    SDValue CPTmp2;
1534    if (Subtarget->isThumb()) {
1535      SDNode *Res = SelectT2CMOVShiftOp(N, FalseVal, TrueVal,
1536                                        CCVal, CCR, InFlag);
1537      if (!Res)
1538        Res = SelectT2CMOVShiftOp(N, TrueVal, FalseVal,
1539                               ARMCC::getOppositeCondition(CCVal), CCR, InFlag);
1540      if (Res)
1541        return Res;
1542    } else {
1543      SDNode *Res = SelectARMCMOVShiftOp(N, FalseVal, TrueVal,
1544                                         CCVal, CCR, InFlag);
1545      if (!Res)
1546        Res = SelectARMCMOVShiftOp(N, TrueVal, FalseVal,
1547                               ARMCC::getOppositeCondition(CCVal), CCR, InFlag);
1548      if (Res)
1549        return Res;
1550    }
1551
1552    // Pattern: (ARMcmov:i32 GPR:i32:$false,
1553    //             (imm:i32)<<P:Pred_so_imm>>:$true,
1554    //             (imm:i32):$cc)
1555    // Emits: (MOVCCi:i32 GPR:i32:$false,
1556    //           (so_imm:i32 (imm:i32):$true), (imm:i32):$cc)
1557    // Pattern complexity = 10  cost = 1  size = 0
1558    if (Subtarget->isThumb()) {
1559      SDNode *Res = SelectT2CMOVSoImmOp(N, FalseVal, TrueVal,
1560                                        CCVal, CCR, InFlag);
1561      if (!Res)
1562        Res = SelectT2CMOVSoImmOp(N, TrueVal, FalseVal,
1563                               ARMCC::getOppositeCondition(CCVal), CCR, InFlag);
1564      if (Res)
1565        return Res;
1566    } else {
1567      SDNode *Res = SelectARMCMOVSoImmOp(N, FalseVal, TrueVal,
1568                                         CCVal, CCR, InFlag);
1569      if (!Res)
1570        Res = SelectARMCMOVSoImmOp(N, TrueVal, FalseVal,
1571                               ARMCC::getOppositeCondition(CCVal), CCR, InFlag);
1572      if (Res)
1573        return Res;
1574    }
1575  }
1576
1577  // Pattern: (ARMcmov:i32 GPR:i32:$false, GPR:i32:$true, (imm:i32):$cc)
1578  // Emits: (MOVCCr:i32 GPR:i32:$false, GPR:i32:$true, (imm:i32):$cc)
1579  // Pattern complexity = 6  cost = 1  size = 0
1580  //
1581  // Pattern: (ARMcmov:i32 GPR:i32:$false, GPR:i32:$true, (imm:i32):$cc)
1582  // Emits: (tMOVCCr:i32 GPR:i32:$false, GPR:i32:$true, (imm:i32):$cc)
1583  // Pattern complexity = 6  cost = 11  size = 0
1584  //
1585  // Also FCPYScc and FCPYDcc.
1586  SDValue Tmp2 = CurDAG->getTargetConstant(CCVal, MVT::i32);
1587  SDValue Ops[] = { FalseVal, TrueVal, Tmp2, CCR, InFlag };
1588  unsigned Opc = 0;
1589  switch (VT.getSimpleVT().SimpleTy) {
1590  default: assert(false && "Illegal conditional move type!");
1591    break;
1592  case MVT::i32:
1593    Opc = Subtarget->isThumb()
1594      ? (Subtarget->hasThumb2() ? ARM::t2MOVCCr : ARM::tMOVCCr_pseudo)
1595      : ARM::MOVCCr;
1596    break;
1597  case MVT::f32:
1598    Opc = ARM::VMOVScc;
1599    break;
1600  case MVT::f64:
1601    Opc = ARM::VMOVDcc;
1602    break;
1603  }
1604  return CurDAG->SelectNodeTo(N, Opc, VT, Ops, 5);
1605}
1606
1607SDNode *ARMDAGToDAGISel::SelectConcatVector(SDNode *N) {
1608  // The only time a CONCAT_VECTORS operation can have legal types is when
1609  // two 64-bit vectors are concatenated to a 128-bit vector.
1610  EVT VT = N->getValueType(0);
1611  if (!VT.is128BitVector() || N->getNumOperands() != 2)
1612    llvm_unreachable("unexpected CONCAT_VECTORS");
1613  DebugLoc dl = N->getDebugLoc();
1614  SDValue V0 = N->getOperand(0);
1615  SDValue V1 = N->getOperand(1);
1616  SDValue SubReg0 = CurDAG->getTargetConstant(ARM::dsub_0, MVT::i32);
1617  SDValue SubReg1 = CurDAG->getTargetConstant(ARM::dsub_1, MVT::i32);
1618  const SDValue Ops[] = { V0, SubReg0, V1, SubReg1 };
1619  return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops, 4);
1620}
1621
1622SDNode *ARMDAGToDAGISel::Select(SDNode *N) {
1623  DebugLoc dl = N->getDebugLoc();
1624
1625  if (N->isMachineOpcode())
1626    return NULL;   // Already selected.
1627
1628  switch (N->getOpcode()) {
1629  default: break;
1630  case ISD::Constant: {
1631    unsigned Val = cast<ConstantSDNode>(N)->getZExtValue();
1632    bool UseCP = true;
1633    if (Subtarget->hasThumb2())
1634      // Thumb2-aware targets have the MOVT instruction, so all immediates can
1635      // be done with MOV + MOVT, at worst.
1636      UseCP = 0;
1637    else {
1638      if (Subtarget->isThumb()) {
1639        UseCP = (Val > 255 &&                          // MOV
1640                 ~Val > 255 &&                         // MOV + MVN
1641                 !ARM_AM::isThumbImmShiftedVal(Val));  // MOV + LSL
1642      } else
1643        UseCP = (ARM_AM::getSOImmVal(Val) == -1 &&     // MOV
1644                 ARM_AM::getSOImmVal(~Val) == -1 &&    // MVN
1645                 !ARM_AM::isSOImmTwoPartVal(Val));     // two instrs.
1646    }
1647
1648    if (UseCP) {
1649      SDValue CPIdx =
1650        CurDAG->getTargetConstantPool(ConstantInt::get(
1651                                  Type::getInt32Ty(*CurDAG->getContext()), Val),
1652                                      TLI.getPointerTy());
1653
1654      SDNode *ResNode;
1655      if (Subtarget->isThumb1Only()) {
1656        SDValue Pred = getAL(CurDAG);
1657        SDValue PredReg = CurDAG->getRegister(0, MVT::i32);
1658        SDValue Ops[] = { CPIdx, Pred, PredReg, CurDAG->getEntryNode() };
1659        ResNode = CurDAG->getMachineNode(ARM::tLDRcp, dl, MVT::i32, MVT::Other,
1660                                         Ops, 4);
1661      } else {
1662        SDValue Ops[] = {
1663          CPIdx,
1664          CurDAG->getRegister(0, MVT::i32),
1665          CurDAG->getTargetConstant(0, MVT::i32),
1666          getAL(CurDAG),
1667          CurDAG->getRegister(0, MVT::i32),
1668          CurDAG->getEntryNode()
1669        };
1670        ResNode=CurDAG->getMachineNode(ARM::LDRcp, dl, MVT::i32, MVT::Other,
1671                                       Ops, 6);
1672      }
1673      ReplaceUses(SDValue(N, 0), SDValue(ResNode, 0));
1674      return NULL;
1675    }
1676
1677    // Other cases are autogenerated.
1678    break;
1679  }
1680  case ISD::FrameIndex: {
1681    // Selects to ADDri FI, 0 which in turn will become ADDri SP, imm.
1682    int FI = cast<FrameIndexSDNode>(N)->getIndex();
1683    SDValue TFI = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
1684    if (Subtarget->isThumb1Only()) {
1685      return CurDAG->SelectNodeTo(N, ARM::tADDrSPi, MVT::i32, TFI,
1686                                  CurDAG->getTargetConstant(0, MVT::i32));
1687    } else {
1688      unsigned Opc = ((Subtarget->isThumb() && Subtarget->hasThumb2()) ?
1689                      ARM::t2ADDri : ARM::ADDri);
1690      SDValue Ops[] = { TFI, CurDAG->getTargetConstant(0, MVT::i32),
1691                        getAL(CurDAG), CurDAG->getRegister(0, MVT::i32),
1692                        CurDAG->getRegister(0, MVT::i32) };
1693      return CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops, 5);
1694    }
1695  }
1696  case ISD::SRL:
1697    if (SDNode *I = SelectV6T2BitfieldExtractOp(N, false))
1698      return I;
1699    break;
1700  case ISD::SRA:
1701    if (SDNode *I = SelectV6T2BitfieldExtractOp(N, true))
1702      return I;
1703    break;
1704  case ISD::MUL:
1705    if (Subtarget->isThumb1Only())
1706      break;
1707    if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(N->getOperand(1))) {
1708      unsigned RHSV = C->getZExtValue();
1709      if (!RHSV) break;
1710      if (isPowerOf2_32(RHSV-1)) {  // 2^n+1?
1711        unsigned ShImm = Log2_32(RHSV-1);
1712        if (ShImm >= 32)
1713          break;
1714        SDValue V = N->getOperand(0);
1715        ShImm = ARM_AM::getSORegOpc(ARM_AM::lsl, ShImm);
1716        SDValue ShImmOp = CurDAG->getTargetConstant(ShImm, MVT::i32);
1717        SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
1718        if (Subtarget->isThumb()) {
1719          SDValue Ops[] = { V, V, ShImmOp, getAL(CurDAG), Reg0, Reg0 };
1720          return CurDAG->SelectNodeTo(N, ARM::t2ADDrs, MVT::i32, Ops, 6);
1721        } else {
1722          SDValue Ops[] = { V, V, Reg0, ShImmOp, getAL(CurDAG), Reg0, Reg0 };
1723          return CurDAG->SelectNodeTo(N, ARM::ADDrs, MVT::i32, Ops, 7);
1724        }
1725      }
1726      if (isPowerOf2_32(RHSV+1)) {  // 2^n-1?
1727        unsigned ShImm = Log2_32(RHSV+1);
1728        if (ShImm >= 32)
1729          break;
1730        SDValue V = N->getOperand(0);
1731        ShImm = ARM_AM::getSORegOpc(ARM_AM::lsl, ShImm);
1732        SDValue ShImmOp = CurDAG->getTargetConstant(ShImm, MVT::i32);
1733        SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
1734        if (Subtarget->isThumb()) {
1735          SDValue Ops[] = { V, V, ShImmOp, getAL(CurDAG), Reg0, Reg0 };
1736          return CurDAG->SelectNodeTo(N, ARM::t2RSBrs, MVT::i32, Ops, 6);
1737        } else {
1738          SDValue Ops[] = { V, V, Reg0, ShImmOp, getAL(CurDAG), Reg0, Reg0 };
1739          return CurDAG->SelectNodeTo(N, ARM::RSBrs, MVT::i32, Ops, 7);
1740        }
1741      }
1742    }
1743    break;
1744  case ISD::AND: {
1745    // Check for unsigned bitfield extract
1746    if (SDNode *I = SelectV6T2BitfieldExtractOp(N, false))
1747      return I;
1748
1749    // (and (or x, c2), c1) and top 16-bits of c1 and c2 match, lower 16-bits
1750    // of c1 are 0xffff, and lower 16-bit of c2 are 0. That is, the top 16-bits
1751    // are entirely contributed by c2 and lower 16-bits are entirely contributed
1752    // by x. That's equal to (or (and x, 0xffff), (and c1, 0xffff0000)).
1753    // Select it to: "movt x, ((c1 & 0xffff) >> 16)
1754    EVT VT = N->getValueType(0);
1755    if (VT != MVT::i32)
1756      break;
1757    unsigned Opc = (Subtarget->isThumb() && Subtarget->hasThumb2())
1758      ? ARM::t2MOVTi16
1759      : (Subtarget->hasV6T2Ops() ? ARM::MOVTi16 : 0);
1760    if (!Opc)
1761      break;
1762    SDValue N0 = N->getOperand(0), N1 = N->getOperand(1);
1763    ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
1764    if (!N1C)
1765      break;
1766    if (N0.getOpcode() == ISD::OR && N0.getNode()->hasOneUse()) {
1767      SDValue N2 = N0.getOperand(1);
1768      ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(N2);
1769      if (!N2C)
1770        break;
1771      unsigned N1CVal = N1C->getZExtValue();
1772      unsigned N2CVal = N2C->getZExtValue();
1773      if ((N1CVal & 0xffff0000U) == (N2CVal & 0xffff0000U) &&
1774          (N1CVal & 0xffffU) == 0xffffU &&
1775          (N2CVal & 0xffffU) == 0x0U) {
1776        SDValue Imm16 = CurDAG->getTargetConstant((N2CVal & 0xFFFF0000U) >> 16,
1777                                                  MVT::i32);
1778        SDValue Ops[] = { N0.getOperand(0), Imm16,
1779                          getAL(CurDAG), CurDAG->getRegister(0, MVT::i32) };
1780        return CurDAG->getMachineNode(Opc, dl, VT, Ops, 4);
1781      }
1782    }
1783    break;
1784  }
1785  case ARMISD::VMOVRRD:
1786    return CurDAG->getMachineNode(ARM::VMOVRRD, dl, MVT::i32, MVT::i32,
1787                                  N->getOperand(0), getAL(CurDAG),
1788                                  CurDAG->getRegister(0, MVT::i32));
1789  case ISD::UMUL_LOHI: {
1790    if (Subtarget->isThumb1Only())
1791      break;
1792    if (Subtarget->isThumb()) {
1793      SDValue Ops[] = { N->getOperand(0), N->getOperand(1),
1794                        getAL(CurDAG), CurDAG->getRegister(0, MVT::i32),
1795                        CurDAG->getRegister(0, MVT::i32) };
1796      return CurDAG->getMachineNode(ARM::t2UMULL, dl, MVT::i32, MVT::i32,Ops,4);
1797    } else {
1798      SDValue Ops[] = { N->getOperand(0), N->getOperand(1),
1799                        getAL(CurDAG), CurDAG->getRegister(0, MVT::i32),
1800                        CurDAG->getRegister(0, MVT::i32) };
1801      return CurDAG->getMachineNode(ARM::UMULL, dl, MVT::i32, MVT::i32, Ops, 5);
1802    }
1803  }
1804  case ISD::SMUL_LOHI: {
1805    if (Subtarget->isThumb1Only())
1806      break;
1807    if (Subtarget->isThumb()) {
1808      SDValue Ops[] = { N->getOperand(0), N->getOperand(1),
1809                        getAL(CurDAG), CurDAG->getRegister(0, MVT::i32) };
1810      return CurDAG->getMachineNode(ARM::t2SMULL, dl, MVT::i32, MVT::i32,Ops,4);
1811    } else {
1812      SDValue Ops[] = { N->getOperand(0), N->getOperand(1),
1813                        getAL(CurDAG), CurDAG->getRegister(0, MVT::i32),
1814                        CurDAG->getRegister(0, MVT::i32) };
1815      return CurDAG->getMachineNode(ARM::SMULL, dl, MVT::i32, MVT::i32, Ops, 5);
1816    }
1817  }
1818  case ISD::LOAD: {
1819    SDNode *ResNode = 0;
1820    if (Subtarget->isThumb() && Subtarget->hasThumb2())
1821      ResNode = SelectT2IndexedLoad(N);
1822    else
1823      ResNode = SelectARMIndexedLoad(N);
1824    if (ResNode)
1825      return ResNode;
1826    // Other cases are autogenerated.
1827    break;
1828  }
1829  case ARMISD::BRCOND: {
1830    // Pattern: (ARMbrcond:void (bb:Other):$dst, (imm:i32):$cc)
1831    // Emits: (Bcc:void (bb:Other):$dst, (imm:i32):$cc)
1832    // Pattern complexity = 6  cost = 1  size = 0
1833
1834    // Pattern: (ARMbrcond:void (bb:Other):$dst, (imm:i32):$cc)
1835    // Emits: (tBcc:void (bb:Other):$dst, (imm:i32):$cc)
1836    // Pattern complexity = 6  cost = 1  size = 0
1837
1838    // Pattern: (ARMbrcond:void (bb:Other):$dst, (imm:i32):$cc)
1839    // Emits: (t2Bcc:void (bb:Other):$dst, (imm:i32):$cc)
1840    // Pattern complexity = 6  cost = 1  size = 0
1841
1842    unsigned Opc = Subtarget->isThumb() ?
1843      ((Subtarget->hasThumb2()) ? ARM::t2Bcc : ARM::tBcc) : ARM::Bcc;
1844    SDValue Chain = N->getOperand(0);
1845    SDValue N1 = N->getOperand(1);
1846    SDValue N2 = N->getOperand(2);
1847    SDValue N3 = N->getOperand(3);
1848    SDValue InFlag = N->getOperand(4);
1849    assert(N1.getOpcode() == ISD::BasicBlock);
1850    assert(N2.getOpcode() == ISD::Constant);
1851    assert(N3.getOpcode() == ISD::Register);
1852
1853    SDValue Tmp2 = CurDAG->getTargetConstant(((unsigned)
1854                               cast<ConstantSDNode>(N2)->getZExtValue()),
1855                               MVT::i32);
1856    SDValue Ops[] = { N1, Tmp2, N3, Chain, InFlag };
1857    SDNode *ResNode = CurDAG->getMachineNode(Opc, dl, MVT::Other,
1858                                             MVT::Flag, Ops, 5);
1859    Chain = SDValue(ResNode, 0);
1860    if (N->getNumValues() == 2) {
1861      InFlag = SDValue(ResNode, 1);
1862      ReplaceUses(SDValue(N, 1), InFlag);
1863    }
1864    ReplaceUses(SDValue(N, 0),
1865                SDValue(Chain.getNode(), Chain.getResNo()));
1866    return NULL;
1867  }
1868  case ARMISD::CMOV:
1869    return SelectCMOVOp(N);
1870  case ARMISD::CNEG: {
1871    EVT VT = N->getValueType(0);
1872    SDValue N0 = N->getOperand(0);
1873    SDValue N1 = N->getOperand(1);
1874    SDValue N2 = N->getOperand(2);
1875    SDValue N3 = N->getOperand(3);
1876    SDValue InFlag = N->getOperand(4);
1877    assert(N2.getOpcode() == ISD::Constant);
1878    assert(N3.getOpcode() == ISD::Register);
1879
1880    SDValue Tmp2 = CurDAG->getTargetConstant(((unsigned)
1881                               cast<ConstantSDNode>(N2)->getZExtValue()),
1882                               MVT::i32);
1883    SDValue Ops[] = { N0, N1, Tmp2, N3, InFlag };
1884    unsigned Opc = 0;
1885    switch (VT.getSimpleVT().SimpleTy) {
1886    default: assert(false && "Illegal conditional move type!");
1887      break;
1888    case MVT::f32:
1889      Opc = ARM::VNEGScc;
1890      break;
1891    case MVT::f64:
1892      Opc = ARM::VNEGDcc;
1893      break;
1894    }
1895    return CurDAG->SelectNodeTo(N, Opc, VT, Ops, 5);
1896  }
1897
1898  case ARMISD::VZIP: {
1899    unsigned Opc = 0;
1900    EVT VT = N->getValueType(0);
1901    switch (VT.getSimpleVT().SimpleTy) {
1902    default: return NULL;
1903    case MVT::v8i8:  Opc = ARM::VZIPd8; break;
1904    case MVT::v4i16: Opc = ARM::VZIPd16; break;
1905    case MVT::v2f32:
1906    case MVT::v2i32: Opc = ARM::VZIPd32; break;
1907    case MVT::v16i8: Opc = ARM::VZIPq8; break;
1908    case MVT::v8i16: Opc = ARM::VZIPq16; break;
1909    case MVT::v4f32:
1910    case MVT::v4i32: Opc = ARM::VZIPq32; break;
1911    }
1912    SDValue Pred = getAL(CurDAG);
1913    SDValue PredReg = CurDAG->getRegister(0, MVT::i32);
1914    SDValue Ops[] = { N->getOperand(0), N->getOperand(1), Pred, PredReg };
1915    return CurDAG->getMachineNode(Opc, dl, VT, VT, Ops, 4);
1916  }
1917  case ARMISD::VUZP: {
1918    unsigned Opc = 0;
1919    EVT VT = N->getValueType(0);
1920    switch (VT.getSimpleVT().SimpleTy) {
1921    default: return NULL;
1922    case MVT::v8i8:  Opc = ARM::VUZPd8; break;
1923    case MVT::v4i16: Opc = ARM::VUZPd16; break;
1924    case MVT::v2f32:
1925    case MVT::v2i32: Opc = ARM::VUZPd32; break;
1926    case MVT::v16i8: Opc = ARM::VUZPq8; break;
1927    case MVT::v8i16: Opc = ARM::VUZPq16; break;
1928    case MVT::v4f32:
1929    case MVT::v4i32: Opc = ARM::VUZPq32; break;
1930    }
1931    SDValue Pred = getAL(CurDAG);
1932    SDValue PredReg = CurDAG->getRegister(0, MVT::i32);
1933    SDValue Ops[] = { N->getOperand(0), N->getOperand(1), Pred, PredReg };
1934    return CurDAG->getMachineNode(Opc, dl, VT, VT, Ops, 4);
1935  }
1936  case ARMISD::VTRN: {
1937    unsigned Opc = 0;
1938    EVT VT = N->getValueType(0);
1939    switch (VT.getSimpleVT().SimpleTy) {
1940    default: return NULL;
1941    case MVT::v8i8:  Opc = ARM::VTRNd8; break;
1942    case MVT::v4i16: Opc = ARM::VTRNd16; break;
1943    case MVT::v2f32:
1944    case MVT::v2i32: Opc = ARM::VTRNd32; break;
1945    case MVT::v16i8: Opc = ARM::VTRNq8; break;
1946    case MVT::v8i16: Opc = ARM::VTRNq16; break;
1947    case MVT::v4f32:
1948    case MVT::v4i32: Opc = ARM::VTRNq32; break;
1949    }
1950    SDValue Pred = getAL(CurDAG);
1951    SDValue PredReg = CurDAG->getRegister(0, MVT::i32);
1952    SDValue Ops[] = { N->getOperand(0), N->getOperand(1), Pred, PredReg };
1953    return CurDAG->getMachineNode(Opc, dl, VT, VT, Ops, 4);
1954  }
1955  case ARMISD::BUILD_VECTOR: {
1956    EVT VecVT = N->getValueType(0);
1957    EVT EltVT = VecVT.getVectorElementType();
1958    unsigned NumElts = VecVT.getVectorNumElements();
1959    if (EltVT.getSimpleVT() == MVT::f64) {
1960      assert(NumElts == 2 && "unexpected type for BUILD_VECTOR");
1961      return PairDRegs(VecVT, N->getOperand(0), N->getOperand(1));
1962    }
1963    assert(EltVT.getSimpleVT() == MVT::f32 &&
1964           "unexpected type for BUILD_VECTOR");
1965    if (NumElts == 2)
1966      return PairSRegs(VecVT, N->getOperand(0), N->getOperand(1));
1967    assert(NumElts == 4 && "unexpected type for BUILD_VECTOR");
1968    return QuadSRegs(VecVT, N->getOperand(0), N->getOperand(1),
1969                     N->getOperand(2), N->getOperand(3));
1970  }
1971
1972  case ISD::INTRINSIC_VOID:
1973  case ISD::INTRINSIC_W_CHAIN: {
1974    unsigned IntNo = cast<ConstantSDNode>(N->getOperand(1))->getZExtValue();
1975    switch (IntNo) {
1976    default:
1977      break;
1978
1979    case Intrinsic::arm_neon_vld1: {
1980      unsigned DOpcodes[] = { ARM::VLD1d8, ARM::VLD1d16,
1981                              ARM::VLD1d32, ARM::VLD1d64 };
1982      unsigned QOpcodes[] = { ARM::VLD1q8Pseudo, ARM::VLD1q16Pseudo,
1983                              ARM::VLD1q32Pseudo, ARM::VLD1q64Pseudo };
1984      return SelectVLD(N, 1, DOpcodes, QOpcodes, 0);
1985    }
1986
1987    case Intrinsic::arm_neon_vld2: {
1988      unsigned DOpcodes[] = { ARM::VLD2d8Pseudo, ARM::VLD2d16Pseudo,
1989                              ARM::VLD2d32Pseudo, ARM::VLD1q64Pseudo };
1990      unsigned QOpcodes[] = { ARM::VLD2q8Pseudo, ARM::VLD2q16Pseudo,
1991                              ARM::VLD2q32Pseudo };
1992      return SelectVLD(N, 2, DOpcodes, QOpcodes, 0);
1993    }
1994
1995    case Intrinsic::arm_neon_vld3: {
1996      unsigned DOpcodes[] = { ARM::VLD3d8Pseudo, ARM::VLD3d16Pseudo,
1997                              ARM::VLD3d32Pseudo, ARM::VLD1d64TPseudo };
1998      unsigned QOpcodes0[] = { ARM::VLD3q8Pseudo_UPD,
1999                               ARM::VLD3q16Pseudo_UPD,
2000                               ARM::VLD3q32Pseudo_UPD };
2001      unsigned QOpcodes1[] = { ARM::VLD3q8oddPseudo_UPD,
2002                               ARM::VLD3q16oddPseudo_UPD,
2003                               ARM::VLD3q32oddPseudo_UPD };
2004      return SelectVLD(N, 3, DOpcodes, QOpcodes0, QOpcodes1);
2005    }
2006
2007    case Intrinsic::arm_neon_vld4: {
2008      unsigned DOpcodes[] = { ARM::VLD4d8Pseudo, ARM::VLD4d16Pseudo,
2009                              ARM::VLD4d32Pseudo, ARM::VLD1d64QPseudo };
2010      unsigned QOpcodes0[] = { ARM::VLD4q8Pseudo_UPD,
2011                               ARM::VLD4q16Pseudo_UPD,
2012                               ARM::VLD4q32Pseudo_UPD };
2013      unsigned QOpcodes1[] = { ARM::VLD4q8oddPseudo_UPD,
2014                               ARM::VLD4q16oddPseudo_UPD,
2015                               ARM::VLD4q32oddPseudo_UPD };
2016      return SelectVLD(N, 4, DOpcodes, QOpcodes0, QOpcodes1);
2017    }
2018
2019    case Intrinsic::arm_neon_vld2lane: {
2020      unsigned DOpcodes[] = { ARM::VLD2LNd8Pseudo, ARM::VLD2LNd16Pseudo,
2021                              ARM::VLD2LNd32Pseudo };
2022      unsigned QOpcodes[] = { ARM::VLD2LNq16Pseudo, ARM::VLD2LNq32Pseudo };
2023      return SelectVLDSTLane(N, true, 2, DOpcodes, QOpcodes);
2024    }
2025
2026    case Intrinsic::arm_neon_vld3lane: {
2027      unsigned DOpcodes[] = { ARM::VLD3LNd8Pseudo, ARM::VLD3LNd16Pseudo,
2028                              ARM::VLD3LNd32Pseudo };
2029      unsigned QOpcodes[] = { ARM::VLD3LNq16Pseudo, ARM::VLD3LNq32Pseudo };
2030      return SelectVLDSTLane(N, true, 3, DOpcodes, QOpcodes);
2031    }
2032
2033    case Intrinsic::arm_neon_vld4lane: {
2034      unsigned DOpcodes[] = { ARM::VLD4LNd8Pseudo, ARM::VLD4LNd16Pseudo,
2035                              ARM::VLD4LNd32Pseudo };
2036      unsigned QOpcodes[] = { ARM::VLD4LNq16Pseudo, ARM::VLD4LNq32Pseudo };
2037      return SelectVLDSTLane(N, true, 4, DOpcodes, QOpcodes);
2038    }
2039
2040    case Intrinsic::arm_neon_vst1: {
2041      unsigned DOpcodes[] = { ARM::VST1d8, ARM::VST1d16,
2042                              ARM::VST1d32, ARM::VST1d64 };
2043      unsigned QOpcodes[] = { ARM::VST1q8Pseudo, ARM::VST1q16Pseudo,
2044                              ARM::VST1q32Pseudo, ARM::VST1q64Pseudo };
2045      return SelectVST(N, 1, DOpcodes, QOpcodes, 0);
2046    }
2047
2048    case Intrinsic::arm_neon_vst2: {
2049      unsigned DOpcodes[] = { ARM::VST2d8Pseudo, ARM::VST2d16Pseudo,
2050                              ARM::VST2d32Pseudo, ARM::VST1q64Pseudo };
2051      unsigned QOpcodes[] = { ARM::VST2q8Pseudo, ARM::VST2q16Pseudo,
2052                              ARM::VST2q32Pseudo };
2053      return SelectVST(N, 2, DOpcodes, QOpcodes, 0);
2054    }
2055
2056    case Intrinsic::arm_neon_vst3: {
2057      unsigned DOpcodes[] = { ARM::VST3d8Pseudo, ARM::VST3d16Pseudo,
2058                              ARM::VST3d32Pseudo, ARM::VST1d64TPseudo };
2059      unsigned QOpcodes0[] = { ARM::VST3q8Pseudo_UPD,
2060                               ARM::VST3q16Pseudo_UPD,
2061                               ARM::VST3q32Pseudo_UPD };
2062      unsigned QOpcodes1[] = { ARM::VST3q8oddPseudo_UPD,
2063                               ARM::VST3q16oddPseudo_UPD,
2064                               ARM::VST3q32oddPseudo_UPD };
2065      return SelectVST(N, 3, DOpcodes, QOpcodes0, QOpcodes1);
2066    }
2067
2068    case Intrinsic::arm_neon_vst4: {
2069      unsigned DOpcodes[] = { ARM::VST4d8Pseudo, ARM::VST4d16Pseudo,
2070                              ARM::VST4d32Pseudo, ARM::VST1d64QPseudo };
2071      unsigned QOpcodes0[] = { ARM::VST4q8Pseudo_UPD,
2072                               ARM::VST4q16Pseudo_UPD,
2073                               ARM::VST4q32Pseudo_UPD };
2074      unsigned QOpcodes1[] = { ARM::VST4q8oddPseudo_UPD,
2075                               ARM::VST4q16oddPseudo_UPD,
2076                               ARM::VST4q32oddPseudo_UPD };
2077      return SelectVST(N, 4, DOpcodes, QOpcodes0, QOpcodes1);
2078    }
2079
2080    case Intrinsic::arm_neon_vst2lane: {
2081      unsigned DOpcodes[] = { ARM::VST2LNd8Pseudo, ARM::VST2LNd16Pseudo,
2082                              ARM::VST2LNd32Pseudo };
2083      unsigned QOpcodes[] = { ARM::VST2LNq16Pseudo, ARM::VST2LNq32Pseudo };
2084      return SelectVLDSTLane(N, false, 2, DOpcodes, QOpcodes);
2085    }
2086
2087    case Intrinsic::arm_neon_vst3lane: {
2088      unsigned DOpcodes[] = { ARM::VST3LNd8Pseudo, ARM::VST3LNd16Pseudo,
2089                              ARM::VST3LNd32Pseudo };
2090      unsigned QOpcodes[] = { ARM::VST3LNq16Pseudo, ARM::VST3LNq32Pseudo };
2091      return SelectVLDSTLane(N, false, 3, DOpcodes, QOpcodes);
2092    }
2093
2094    case Intrinsic::arm_neon_vst4lane: {
2095      unsigned DOpcodes[] = { ARM::VST4LNd8Pseudo, ARM::VST4LNd16Pseudo,
2096                              ARM::VST4LNd32Pseudo };
2097      unsigned QOpcodes[] = { ARM::VST4LNq16Pseudo, ARM::VST4LNq32Pseudo };
2098      return SelectVLDSTLane(N, false, 4, DOpcodes, QOpcodes);
2099    }
2100    }
2101    break;
2102  }
2103
2104  case ISD::INTRINSIC_WO_CHAIN: {
2105    unsigned IntNo = cast<ConstantSDNode>(N->getOperand(0))->getZExtValue();
2106    switch (IntNo) {
2107    default:
2108      break;
2109
2110    case Intrinsic::arm_neon_vtbl2:
2111      return SelectVTBL(N, false, 2, ARM::VTBL2Pseudo);
2112    case Intrinsic::arm_neon_vtbl3:
2113      return SelectVTBL(N, false, 3, ARM::VTBL3Pseudo);
2114    case Intrinsic::arm_neon_vtbl4:
2115      return SelectVTBL(N, false, 4, ARM::VTBL4Pseudo);
2116
2117    case Intrinsic::arm_neon_vtbx2:
2118      return SelectVTBL(N, true, 2, ARM::VTBX2Pseudo);
2119    case Intrinsic::arm_neon_vtbx3:
2120      return SelectVTBL(N, true, 3, ARM::VTBX3Pseudo);
2121    case Intrinsic::arm_neon_vtbx4:
2122      return SelectVTBL(N, true, 4, ARM::VTBX4Pseudo);
2123    }
2124    break;
2125  }
2126
2127  case ISD::CONCAT_VECTORS:
2128    return SelectConcatVector(N);
2129  }
2130
2131  return SelectCode(N);
2132}
2133
2134bool ARMDAGToDAGISel::
2135SelectInlineAsmMemoryOperand(const SDValue &Op, char ConstraintCode,
2136                             std::vector<SDValue> &OutOps) {
2137  assert(ConstraintCode == 'm' && "unexpected asm memory constraint");
2138  // Require the address to be in a register.  That is safe for all ARM
2139  // variants and it is hard to do anything much smarter without knowing
2140  // how the operand is used.
2141  OutOps.push_back(Op);
2142  return false;
2143}
2144
2145/// createARMISelDag - This pass converts a legalized DAG into a
2146/// ARM-specific DAG, ready for instruction scheduling.
2147///
2148FunctionPass *llvm::createARMISelDag(ARMBaseTargetMachine &TM,
2149                                     CodeGenOpt::Level OptLevel) {
2150  return new ARMDAGToDAGISel(TM, OptLevel);
2151}
2152