ARMISelDAGToDAG.cpp revision 603afbfe2ac8ccc21283b149a76a81eb44b956b0 
1//===-- ARMISelDAGToDAG.cpp - A dag to dag inst selector for ARM ----------===//
2//
3//                     The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10// This file defines an instruction selector for the ARM target.
11//
12//===----------------------------------------------------------------------===//
13
14#include "ARM.h"
15#include "ARMAddressingModes.h"
16#include "ARMTargetMachine.h"
17#include "llvm/CallingConv.h"
18#include "llvm/Constants.h"
19#include "llvm/DerivedTypes.h"
20#include "llvm/Function.h"
21#include "llvm/Intrinsics.h"
22#include "llvm/LLVMContext.h"
23#include "llvm/CodeGen/MachineFrameInfo.h"
24#include "llvm/CodeGen/MachineFunction.h"
25#include "llvm/CodeGen/MachineInstrBuilder.h"
26#include "llvm/CodeGen/SelectionDAG.h"
27#include "llvm/CodeGen/SelectionDAGISel.h"
28#include "llvm/Target/TargetLowering.h"
29#include "llvm/Target/TargetOptions.h"
30#include "llvm/Support/CommandLine.h"
31#include "llvm/Support/Compiler.h"
32#include "llvm/Support/Debug.h"
33#include "llvm/Support/ErrorHandling.h"
34#include "llvm/Support/raw_ostream.h"
35
36using namespace llvm;
37
38static cl::opt<bool>
39UseRegSeq("neon-reg-sequence", cl::Hidden,
40          cl::desc("Use reg_sequence to model ld / st of multiple neon regs"));
41
42//===--------------------------------------------------------------------===//
43/// ARMDAGToDAGISel - ARM specific code to select ARM machine
44/// instructions for SelectionDAG operations.
45///
46namespace {
47class ARMDAGToDAGISel : public SelectionDAGISel {
48  ARMBaseTargetMachine &TM;
49
50  /// Subtarget - Keep a pointer to the ARMSubtarget around so that we can
51  /// make the right decision when generating code for different targets.
52  const ARMSubtarget *Subtarget;
53
54public:
55  explicit ARMDAGToDAGISel(ARMBaseTargetMachine &tm,
56                           CodeGenOpt::Level OptLevel)
57    : SelectionDAGISel(tm, OptLevel), TM(tm),
58    Subtarget(&TM.getSubtarget<ARMSubtarget>()) {
59  }
60
61  virtual const char *getPassName() const {
62    return "ARM Instruction Selection";
63  }
64
65  /// getI32Imm - Return a target constant of type i32 with the specified
66  /// value.
67  inline SDValue getI32Imm(unsigned Imm) {
68    return CurDAG->getTargetConstant(Imm, MVT::i32);
69  }
70
71  SDNode *Select(SDNode *N);
72
73  bool SelectShifterOperandReg(SDNode *Op, SDValue N, SDValue &A,
74                               SDValue &B, SDValue &C);
75  bool SelectAddrMode2(SDNode *Op, SDValue N, SDValue &Base,
76                       SDValue &Offset, SDValue &Opc);
77  bool SelectAddrMode2Offset(SDNode *Op, SDValue N,
78                             SDValue &Offset, SDValue &Opc);
79  bool SelectAddrMode3(SDNode *Op, SDValue N, SDValue &Base,
80                       SDValue &Offset, SDValue &Opc);
81  bool SelectAddrMode3Offset(SDNode *Op, SDValue N,
82                             SDValue &Offset, SDValue &Opc);
83  bool SelectAddrMode4(SDNode *Op, SDValue N, SDValue &Addr,
84                       SDValue &Mode);
85  bool SelectAddrMode5(SDNode *Op, SDValue N, SDValue &Base,
86                       SDValue &Offset);
87  bool SelectAddrMode6(SDNode *Op, SDValue N, SDValue &Addr, SDValue &Align);
88
89  bool SelectAddrModePC(SDNode *Op, SDValue N, SDValue &Offset,
90                        SDValue &Label);
91
92  bool SelectThumbAddrModeRR(SDNode *Op, SDValue N, SDValue &Base,
93                             SDValue &Offset);
94  bool SelectThumbAddrModeRI5(SDNode *Op, SDValue N, unsigned Scale,
95                              SDValue &Base, SDValue &OffImm,
96                              SDValue &Offset);
97  bool SelectThumbAddrModeS1(SDNode *Op, SDValue N, SDValue &Base,
98                             SDValue &OffImm, SDValue &Offset);
99  bool SelectThumbAddrModeS2(SDNode *Op, SDValue N, SDValue &Base,
100                             SDValue &OffImm, SDValue &Offset);
101  bool SelectThumbAddrModeS4(SDNode *Op, SDValue N, SDValue &Base,
102                             SDValue &OffImm, SDValue &Offset);
103  bool SelectThumbAddrModeSP(SDNode *Op, SDValue N, SDValue &Base,
104                             SDValue &OffImm);
105
106  bool SelectT2ShifterOperandReg(SDNode *Op, SDValue N,
107                                 SDValue &BaseReg, SDValue &Opc);
108  bool SelectT2AddrModeImm12(SDNode *Op, SDValue N, SDValue &Base,
109                             SDValue &OffImm);
110  bool SelectT2AddrModeImm8(SDNode *Op, SDValue N, SDValue &Base,
111                            SDValue &OffImm);
112  bool SelectT2AddrModeImm8Offset(SDNode *Op, SDValue N,
113                                 SDValue &OffImm);
114  bool SelectT2AddrModeImm8s4(SDNode *Op, SDValue N, SDValue &Base,
115                              SDValue &OffImm);
116  bool SelectT2AddrModeSoReg(SDNode *Op, SDValue N, SDValue &Base,
117                             SDValue &OffReg, SDValue &ShImm);
118
119  // Include the pieces autogenerated from the target description.
120#include "ARMGenDAGISel.inc"
121
122private:
123  /// SelectARMIndexedLoad - Indexed (pre/post inc/dec) load matching code for
124  /// ARM.
125  SDNode *SelectARMIndexedLoad(SDNode *N);
126  SDNode *SelectT2IndexedLoad(SDNode *N);
127
128  /// SelectVLD - Select NEON load intrinsics.  NumVecs should be
129  /// 1, 2, 3 or 4.  The opcode arrays specify the instructions used for
130  /// loads of D registers and even subregs and odd subregs of Q registers.
131  /// For NumVecs <= 2, QOpcodes1 is not used.
132  SDNode *SelectVLD(SDNode *N, unsigned NumVecs, unsigned *DOpcodes,
133                    unsigned *QOpcodes0, unsigned *QOpcodes1);
134
135  /// SelectVST - Select NEON store intrinsics.  NumVecs should
136  /// be 1, 2, 3 or 4.  The opcode arrays specify the instructions used for
137  /// stores of D registers and even subregs and odd subregs of Q registers.
138  /// For NumVecs <= 2, QOpcodes1 is not used.
139  SDNode *SelectVST(SDNode *N, unsigned NumVecs, unsigned *DOpcodes,
140                    unsigned *QOpcodes0, unsigned *QOpcodes1);
141
142  /// SelectVLDSTLane - Select NEON load/store lane intrinsics.  NumVecs should
143  /// be 2, 3 or 4.  The opcode arrays specify the instructions used for
144  /// load/store of D registers and even subregs and odd subregs of Q registers.
145  SDNode *SelectVLDSTLane(SDNode *N, bool IsLoad, unsigned NumVecs,
146                          unsigned *DOpcodes, unsigned *QOpcodes0,
147                          unsigned *QOpcodes1);
148
149  /// SelectV6T2BitfieldExtractOp - Select SBFX/UBFX instructions for ARM.
150  SDNode *SelectV6T2BitfieldExtractOp(SDNode *N, bool isSigned);
151
152  /// SelectCMOVOp - Select CMOV instructions for ARM.
153  SDNode *SelectCMOVOp(SDNode *N);
154  SDNode *SelectT2CMOVShiftOp(SDNode *N, SDValue FalseVal, SDValue TrueVal,
155                              ARMCC::CondCodes CCVal, SDValue CCR,
156                              SDValue InFlag);
157  SDNode *SelectARMCMOVShiftOp(SDNode *N, SDValue FalseVal, SDValue TrueVal,
158                               ARMCC::CondCodes CCVal, SDValue CCR,
159                               SDValue InFlag);
160  SDNode *SelectT2CMOVSoImmOp(SDNode *N, SDValue FalseVal, SDValue TrueVal,
161                              ARMCC::CondCodes CCVal, SDValue CCR,
162                              SDValue InFlag);
163  SDNode *SelectARMCMOVSoImmOp(SDNode *N, SDValue FalseVal, SDValue TrueVal,
164                               ARMCC::CondCodes CCVal, SDValue CCR,
165                               SDValue InFlag);
166
167  SDNode *SelectConcatVector(SDNode *N);
168
169  /// SelectInlineAsmMemoryOperand - Implement addressing mode selection for
170  /// inline asm expressions.
171  virtual bool SelectInlineAsmMemoryOperand(const SDValue &Op,
172                                            char ConstraintCode,
173                                            std::vector<SDValue> &OutOps);
174
175  /// PairDRegs - Form a quad register from a pair of D registers.
176  ///
177  SDNode *PairDRegs(EVT VT, SDValue V0, SDValue V1);
178
179  /// PairDRegs - Form a quad register pair from a pair of Q registers.
180  ///
181  SDNode *PairQRegs(EVT VT, SDValue V0, SDValue V1);
182
183  /// QuadDRegs - Form a quad register pair from a quad of D registers.
184  ///
185  SDNode *QuadDRegs(EVT VT, SDValue V0, SDValue V1, SDValue V2, SDValue V3);
186};
187}
188
189/// isInt32Immediate - This method tests to see if the node is a 32-bit constant
190/// operand. If so Imm will receive the 32-bit value.
191static bool isInt32Immediate(SDNode *N, unsigned &Imm) {
192  if (N->getOpcode() == ISD::Constant && N->getValueType(0) == MVT::i32) {
193    Imm = cast<ConstantSDNode>(N)->getZExtValue();
194    return true;
195  }
196  return false;
197}
198
199// isInt32Immediate - This method tests to see if a constant operand.
200// If so Imm will receive the 32 bit value.
201static bool isInt32Immediate(SDValue N, unsigned &Imm) {
202  return isInt32Immediate(N.getNode(), Imm);
203}
204
205// isOpcWithIntImmediate - This method tests to see if the node is a specific
206// opcode and that it has a immediate integer right operand.
207// If so Imm will receive the 32 bit value.
208static bool isOpcWithIntImmediate(SDNode *N, unsigned Opc, unsigned& Imm) {
209  return N->getOpcode() == Opc &&
210         isInt32Immediate(N->getOperand(1).getNode(), Imm);
211}
212
213
214bool ARMDAGToDAGISel::SelectShifterOperandReg(SDNode *Op,
215                                              SDValue N,
216                                              SDValue &BaseReg,
217                                              SDValue &ShReg,
218                                              SDValue &Opc) {
219  ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(N);
220
221  // Don't match base register only case. That is matched to a separate
222  // lower complexity pattern with explicit register operand.
223  if (ShOpcVal == ARM_AM::no_shift) return false;
224
225  BaseReg = N.getOperand(0);
226  unsigned ShImmVal = 0;
227  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
228    ShReg = CurDAG->getRegister(0, MVT::i32);
229    ShImmVal = RHS->getZExtValue() & 31;
230  } else {
231    ShReg = N.getOperand(1);
232  }
233  Opc = CurDAG->getTargetConstant(ARM_AM::getSORegOpc(ShOpcVal, ShImmVal),
234                                  MVT::i32);
235  return true;
236}
237
238bool ARMDAGToDAGISel::SelectAddrMode2(SDNode *Op, SDValue N,
239                                      SDValue &Base, SDValue &Offset,
240                                      SDValue &Opc) {
241  if (N.getOpcode() == ISD::MUL) {
242    if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
243      // X * [3,5,9] -> X + X * [2,4,8] etc.
244      int RHSC = (int)RHS->getZExtValue();
245      if (RHSC & 1) {
246        RHSC = RHSC & ~1;
247        ARM_AM::AddrOpc AddSub = ARM_AM::add;
248        if (RHSC < 0) {
249          AddSub = ARM_AM::sub;
250          RHSC = - RHSC;
251        }
252        if (isPowerOf2_32(RHSC)) {
253          unsigned ShAmt = Log2_32(RHSC);
254          Base = Offset = N.getOperand(0);
255          Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, ShAmt,
256                                                            ARM_AM::lsl),
257                                          MVT::i32);
258          return true;
259        }
260      }
261    }
262  }
263
264  if (N.getOpcode() != ISD::ADD && N.getOpcode() != ISD::SUB) {
265    Base = N;
266    if (N.getOpcode() == ISD::FrameIndex) {
267      int FI = cast<FrameIndexSDNode>(N)->getIndex();
268      Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
269    } else if (N.getOpcode() == ARMISD::Wrapper &&
270               !(Subtarget->useMovt() &&
271                 N.getOperand(0).getOpcode() == ISD::TargetGlobalAddress)) {
272      Base = N.getOperand(0);
273    }
274    Offset = CurDAG->getRegister(0, MVT::i32);
275    Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(ARM_AM::add, 0,
276                                                      ARM_AM::no_shift),
277                                    MVT::i32);
278    return true;
279  }
280
281  // Match simple R +/- imm12 operands.
282  if (N.getOpcode() == ISD::ADD)
283    if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
284      int RHSC = (int)RHS->getZExtValue();
285      if ((RHSC >= 0 && RHSC < 0x1000) ||
286          (RHSC < 0 && RHSC > -0x1000)) { // 12 bits.
287        Base = N.getOperand(0);
288        if (Base.getOpcode() == ISD::FrameIndex) {
289          int FI = cast<FrameIndexSDNode>(Base)->getIndex();
290          Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
291        }
292        Offset = CurDAG->getRegister(0, MVT::i32);
293
294        ARM_AM::AddrOpc AddSub = ARM_AM::add;
295        if (RHSC < 0) {
296          AddSub = ARM_AM::sub;
297          RHSC = - RHSC;
298        }
299        Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, RHSC,
300                                                          ARM_AM::no_shift),
301                                        MVT::i32);
302        return true;
303      }
304    }
305
306  // Otherwise this is R +/- [possibly shifted] R.
307  ARM_AM::AddrOpc AddSub = N.getOpcode() == ISD::ADD ? ARM_AM::add:ARM_AM::sub;
308  ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(N.getOperand(1));
309  unsigned ShAmt = 0;
310
311  Base   = N.getOperand(0);
312  Offset = N.getOperand(1);
313
314  if (ShOpcVal != ARM_AM::no_shift) {
315    // Check to see if the RHS of the shift is a constant, if not, we can't fold
316    // it.
317    if (ConstantSDNode *Sh =
318           dyn_cast<ConstantSDNode>(N.getOperand(1).getOperand(1))) {
319      ShAmt = Sh->getZExtValue();
320      Offset = N.getOperand(1).getOperand(0);
321    } else {
322      ShOpcVal = ARM_AM::no_shift;
323    }
324  }
325
326  // Try matching (R shl C) + (R).
327  if (N.getOpcode() == ISD::ADD && ShOpcVal == ARM_AM::no_shift) {
328    ShOpcVal = ARM_AM::getShiftOpcForNode(N.getOperand(0));
329    if (ShOpcVal != ARM_AM::no_shift) {
330      // Check to see if the RHS of the shift is a constant, if not, we can't
331      // fold it.
332      if (ConstantSDNode *Sh =
333          dyn_cast<ConstantSDNode>(N.getOperand(0).getOperand(1))) {
334        ShAmt = Sh->getZExtValue();
335        Offset = N.getOperand(0).getOperand(0);
336        Base = N.getOperand(1);
337      } else {
338        ShOpcVal = ARM_AM::no_shift;
339      }
340    }
341  }
342
343  Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, ShAmt, ShOpcVal),
344                                  MVT::i32);
345  return true;
346}
347
348bool ARMDAGToDAGISel::SelectAddrMode2Offset(SDNode *Op, SDValue N,
349                                            SDValue &Offset, SDValue &Opc) {
350  unsigned Opcode = Op->getOpcode();
351  ISD::MemIndexedMode AM = (Opcode == ISD::LOAD)
352    ? cast<LoadSDNode>(Op)->getAddressingMode()
353    : cast<StoreSDNode>(Op)->getAddressingMode();
354  ARM_AM::AddrOpc AddSub = (AM == ISD::PRE_INC || AM == ISD::POST_INC)
355    ? ARM_AM::add : ARM_AM::sub;
356  if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(N)) {
357    int Val = (int)C->getZExtValue();
358    if (Val >= 0 && Val < 0x1000) { // 12 bits.
359      Offset = CurDAG->getRegister(0, MVT::i32);
360      Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, Val,
361                                                        ARM_AM::no_shift),
362                                      MVT::i32);
363      return true;
364    }
365  }
366
367  Offset = N;
368  ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(N);
369  unsigned ShAmt = 0;
370  if (ShOpcVal != ARM_AM::no_shift) {
371    // Check to see if the RHS of the shift is a constant, if not, we can't fold
372    // it.
373    if (ConstantSDNode *Sh = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
374      ShAmt = Sh->getZExtValue();
375      Offset = N.getOperand(0);
376    } else {
377      ShOpcVal = ARM_AM::no_shift;
378    }
379  }
380
381  Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, ShAmt, ShOpcVal),
382                                  MVT::i32);
383  return true;
384}
385
386
387bool ARMDAGToDAGISel::SelectAddrMode3(SDNode *Op, SDValue N,
388                                      SDValue &Base, SDValue &Offset,
389                                      SDValue &Opc) {
390  if (N.getOpcode() == ISD::SUB) {
391    // X - C  is canonicalize to X + -C, no need to handle it here.
392    Base = N.getOperand(0);
393    Offset = N.getOperand(1);
394    Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(ARM_AM::sub, 0),MVT::i32);
395    return true;
396  }
397
398  if (N.getOpcode() != ISD::ADD) {
399    Base = N;
400    if (N.getOpcode() == ISD::FrameIndex) {
401      int FI = cast<FrameIndexSDNode>(N)->getIndex();
402      Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
403    }
404    Offset = CurDAG->getRegister(0, MVT::i32);
405    Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(ARM_AM::add, 0),MVT::i32);
406    return true;
407  }
408
409  // If the RHS is +/- imm8, fold into addr mode.
410  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
411    int RHSC = (int)RHS->getZExtValue();
412    if ((RHSC >= 0 && RHSC < 256) ||
413        (RHSC < 0 && RHSC > -256)) { // note -256 itself isn't allowed.
414      Base = N.getOperand(0);
415      if (Base.getOpcode() == ISD::FrameIndex) {
416        int FI = cast<FrameIndexSDNode>(Base)->getIndex();
417        Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
418      }
419      Offset = CurDAG->getRegister(0, MVT::i32);
420
421      ARM_AM::AddrOpc AddSub = ARM_AM::add;
422      if (RHSC < 0) {
423        AddSub = ARM_AM::sub;
424        RHSC = - RHSC;
425      }
426      Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(AddSub, RHSC),MVT::i32);
427      return true;
428    }
429  }
430
431  Base = N.getOperand(0);
432  Offset = N.getOperand(1);
433  Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(ARM_AM::add, 0), MVT::i32);
434  return true;
435}
436
437bool ARMDAGToDAGISel::SelectAddrMode3Offset(SDNode *Op, SDValue N,
438                                            SDValue &Offset, SDValue &Opc) {
439  unsigned Opcode = Op->getOpcode();
440  ISD::MemIndexedMode AM = (Opcode == ISD::LOAD)
441    ? cast<LoadSDNode>(Op)->getAddressingMode()
442    : cast<StoreSDNode>(Op)->getAddressingMode();
443  ARM_AM::AddrOpc AddSub = (AM == ISD::PRE_INC || AM == ISD::POST_INC)
444    ? ARM_AM::add : ARM_AM::sub;
445  if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(N)) {
446    int Val = (int)C->getZExtValue();
447    if (Val >= 0 && Val < 256) {
448      Offset = CurDAG->getRegister(0, MVT::i32);
449      Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(AddSub, Val), MVT::i32);
450      return true;
451    }
452  }
453
454  Offset = N;
455  Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(AddSub, 0), MVT::i32);
456  return true;
457}
458
459bool ARMDAGToDAGISel::SelectAddrMode4(SDNode *Op, SDValue N,
460                                      SDValue &Addr, SDValue &Mode) {
461  Addr = N;
462  Mode = CurDAG->getTargetConstant(0, MVT::i32);
463  return true;
464}
465
466bool ARMDAGToDAGISel::SelectAddrMode5(SDNode *Op, SDValue N,
467                                      SDValue &Base, SDValue &Offset) {
468  if (N.getOpcode() != ISD::ADD) {
469    Base = N;
470    if (N.getOpcode() == ISD::FrameIndex) {
471      int FI = cast<FrameIndexSDNode>(N)->getIndex();
472      Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
473    } else if (N.getOpcode() == ARMISD::Wrapper &&
474               !(Subtarget->useMovt() &&
475                 N.getOperand(0).getOpcode() == ISD::TargetGlobalAddress)) {
476      Base = N.getOperand(0);
477    }
478    Offset = CurDAG->getTargetConstant(ARM_AM::getAM5Opc(ARM_AM::add, 0),
479                                       MVT::i32);
480    return true;
481  }
482
483  // If the RHS is +/- imm8, fold into addr mode.
484  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
485    int RHSC = (int)RHS->getZExtValue();
486    if ((RHSC & 3) == 0) {  // The constant is implicitly multiplied by 4.
487      RHSC >>= 2;
488      if ((RHSC >= 0 && RHSC < 256) ||
489          (RHSC < 0 && RHSC > -256)) { // note -256 itself isn't allowed.
490        Base = N.getOperand(0);
491        if (Base.getOpcode() == ISD::FrameIndex) {
492          int FI = cast<FrameIndexSDNode>(Base)->getIndex();
493          Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
494        }
495
496        ARM_AM::AddrOpc AddSub = ARM_AM::add;
497        if (RHSC < 0) {
498          AddSub = ARM_AM::sub;
499          RHSC = - RHSC;
500        }
501        Offset = CurDAG->getTargetConstant(ARM_AM::getAM5Opc(AddSub, RHSC),
502                                           MVT::i32);
503        return true;
504      }
505    }
506  }
507
508  Base = N;
509  Offset = CurDAG->getTargetConstant(ARM_AM::getAM5Opc(ARM_AM::add, 0),
510                                     MVT::i32);
511  return true;
512}
513
514bool ARMDAGToDAGISel::SelectAddrMode6(SDNode *Op, SDValue N,
515                                      SDValue &Addr, SDValue &Align) {
516  Addr = N;
517  // Default to no alignment.
518  Align = CurDAG->getTargetConstant(0, MVT::i32);
519  return true;
520}
521
522bool ARMDAGToDAGISel::SelectAddrModePC(SDNode *Op, SDValue N,
523                                       SDValue &Offset, SDValue &Label) {
524  if (N.getOpcode() == ARMISD::PIC_ADD && N.hasOneUse()) {
525    Offset = N.getOperand(0);
526    SDValue N1 = N.getOperand(1);
527    Label  = CurDAG->getTargetConstant(cast<ConstantSDNode>(N1)->getZExtValue(),
528                                       MVT::i32);
529    return true;
530  }
531  return false;
532}
533
534bool ARMDAGToDAGISel::SelectThumbAddrModeRR(SDNode *Op, SDValue N,
535                                            SDValue &Base, SDValue &Offset){
536  // FIXME dl should come from the parent load or store, not the address
537  DebugLoc dl = Op->getDebugLoc();
538  if (N.getOpcode() != ISD::ADD) {
539    ConstantSDNode *NC = dyn_cast<ConstantSDNode>(N);
540    if (!NC || NC->getZExtValue() != 0)
541      return false;
542
543    Base = Offset = N;
544    return true;
545  }
546
547  Base = N.getOperand(0);
548  Offset = N.getOperand(1);
549  return true;
550}
551
552bool
553ARMDAGToDAGISel::SelectThumbAddrModeRI5(SDNode *Op, SDValue N,
554                                        unsigned Scale, SDValue &Base,
555                                        SDValue &OffImm, SDValue &Offset) {
556  if (Scale == 4) {
557    SDValue TmpBase, TmpOffImm;
558    if (SelectThumbAddrModeSP(Op, N, TmpBase, TmpOffImm))
559      return false;  // We want to select tLDRspi / tSTRspi instead.
560    if (N.getOpcode() == ARMISD::Wrapper &&
561        N.getOperand(0).getOpcode() == ISD::TargetConstantPool)
562      return false;  // We want to select tLDRpci instead.
563  }
564
565  if (N.getOpcode() != ISD::ADD) {
566    if (N.getOpcode() == ARMISD::Wrapper &&
567        !(Subtarget->useMovt() &&
568          N.getOperand(0).getOpcode() == ISD::TargetGlobalAddress)) {
569      Base = N.getOperand(0);
570    } else
571      Base = N;
572
573    Offset = CurDAG->getRegister(0, MVT::i32);
574    OffImm = CurDAG->getTargetConstant(0, MVT::i32);
575    return true;
576  }
577
578  // Thumb does not have [sp, r] address mode.
579  RegisterSDNode *LHSR = dyn_cast<RegisterSDNode>(N.getOperand(0));
580  RegisterSDNode *RHSR = dyn_cast<RegisterSDNode>(N.getOperand(1));
581  if ((LHSR && LHSR->getReg() == ARM::SP) ||
582      (RHSR && RHSR->getReg() == ARM::SP)) {
583    Base = N;
584    Offset = CurDAG->getRegister(0, MVT::i32);
585    OffImm = CurDAG->getTargetConstant(0, MVT::i32);
586    return true;
587  }
588
589  // If the RHS is + imm5 * scale, fold into addr mode.
590  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
591    int RHSC = (int)RHS->getZExtValue();
592    if ((RHSC & (Scale-1)) == 0) {  // The constant is implicitly multiplied.
593      RHSC /= Scale;
594      if (RHSC >= 0 && RHSC < 32) {
595        Base = N.getOperand(0);
596        Offset = CurDAG->getRegister(0, MVT::i32);
597        OffImm = CurDAG->getTargetConstant(RHSC, MVT::i32);
598        return true;
599      }
600    }
601  }
602
603  Base = N.getOperand(0);
604  Offset = N.getOperand(1);
605  OffImm = CurDAG->getTargetConstant(0, MVT::i32);
606  return true;
607}
608
609bool ARMDAGToDAGISel::SelectThumbAddrModeS1(SDNode *Op, SDValue N,
610                                            SDValue &Base, SDValue &OffImm,
611                                            SDValue &Offset) {
612  return SelectThumbAddrModeRI5(Op, N, 1, Base, OffImm, Offset);
613}
614
615bool ARMDAGToDAGISel::SelectThumbAddrModeS2(SDNode *Op, SDValue N,
616                                            SDValue &Base, SDValue &OffImm,
617                                            SDValue &Offset) {
618  return SelectThumbAddrModeRI5(Op, N, 2, Base, OffImm, Offset);
619}
620
621bool ARMDAGToDAGISel::SelectThumbAddrModeS4(SDNode *Op, SDValue N,
622                                            SDValue &Base, SDValue &OffImm,
623                                            SDValue &Offset) {
624  return SelectThumbAddrModeRI5(Op, N, 4, Base, OffImm, Offset);
625}
626
627bool ARMDAGToDAGISel::SelectThumbAddrModeSP(SDNode *Op, SDValue N,
628                                           SDValue &Base, SDValue &OffImm) {
629  if (N.getOpcode() == ISD::FrameIndex) {
630    int FI = cast<FrameIndexSDNode>(N)->getIndex();
631    Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
632    OffImm = CurDAG->getTargetConstant(0, MVT::i32);
633    return true;
634  }
635
636  if (N.getOpcode() != ISD::ADD)
637    return false;
638
639  RegisterSDNode *LHSR = dyn_cast<RegisterSDNode>(N.getOperand(0));
640  if (N.getOperand(0).getOpcode() == ISD::FrameIndex ||
641      (LHSR && LHSR->getReg() == ARM::SP)) {
642    // If the RHS is + imm8 * scale, fold into addr mode.
643    if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
644      int RHSC = (int)RHS->getZExtValue();
645      if ((RHSC & 3) == 0) {  // The constant is implicitly multiplied.
646        RHSC >>= 2;
647        if (RHSC >= 0 && RHSC < 256) {
648          Base = N.getOperand(0);
649          if (Base.getOpcode() == ISD::FrameIndex) {
650            int FI = cast<FrameIndexSDNode>(Base)->getIndex();
651            Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
652          }
653          OffImm = CurDAG->getTargetConstant(RHSC, MVT::i32);
654          return true;
655        }
656      }
657    }
658  }
659
660  return false;
661}
662
663bool ARMDAGToDAGISel::SelectT2ShifterOperandReg(SDNode *Op, SDValue N,
664                                                SDValue &BaseReg,
665                                                SDValue &Opc) {
666  ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(N);
667
668  // Don't match base register only case. That is matched to a separate
669  // lower complexity pattern with explicit register operand.
670  if (ShOpcVal == ARM_AM::no_shift) return false;
671
672  BaseReg = N.getOperand(0);
673  unsigned ShImmVal = 0;
674  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
675    ShImmVal = RHS->getZExtValue() & 31;
676    Opc = getI32Imm(ARM_AM::getSORegOpc(ShOpcVal, ShImmVal));
677    return true;
678  }
679
680  return false;
681}
682
683bool ARMDAGToDAGISel::SelectT2AddrModeImm12(SDNode *Op, SDValue N,
684                                            SDValue &Base, SDValue &OffImm) {
685  // Match simple R + imm12 operands.
686
687  // Base only.
688  if (N.getOpcode() != ISD::ADD && N.getOpcode() != ISD::SUB) {
689    if (N.getOpcode() == ISD::FrameIndex) {
690      // Match frame index...
691      int FI = cast<FrameIndexSDNode>(N)->getIndex();
692      Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
693      OffImm  = CurDAG->getTargetConstant(0, MVT::i32);
694      return true;
695    } else if (N.getOpcode() == ARMISD::Wrapper &&
696               !(Subtarget->useMovt() &&
697                 N.getOperand(0).getOpcode() == ISD::TargetGlobalAddress)) {
698      Base = N.getOperand(0);
699      if (Base.getOpcode() == ISD::TargetConstantPool)
700        return false;  // We want to select t2LDRpci instead.
701    } else
702      Base = N;
703    OffImm  = CurDAG->getTargetConstant(0, MVT::i32);
704    return true;
705  }
706
707  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
708    if (SelectT2AddrModeImm8(Op, N, Base, OffImm))
709      // Let t2LDRi8 handle (R - imm8).
710      return false;
711
712    int RHSC = (int)RHS->getZExtValue();
713    if (N.getOpcode() == ISD::SUB)
714      RHSC = -RHSC;
715
716    if (RHSC >= 0 && RHSC < 0x1000) { // 12 bits (unsigned)
717      Base   = N.getOperand(0);
718      if (Base.getOpcode() == ISD::FrameIndex) {
719        int FI = cast<FrameIndexSDNode>(Base)->getIndex();
720        Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
721      }
722      OffImm = CurDAG->getTargetConstant(RHSC, MVT::i32);
723      return true;
724    }
725  }
726
727  // Base only.
728  Base = N;
729  OffImm  = CurDAG->getTargetConstant(0, MVT::i32);
730  return true;
731}
732
733bool ARMDAGToDAGISel::SelectT2AddrModeImm8(SDNode *Op, SDValue N,
734                                           SDValue &Base, SDValue &OffImm) {
735  // Match simple R - imm8 operands.
736  if (N.getOpcode() == ISD::ADD || N.getOpcode() == ISD::SUB) {
737    if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
738      int RHSC = (int)RHS->getSExtValue();
739      if (N.getOpcode() == ISD::SUB)
740        RHSC = -RHSC;
741
742      if ((RHSC >= -255) && (RHSC < 0)) { // 8 bits (always negative)
743        Base = N.getOperand(0);
744        if (Base.getOpcode() == ISD::FrameIndex) {
745          int FI = cast<FrameIndexSDNode>(Base)->getIndex();
746          Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
747        }
748        OffImm = CurDAG->getTargetConstant(RHSC, MVT::i32);
749        return true;
750      }
751    }
752  }
753
754  return false;
755}
756
757bool ARMDAGToDAGISel::SelectT2AddrModeImm8Offset(SDNode *Op, SDValue N,
758                                                 SDValue &OffImm){
759  unsigned Opcode = Op->getOpcode();
760  ISD::MemIndexedMode AM = (Opcode == ISD::LOAD)
761    ? cast<LoadSDNode>(Op)->getAddressingMode()
762    : cast<StoreSDNode>(Op)->getAddressingMode();
763  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N)) {
764    int RHSC = (int)RHS->getZExtValue();
765    if (RHSC >= 0 && RHSC < 0x100) { // 8 bits.
766      OffImm = ((AM == ISD::PRE_INC) || (AM == ISD::POST_INC))
767        ? CurDAG->getTargetConstant(RHSC, MVT::i32)
768        : CurDAG->getTargetConstant(-RHSC, MVT::i32);
769      return true;
770    }
771  }
772
773  return false;
774}
775
776bool ARMDAGToDAGISel::SelectT2AddrModeImm8s4(SDNode *Op, SDValue N,
777                                             SDValue &Base, SDValue &OffImm) {
778  if (N.getOpcode() == ISD::ADD) {
779    if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
780      int RHSC = (int)RHS->getZExtValue();
781      if (((RHSC & 0x3) == 0) &&
782          ((RHSC >= 0 && RHSC < 0x400) || (RHSC < 0 && RHSC > -0x400))) { // 8 bits.
783        Base   = N.getOperand(0);
784        OffImm = CurDAG->getTargetConstant(RHSC, MVT::i32);
785        return true;
786      }
787    }
788  } else if (N.getOpcode() == ISD::SUB) {
789    if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
790      int RHSC = (int)RHS->getZExtValue();
791      if (((RHSC & 0x3) == 0) && (RHSC >= 0 && RHSC < 0x400)) { // 8 bits.
792        Base   = N.getOperand(0);
793        OffImm = CurDAG->getTargetConstant(-RHSC, MVT::i32);
794        return true;
795      }
796    }
797  }
798
799  return false;
800}
801
802bool ARMDAGToDAGISel::SelectT2AddrModeSoReg(SDNode *Op, SDValue N,
803                                            SDValue &Base,
804                                            SDValue &OffReg, SDValue &ShImm) {
805  // (R - imm8) should be handled by t2LDRi8. The rest are handled by t2LDRi12.
806  if (N.getOpcode() != ISD::ADD)
807    return false;
808
809  // Leave (R + imm12) for t2LDRi12, (R - imm8) for t2LDRi8.
810  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
811    int RHSC = (int)RHS->getZExtValue();
812    if (RHSC >= 0 && RHSC < 0x1000) // 12 bits (unsigned)
813      return false;
814    else if (RHSC < 0 && RHSC >= -255) // 8 bits
815      return false;
816  }
817
818  // Look for (R + R) or (R + (R << [1,2,3])).
819  unsigned ShAmt = 0;
820  Base   = N.getOperand(0);
821  OffReg = N.getOperand(1);
822
823  // Swap if it is ((R << c) + R).
824  ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(OffReg);
825  if (ShOpcVal != ARM_AM::lsl) {
826    ShOpcVal = ARM_AM::getShiftOpcForNode(Base);
827    if (ShOpcVal == ARM_AM::lsl)
828      std::swap(Base, OffReg);
829  }
830
831  if (ShOpcVal == ARM_AM::lsl) {
832    // Check to see if the RHS of the shift is a constant, if not, we can't fold
833    // it.
834    if (ConstantSDNode *Sh = dyn_cast<ConstantSDNode>(OffReg.getOperand(1))) {
835      ShAmt = Sh->getZExtValue();
836      if (ShAmt >= 4) {
837        ShAmt = 0;
838        ShOpcVal = ARM_AM::no_shift;
839      } else
840        OffReg = OffReg.getOperand(0);
841    } else {
842      ShOpcVal = ARM_AM::no_shift;
843    }
844  }
845
846  ShImm = CurDAG->getTargetConstant(ShAmt, MVT::i32);
847
848  return true;
849}
850
851//===--------------------------------------------------------------------===//
852
853/// getAL - Returns a ARMCC::AL immediate node.
854static inline SDValue getAL(SelectionDAG *CurDAG) {
855  return CurDAG->getTargetConstant((uint64_t)ARMCC::AL, MVT::i32);
856}
857
858SDNode *ARMDAGToDAGISel::SelectARMIndexedLoad(SDNode *N) {
859  LoadSDNode *LD = cast<LoadSDNode>(N);
860  ISD::MemIndexedMode AM = LD->getAddressingMode();
861  if (AM == ISD::UNINDEXED)
862    return NULL;
863
864  EVT LoadedVT = LD->getMemoryVT();
865  SDValue Offset, AMOpc;
866  bool isPre = (AM == ISD::PRE_INC) || (AM == ISD::PRE_DEC);
867  unsigned Opcode = 0;
868  bool Match = false;
869  if (LoadedVT == MVT::i32 &&
870      SelectAddrMode2Offset(N, LD->getOffset(), Offset, AMOpc)) {
871    Opcode = isPre ? ARM::LDR_PRE : ARM::LDR_POST;
872    Match = true;
873  } else if (LoadedVT == MVT::i16 &&
874             SelectAddrMode3Offset(N, LD->getOffset(), Offset, AMOpc)) {
875    Match = true;
876    Opcode = (LD->getExtensionType() == ISD::SEXTLOAD)
877      ? (isPre ? ARM::LDRSH_PRE : ARM::LDRSH_POST)
878      : (isPre ? ARM::LDRH_PRE : ARM::LDRH_POST);
879  } else if (LoadedVT == MVT::i8 || LoadedVT == MVT::i1) {
880    if (LD->getExtensionType() == ISD::SEXTLOAD) {
881      if (SelectAddrMode3Offset(N, LD->getOffset(), Offset, AMOpc)) {
882        Match = true;
883        Opcode = isPre ? ARM::LDRSB_PRE : ARM::LDRSB_POST;
884      }
885    } else {
886      if (SelectAddrMode2Offset(N, LD->getOffset(), Offset, AMOpc)) {
887        Match = true;
888        Opcode = isPre ? ARM::LDRB_PRE : ARM::LDRB_POST;
889      }
890    }
891  }
892
893  if (Match) {
894    SDValue Chain = LD->getChain();
895    SDValue Base = LD->getBasePtr();
896    SDValue Ops[]= { Base, Offset, AMOpc, getAL(CurDAG),
897                     CurDAG->getRegister(0, MVT::i32), Chain };
898    return CurDAG->getMachineNode(Opcode, N->getDebugLoc(), MVT::i32, MVT::i32,
899                                  MVT::Other, Ops, 6);
900  }
901
902  return NULL;
903}
904
905SDNode *ARMDAGToDAGISel::SelectT2IndexedLoad(SDNode *N) {
906  LoadSDNode *LD = cast<LoadSDNode>(N);
907  ISD::MemIndexedMode AM = LD->getAddressingMode();
908  if (AM == ISD::UNINDEXED)
909    return NULL;
910
911  EVT LoadedVT = LD->getMemoryVT();
912  bool isSExtLd = LD->getExtensionType() == ISD::SEXTLOAD;
913  SDValue Offset;
914  bool isPre = (AM == ISD::PRE_INC) || (AM == ISD::PRE_DEC);
915  unsigned Opcode = 0;
916  bool Match = false;
917  if (SelectT2AddrModeImm8Offset(N, LD->getOffset(), Offset)) {
918    switch (LoadedVT.getSimpleVT().SimpleTy) {
919    case MVT::i32:
920      Opcode = isPre ? ARM::t2LDR_PRE : ARM::t2LDR_POST;
921      break;
922    case MVT::i16:
923      if (isSExtLd)
924        Opcode = isPre ? ARM::t2LDRSH_PRE : ARM::t2LDRSH_POST;
925      else
926        Opcode = isPre ? ARM::t2LDRH_PRE : ARM::t2LDRH_POST;
927      break;
928    case MVT::i8:
929    case MVT::i1:
930      if (isSExtLd)
931        Opcode = isPre ? ARM::t2LDRSB_PRE : ARM::t2LDRSB_POST;
932      else
933        Opcode = isPre ? ARM::t2LDRB_PRE : ARM::t2LDRB_POST;
934      break;
935    default:
936      return NULL;
937    }
938    Match = true;
939  }
940
941  if (Match) {
942    SDValue Chain = LD->getChain();
943    SDValue Base = LD->getBasePtr();
944    SDValue Ops[]= { Base, Offset, getAL(CurDAG),
945                     CurDAG->getRegister(0, MVT::i32), Chain };
946    return CurDAG->getMachineNode(Opcode, N->getDebugLoc(), MVT::i32, MVT::i32,
947                                  MVT::Other, Ops, 5);
948  }
949
950  return NULL;
951}
952
953/// PairDRegs - Form a quad register from a pair of D registers.
954///
955SDNode *ARMDAGToDAGISel::PairDRegs(EVT VT, SDValue V0, SDValue V1) {
956  DebugLoc dl = V0.getNode()->getDebugLoc();
957  SDValue SubReg0 = CurDAG->getTargetConstant(ARM::DSUBREG_0, MVT::i32);
958  SDValue SubReg1 = CurDAG->getTargetConstant(ARM::DSUBREG_1, MVT::i32);
959  if (llvm::ModelWithRegSequence()) {
960    const SDValue Ops[] = { V0, SubReg0, V1, SubReg1 };
961    return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops, 4);
962  }
963  SDValue Undef =
964    SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, dl, VT), 0);
965  SDNode *Pair = CurDAG->getMachineNode(TargetOpcode::INSERT_SUBREG, dl,
966                                        VT, Undef, V0, SubReg0);
967  return CurDAG->getMachineNode(TargetOpcode::INSERT_SUBREG, dl,
968                                VT, SDValue(Pair, 0), V1, SubReg1);
969}
970
971/// PairDRegs - Form a quad register pair from a pair of Q registers.
972///
973SDNode *ARMDAGToDAGISel::PairQRegs(EVT VT, SDValue V0, SDValue V1) {
974  DebugLoc dl = V0.getNode()->getDebugLoc();
975  SDValue SubReg0 = CurDAG->getTargetConstant(ARM::QSUBREG_0, MVT::i32);
976  SDValue SubReg1 = CurDAG->getTargetConstant(ARM::QSUBREG_1, MVT::i32);
977  const SDValue Ops[] = { V0, SubReg0, V1, SubReg1 };
978  return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops, 4);
979}
980
981/// QuadDRegs - Form a octo register from a quad of D registers.
982///
983SDNode *ARMDAGToDAGISel::QuadDRegs(EVT VT, SDValue V0, SDValue V1,
984                                   SDValue V2, SDValue V3) {
985  DebugLoc dl = V0.getNode()->getDebugLoc();
986  SDValue SubReg0 = CurDAG->getTargetConstant(ARM::DSUBREG_0, MVT::i32);
987  SDValue SubReg1 = CurDAG->getTargetConstant(ARM::DSUBREG_1, MVT::i32);
988  SDValue SubReg2 = CurDAG->getTargetConstant(ARM::DSUBREG_2, MVT::i32);
989  SDValue SubReg3 = CurDAG->getTargetConstant(ARM::DSUBREG_3, MVT::i32);
990  const SDValue Ops[] = { V0, SubReg0, V1, SubReg1, V2, SubReg2, V3, SubReg3 };
991  return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops, 8);
992}
993
994/// GetNEONSubregVT - Given a type for a 128-bit NEON vector, return the type
995/// for a 64-bit subregister of the vector.
996static EVT GetNEONSubregVT(EVT VT) {
997  switch (VT.getSimpleVT().SimpleTy) {
998  default: llvm_unreachable("unhandled NEON type");
999  case MVT::v16i8: return MVT::v8i8;
1000  case MVT::v8i16: return MVT::v4i16;
1001  case MVT::v4f32: return MVT::v2f32;
1002  case MVT::v4i32: return MVT::v2i32;
1003  case MVT::v2i64: return MVT::v1i64;
1004  }
1005}
1006
1007SDNode *ARMDAGToDAGISel::SelectVLD(SDNode *N, unsigned NumVecs,
1008                                   unsigned *DOpcodes, unsigned *QOpcodes0,
1009                                   unsigned *QOpcodes1) {
1010  assert(NumVecs >= 1 && NumVecs <= 4 && "VLD NumVecs out-of-range");
1011  DebugLoc dl = N->getDebugLoc();
1012
1013  SDValue MemAddr, Align;
1014  if (!SelectAddrMode6(N, N->getOperand(2), MemAddr, Align))
1015    return NULL;
1016
1017  SDValue Chain = N->getOperand(0);
1018  EVT VT = N->getValueType(0);
1019  bool is64BitVector = VT.is64BitVector();
1020
1021  unsigned OpcodeIndex;
1022  switch (VT.getSimpleVT().SimpleTy) {
1023  default: llvm_unreachable("unhandled vld type");
1024    // Double-register operations:
1025  case MVT::v8i8:  OpcodeIndex = 0; break;
1026  case MVT::v4i16: OpcodeIndex = 1; break;
1027  case MVT::v2f32:
1028  case MVT::v2i32: OpcodeIndex = 2; break;
1029  case MVT::v1i64: OpcodeIndex = 3; break;
1030    // Quad-register operations:
1031  case MVT::v16i8: OpcodeIndex = 0; break;
1032  case MVT::v8i16: OpcodeIndex = 1; break;
1033  case MVT::v4f32:
1034  case MVT::v4i32: OpcodeIndex = 2; break;
1035  case MVT::v2i64: OpcodeIndex = 3;
1036    assert(NumVecs == 1 && "v2i64 type only supported for VLD1");
1037    break;
1038  }
1039
1040  SDValue Pred = getAL(CurDAG);
1041  SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
1042  if (is64BitVector) {
1043    unsigned Opc = DOpcodes[OpcodeIndex];
1044    const SDValue Ops[] = { MemAddr, Align, Pred, Reg0, Chain };
1045    std::vector<EVT> ResTys(NumVecs, VT);
1046    ResTys.push_back(MVT::Other);
1047    return CurDAG->getMachineNode(Opc, dl, ResTys, Ops, 5);
1048  }
1049
1050  EVT RegVT = GetNEONSubregVT(VT);
1051  if (NumVecs <= 2) {
1052    // Quad registers are directly supported for VLD1 and VLD2,
1053    // loading pairs of D regs.
1054    unsigned Opc = QOpcodes0[OpcodeIndex];
1055    const SDValue Ops[] = { MemAddr, Align, Pred, Reg0, Chain };
1056    std::vector<EVT> ResTys(2 * NumVecs, RegVT);
1057    ResTys.push_back(MVT::Other);
1058    SDNode *VLd = CurDAG->getMachineNode(Opc, dl, ResTys, Ops, 5);
1059    Chain = SDValue(VLd, 2 * NumVecs);
1060
1061    // Combine the even and odd subregs to produce the result.
1062    if (llvm::ModelWithRegSequence()) {
1063      if (NumVecs == 1) {
1064        SDNode *Q = PairDRegs(VT, SDValue(VLd, 0), SDValue(VLd, 1));
1065        ReplaceUses(SDValue(N, 0), SDValue(Q, 0));
1066      } else {
1067        SDValue QQ = SDValue(QuadDRegs(MVT::v4i64,
1068                                       SDValue(VLd, 0), SDValue(VLd, 1),
1069                                       SDValue(VLd, 2), SDValue(VLd, 3)), 0);
1070        SDValue Q0 = CurDAG->getTargetExtractSubreg(ARM::QSUBREG_0, dl, VT, QQ);
1071        SDValue Q1 = CurDAG->getTargetExtractSubreg(ARM::QSUBREG_1, dl, VT, QQ);
1072        ReplaceUses(SDValue(N, 0), Q0);
1073        ReplaceUses(SDValue(N, 1), Q1);
1074      }
1075    } else {
1076      for (unsigned Vec = 0; Vec < NumVecs; ++Vec) {
1077        SDNode *Q = PairDRegs(VT, SDValue(VLd, 2*Vec), SDValue(VLd, 2*Vec+1));
1078        ReplaceUses(SDValue(N, Vec), SDValue(Q, 0));
1079      }
1080    }
1081  } else {
1082    // Otherwise, quad registers are loaded with two separate instructions,
1083    // where one loads the even registers and the other loads the odd registers.
1084
1085    std::vector<EVT> ResTys(NumVecs, RegVT);
1086    ResTys.push_back(MemAddr.getValueType());
1087    ResTys.push_back(MVT::Other);
1088
1089    // Load the even subregs.
1090    unsigned Opc = QOpcodes0[OpcodeIndex];
1091    const SDValue OpsA[] = { MemAddr, Align, Reg0, Pred, Reg0, Chain };
1092    SDNode *VLdA = CurDAG->getMachineNode(Opc, dl, ResTys, OpsA, 6);
1093    Chain = SDValue(VLdA, NumVecs+1);
1094
1095    // Load the odd subregs.
1096    Opc = QOpcodes1[OpcodeIndex];
1097    const SDValue OpsB[] = { SDValue(VLdA, NumVecs),
1098                             Align, Reg0, Pred, Reg0, Chain };
1099    SDNode *VLdB = CurDAG->getMachineNode(Opc, dl, ResTys, OpsB, 6);
1100    Chain = SDValue(VLdB, NumVecs+1);
1101
1102    // Combine the even and odd subregs to produce the result.
1103    for (unsigned Vec = 0; Vec < NumVecs; ++Vec) {
1104      SDNode *Q = PairDRegs(VT, SDValue(VLdA, Vec), SDValue(VLdB, Vec));
1105      ReplaceUses(SDValue(N, Vec), SDValue(Q, 0));
1106    }
1107  }
1108  ReplaceUses(SDValue(N, NumVecs), Chain);
1109  return NULL;
1110}
1111
1112SDNode *ARMDAGToDAGISel::SelectVST(SDNode *N, unsigned NumVecs,
1113                                   unsigned *DOpcodes, unsigned *QOpcodes0,
1114                                   unsigned *QOpcodes1) {
1115  assert(NumVecs >=1 && NumVecs <= 4 && "VST NumVecs out-of-range");
1116  DebugLoc dl = N->getDebugLoc();
1117
1118  SDValue MemAddr, Align;
1119  if (!SelectAddrMode6(N, N->getOperand(2), MemAddr, Align))
1120    return NULL;
1121
1122  SDValue Chain = N->getOperand(0);
1123  EVT VT = N->getOperand(3).getValueType();
1124  bool is64BitVector = VT.is64BitVector();
1125
1126  unsigned OpcodeIndex;
1127  switch (VT.getSimpleVT().SimpleTy) {
1128  default: llvm_unreachable("unhandled vst type");
1129    // Double-register operations:
1130  case MVT::v8i8:  OpcodeIndex = 0; break;
1131  case MVT::v4i16: OpcodeIndex = 1; break;
1132  case MVT::v2f32:
1133  case MVT::v2i32: OpcodeIndex = 2; break;
1134  case MVT::v1i64: OpcodeIndex = 3; break;
1135    // Quad-register operations:
1136  case MVT::v16i8: OpcodeIndex = 0; break;
1137  case MVT::v8i16: OpcodeIndex = 1; break;
1138  case MVT::v4f32:
1139  case MVT::v4i32: OpcodeIndex = 2; break;
1140  case MVT::v2i64: OpcodeIndex = 3;
1141    assert(NumVecs == 1 && "v2i64 type only supported for VST1");
1142    break;
1143  }
1144
1145  SDValue Pred = getAL(CurDAG);
1146  SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
1147
1148  SmallVector<SDValue, 10> Ops;
1149  Ops.push_back(MemAddr);
1150  Ops.push_back(Align);
1151
1152  if (is64BitVector) {
1153    unsigned Opc = DOpcodes[OpcodeIndex];
1154    for (unsigned Vec = 0; Vec < NumVecs; ++Vec)
1155      Ops.push_back(N->getOperand(Vec+3));
1156    Ops.push_back(Pred);
1157    Ops.push_back(Reg0); // predicate register
1158    Ops.push_back(Chain);
1159    return CurDAG->getMachineNode(Opc, dl, MVT::Other, Ops.data(), NumVecs+5);
1160  }
1161
1162  EVT RegVT = GetNEONSubregVT(VT);
1163  if (NumVecs <= 2) {
1164    // Quad registers are directly supported for VST1 and VST2,
1165    // storing pairs of D regs.
1166    unsigned Opc = QOpcodes0[OpcodeIndex];
1167    if (llvm::ModelWithRegSequence() && NumVecs == 2) {
1168      // First extract the quad D registers.
1169      SDValue Q0 = N->getOperand(3);
1170      SDValue Q1 = N->getOperand(4);
1171
1172      // Form a QQ register.
1173      SDValue QQ = SDValue(PairQRegs(MVT::v4i64, Q0, Q1), 0);
1174
1175      // Now extract the D registers back out.
1176      Ops.push_back(CurDAG->getTargetExtractSubreg(ARM::DSUBREG_0, dl, RegVT,
1177                                                   QQ));
1178      Ops.push_back(CurDAG->getTargetExtractSubreg(ARM::DSUBREG_1, dl, RegVT,
1179                                                   QQ));
1180      Ops.push_back(CurDAG->getTargetExtractSubreg(ARM::DSUBREG_2, dl, RegVT,
1181                                                   QQ));
1182      Ops.push_back(CurDAG->getTargetExtractSubreg(ARM::DSUBREG_3, dl, RegVT,
1183                                                   QQ));
1184      Ops.push_back(Pred);
1185      Ops.push_back(Reg0); // predicate register
1186      Ops.push_back(Chain);
1187      return CurDAG->getMachineNode(Opc, dl, MVT::Other, Ops.data(), 5 + 4);
1188    } else {
1189      for (unsigned Vec = 0; Vec < NumVecs; ++Vec) {
1190        Ops.push_back(CurDAG->getTargetExtractSubreg(ARM::DSUBREG_0, dl, RegVT,
1191                                                     N->getOperand(Vec+3)));
1192        Ops.push_back(CurDAG->getTargetExtractSubreg(ARM::DSUBREG_1, dl, RegVT,
1193                                                     N->getOperand(Vec+3)));
1194      }
1195      Ops.push_back(Pred);
1196      Ops.push_back(Reg0); // predicate register
1197      Ops.push_back(Chain);
1198      return CurDAG->getMachineNode(Opc, dl, MVT::Other, Ops.data(),
1199                                    5 + 2 * NumVecs);
1200    }
1201  }
1202
1203  // Otherwise, quad registers are stored with two separate instructions,
1204  // where one stores the even registers and the other stores the odd registers.
1205
1206  Ops.push_back(Reg0); // post-access address offset
1207
1208  // Store the even subregs.
1209  for (unsigned Vec = 0; Vec < NumVecs; ++Vec)
1210    Ops.push_back(CurDAG->getTargetExtractSubreg(ARM::DSUBREG_0, dl, RegVT,
1211                                                 N->getOperand(Vec+3)));
1212  Ops.push_back(Pred);
1213  Ops.push_back(Reg0); // predicate register
1214  Ops.push_back(Chain);
1215  unsigned Opc = QOpcodes0[OpcodeIndex];
1216  SDNode *VStA = CurDAG->getMachineNode(Opc, dl, MemAddr.getValueType(),
1217                                        MVT::Other, Ops.data(), NumVecs+6);
1218  Chain = SDValue(VStA, 1);
1219
1220  // Store the odd subregs.
1221  Ops[0] = SDValue(VStA, 0); // MemAddr
1222  for (unsigned Vec = 0; Vec < NumVecs; ++Vec)
1223    Ops[Vec+3] = CurDAG->getTargetExtractSubreg(ARM::DSUBREG_1, dl, RegVT,
1224                                                N->getOperand(Vec+3));
1225  Ops[NumVecs+5] = Chain;
1226  Opc = QOpcodes1[OpcodeIndex];
1227  SDNode *VStB = CurDAG->getMachineNode(Opc, dl, MemAddr.getValueType(),
1228                                        MVT::Other, Ops.data(), NumVecs+6);
1229  Chain = SDValue(VStB, 1);
1230  ReplaceUses(SDValue(N, 0), Chain);
1231  return NULL;
1232}
1233
1234SDNode *ARMDAGToDAGISel::SelectVLDSTLane(SDNode *N, bool IsLoad,
1235                                         unsigned NumVecs, unsigned *DOpcodes,
1236                                         unsigned *QOpcodes0,
1237                                         unsigned *QOpcodes1) {
1238  assert(NumVecs >=2 && NumVecs <= 4 && "VLDSTLane NumVecs out-of-range");
1239  DebugLoc dl = N->getDebugLoc();
1240
1241  SDValue MemAddr, Align;
1242  if (!SelectAddrMode6(N, N->getOperand(2), MemAddr, Align))
1243    return NULL;
1244
1245  SDValue Chain = N->getOperand(0);
1246  unsigned Lane =
1247    cast<ConstantSDNode>(N->getOperand(NumVecs+3))->getZExtValue();
1248  EVT VT = IsLoad ? N->getValueType(0) : N->getOperand(3).getValueType();
1249  bool is64BitVector = VT.is64BitVector();
1250
1251  // Quad registers are handled by load/store of subregs. Find the subreg info.
1252  unsigned NumElts = 0;
1253  int SubregIdx = 0;
1254  EVT RegVT = VT;
1255  if (!is64BitVector) {
1256    RegVT = GetNEONSubregVT(VT);
1257    NumElts = RegVT.getVectorNumElements();
1258    SubregIdx = (Lane < NumElts) ? ARM::DSUBREG_0 : ARM::DSUBREG_1;
1259  }
1260
1261  unsigned OpcodeIndex;
1262  switch (VT.getSimpleVT().SimpleTy) {
1263  default: llvm_unreachable("unhandled vld/vst lane type");
1264    // Double-register operations:
1265  case MVT::v8i8:  OpcodeIndex = 0; break;
1266  case MVT::v4i16: OpcodeIndex = 1; break;
1267  case MVT::v2f32:
1268  case MVT::v2i32: OpcodeIndex = 2; break;
1269    // Quad-register operations:
1270  case MVT::v8i16: OpcodeIndex = 0; break;
1271  case MVT::v4f32:
1272  case MVT::v4i32: OpcodeIndex = 1; break;
1273  }
1274
1275  SDValue Pred = getAL(CurDAG);
1276  SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
1277
1278  SmallVector<SDValue, 10> Ops;
1279  Ops.push_back(MemAddr);
1280  Ops.push_back(Align);
1281
1282  unsigned Opc = 0;
1283  if (is64BitVector) {
1284    Opc = DOpcodes[OpcodeIndex];
1285    for (unsigned Vec = 0; Vec < NumVecs; ++Vec)
1286      Ops.push_back(N->getOperand(Vec+3));
1287  } else {
1288    // Check if this is loading the even or odd subreg of a Q register.
1289    if (Lane < NumElts) {
1290      Opc = QOpcodes0[OpcodeIndex];
1291    } else {
1292      Lane -= NumElts;
1293      Opc = QOpcodes1[OpcodeIndex];
1294    }
1295    // Extract the subregs of the input vector.
1296    for (unsigned Vec = 0; Vec < NumVecs; ++Vec)
1297      Ops.push_back(CurDAG->getTargetExtractSubreg(SubregIdx, dl, RegVT,
1298                                                   N->getOperand(Vec+3)));
1299  }
1300  Ops.push_back(getI32Imm(Lane));
1301  Ops.push_back(Pred);
1302  Ops.push_back(Reg0);
1303  Ops.push_back(Chain);
1304
1305  if (!IsLoad)
1306    return CurDAG->getMachineNode(Opc, dl, MVT::Other, Ops.data(), NumVecs+6);
1307
1308  std::vector<EVT> ResTys(NumVecs, RegVT);
1309  ResTys.push_back(MVT::Other);
1310  SDNode *VLdLn =
1311    CurDAG->getMachineNode(Opc, dl, ResTys, Ops.data(), NumVecs+6);
1312  // For a 64-bit vector load to D registers, nothing more needs to be done.
1313  if (is64BitVector)
1314    return VLdLn;
1315
1316  // For 128-bit vectors, take the 64-bit results of the load and insert them
1317  // as subregs into the result.
1318  for (unsigned Vec = 0; Vec < NumVecs; ++Vec) {
1319    SDValue QuadVec = CurDAG->getTargetInsertSubreg(SubregIdx, dl, VT,
1320                                                    N->getOperand(Vec+3),
1321                                                    SDValue(VLdLn, Vec));
1322    ReplaceUses(SDValue(N, Vec), QuadVec);
1323  }
1324
1325  Chain = SDValue(VLdLn, NumVecs);
1326  ReplaceUses(SDValue(N, NumVecs), Chain);
1327  return NULL;
1328}
1329
1330SDNode *ARMDAGToDAGISel::SelectV6T2BitfieldExtractOp(SDNode *N,
1331                                                     bool isSigned) {
1332  if (!Subtarget->hasV6T2Ops())
1333    return NULL;
1334
1335  unsigned Opc = isSigned ? (Subtarget->isThumb() ? ARM::t2SBFX : ARM::SBFX)
1336    : (Subtarget->isThumb() ? ARM::t2UBFX : ARM::UBFX);
1337
1338
1339  // For unsigned extracts, check for a shift right and mask
1340  unsigned And_imm = 0;
1341  if (N->getOpcode() == ISD::AND) {
1342    if (isOpcWithIntImmediate(N, ISD::AND, And_imm)) {
1343
1344      // The immediate is a mask of the low bits iff imm & (imm+1) == 0
1345      if (And_imm & (And_imm + 1))
1346        return NULL;
1347
1348      unsigned Srl_imm = 0;
1349      if (isOpcWithIntImmediate(N->getOperand(0).getNode(), ISD::SRL,
1350                                Srl_imm)) {
1351        assert(Srl_imm > 0 && Srl_imm < 32 && "bad amount in shift node!");
1352
1353        unsigned Width = CountTrailingOnes_32(And_imm);
1354        unsigned LSB = Srl_imm;
1355        SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
1356        SDValue Ops[] = { N->getOperand(0).getOperand(0),
1357                          CurDAG->getTargetConstant(LSB, MVT::i32),
1358                          CurDAG->getTargetConstant(Width, MVT::i32),
1359          getAL(CurDAG), Reg0 };
1360        return CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops, 5);
1361      }
1362    }
1363    return NULL;
1364  }
1365
1366  // Otherwise, we're looking for a shift of a shift
1367  unsigned Shl_imm = 0;
1368  if (isOpcWithIntImmediate(N->getOperand(0).getNode(), ISD::SHL, Shl_imm)) {
1369    assert(Shl_imm > 0 && Shl_imm < 32 && "bad amount in shift node!");
1370    unsigned Srl_imm = 0;
1371    if (isInt32Immediate(N->getOperand(1), Srl_imm)) {
1372      assert(Srl_imm > 0 && Srl_imm < 32 && "bad amount in shift node!");
1373      unsigned Width = 32 - Srl_imm;
1374      int LSB = Srl_imm - Shl_imm;
1375      if (LSB < 0)
1376        return NULL;
1377      SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
1378      SDValue Ops[] = { N->getOperand(0).getOperand(0),
1379                        CurDAG->getTargetConstant(LSB, MVT::i32),
1380                        CurDAG->getTargetConstant(Width, MVT::i32),
1381                        getAL(CurDAG), Reg0 };
1382      return CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops, 5);
1383    }
1384  }
1385  return NULL;
1386}
1387
1388SDNode *ARMDAGToDAGISel::
1389SelectT2CMOVShiftOp(SDNode *N, SDValue FalseVal, SDValue TrueVal,
1390                    ARMCC::CondCodes CCVal, SDValue CCR, SDValue InFlag) {
1391  SDValue CPTmp0;
1392  SDValue CPTmp1;
1393  if (SelectT2ShifterOperandReg(N, TrueVal, CPTmp0, CPTmp1)) {
1394    unsigned SOVal = cast<ConstantSDNode>(CPTmp1)->getZExtValue();
1395    unsigned SOShOp = ARM_AM::getSORegShOp(SOVal);
1396    unsigned Opc = 0;
1397    switch (SOShOp) {
1398    case ARM_AM::lsl: Opc = ARM::t2MOVCClsl; break;
1399    case ARM_AM::lsr: Opc = ARM::t2MOVCClsr; break;
1400    case ARM_AM::asr: Opc = ARM::t2MOVCCasr; break;
1401    case ARM_AM::ror: Opc = ARM::t2MOVCCror; break;
1402    default:
1403      llvm_unreachable("Unknown so_reg opcode!");
1404      break;
1405    }
1406    SDValue SOShImm =
1407      CurDAG->getTargetConstant(ARM_AM::getSORegOffset(SOVal), MVT::i32);
1408    SDValue CC = CurDAG->getTargetConstant(CCVal, MVT::i32);
1409    SDValue Ops[] = { FalseVal, CPTmp0, SOShImm, CC, CCR, InFlag };
1410    return CurDAG->SelectNodeTo(N, Opc, MVT::i32,Ops, 6);
1411  }
1412  return 0;
1413}
1414
1415SDNode *ARMDAGToDAGISel::
1416SelectARMCMOVShiftOp(SDNode *N, SDValue FalseVal, SDValue TrueVal,
1417                     ARMCC::CondCodes CCVal, SDValue CCR, SDValue InFlag) {
1418  SDValue CPTmp0;
1419  SDValue CPTmp1;
1420  SDValue CPTmp2;
1421  if (SelectShifterOperandReg(N, TrueVal, CPTmp0, CPTmp1, CPTmp2)) {
1422    SDValue CC = CurDAG->getTargetConstant(CCVal, MVT::i32);
1423    SDValue Ops[] = { FalseVal, CPTmp0, CPTmp1, CPTmp2, CC, CCR, InFlag };
1424    return CurDAG->SelectNodeTo(N, ARM::MOVCCs, MVT::i32, Ops, 7);
1425  }
1426  return 0;
1427}
1428
1429SDNode *ARMDAGToDAGISel::
1430SelectT2CMOVSoImmOp(SDNode *N, SDValue FalseVal, SDValue TrueVal,
1431                    ARMCC::CondCodes CCVal, SDValue CCR, SDValue InFlag) {
1432  ConstantSDNode *T = dyn_cast<ConstantSDNode>(TrueVal);
1433  if (!T)
1434    return 0;
1435
1436  if (Predicate_t2_so_imm(TrueVal.getNode())) {
1437    SDValue True = CurDAG->getTargetConstant(T->getZExtValue(), MVT::i32);
1438    SDValue CC = CurDAG->getTargetConstant(CCVal, MVT::i32);
1439    SDValue Ops[] = { FalseVal, True, CC, CCR, InFlag };
1440    return CurDAG->SelectNodeTo(N,
1441                                ARM::t2MOVCCi, MVT::i32, Ops, 5);
1442  }
1443  return 0;
1444}
1445
1446SDNode *ARMDAGToDAGISel::
1447SelectARMCMOVSoImmOp(SDNode *N, SDValue FalseVal, SDValue TrueVal,
1448                     ARMCC::CondCodes CCVal, SDValue CCR, SDValue InFlag) {
1449  ConstantSDNode *T = dyn_cast<ConstantSDNode>(TrueVal);
1450  if (!T)
1451    return 0;
1452
1453  if (Predicate_so_imm(TrueVal.getNode())) {
1454    SDValue True = CurDAG->getTargetConstant(T->getZExtValue(), MVT::i32);
1455    SDValue CC = CurDAG->getTargetConstant(CCVal, MVT::i32);
1456    SDValue Ops[] = { FalseVal, True, CC, CCR, InFlag };
1457    return CurDAG->SelectNodeTo(N,
1458                                ARM::MOVCCi, MVT::i32, Ops, 5);
1459  }
1460  return 0;
1461}
1462
1463SDNode *ARMDAGToDAGISel::SelectCMOVOp(SDNode *N) {
1464  EVT VT = N->getValueType(0);
1465  SDValue FalseVal = N->getOperand(0);
1466  SDValue TrueVal  = N->getOperand(1);
1467  SDValue CC = N->getOperand(2);
1468  SDValue CCR = N->getOperand(3);
1469  SDValue InFlag = N->getOperand(4);
1470  assert(CC.getOpcode() == ISD::Constant);
1471  assert(CCR.getOpcode() == ISD::Register);
1472  ARMCC::CondCodes CCVal =
1473    (ARMCC::CondCodes)cast<ConstantSDNode>(CC)->getZExtValue();
1474
1475  if (!Subtarget->isThumb1Only() && VT == MVT::i32) {
1476    // Pattern: (ARMcmov:i32 GPR:i32:$false, so_reg:i32:$true, (imm:i32):$cc)
1477    // Emits: (MOVCCs:i32 GPR:i32:$false, so_reg:i32:$true, (imm:i32):$cc)
1478    // Pattern complexity = 18  cost = 1  size = 0
1479    SDValue CPTmp0;
1480    SDValue CPTmp1;
1481    SDValue CPTmp2;
1482    if (Subtarget->isThumb()) {
1483      SDNode *Res = SelectT2CMOVShiftOp(N, FalseVal, TrueVal,
1484                                        CCVal, CCR, InFlag);
1485      if (!Res)
1486        Res = SelectT2CMOVShiftOp(N, TrueVal, FalseVal,
1487                               ARMCC::getOppositeCondition(CCVal), CCR, InFlag);
1488      if (Res)
1489        return Res;
1490    } else {
1491      SDNode *Res = SelectARMCMOVShiftOp(N, FalseVal, TrueVal,
1492                                         CCVal, CCR, InFlag);
1493      if (!Res)
1494        Res = SelectARMCMOVShiftOp(N, TrueVal, FalseVal,
1495                               ARMCC::getOppositeCondition(CCVal), CCR, InFlag);
1496      if (Res)
1497        return Res;
1498    }
1499
1500    // Pattern: (ARMcmov:i32 GPR:i32:$false,
1501    //             (imm:i32)<<P:Predicate_so_imm>>:$true,
1502    //             (imm:i32):$cc)
1503    // Emits: (MOVCCi:i32 GPR:i32:$false,
1504    //           (so_imm:i32 (imm:i32):$true), (imm:i32):$cc)
1505    // Pattern complexity = 10  cost = 1  size = 0
1506    if (Subtarget->isThumb()) {
1507      SDNode *Res = SelectT2CMOVSoImmOp(N, FalseVal, TrueVal,
1508                                        CCVal, CCR, InFlag);
1509      if (!Res)
1510        Res = SelectT2CMOVSoImmOp(N, TrueVal, FalseVal,
1511                               ARMCC::getOppositeCondition(CCVal), CCR, InFlag);
1512      if (Res)
1513        return Res;
1514    } else {
1515      SDNode *Res = SelectARMCMOVSoImmOp(N, FalseVal, TrueVal,
1516                                         CCVal, CCR, InFlag);
1517      if (!Res)
1518        Res = SelectARMCMOVSoImmOp(N, TrueVal, FalseVal,
1519                               ARMCC::getOppositeCondition(CCVal), CCR, InFlag);
1520      if (Res)
1521        return Res;
1522    }
1523  }
1524
1525  // Pattern: (ARMcmov:i32 GPR:i32:$false, GPR:i32:$true, (imm:i32):$cc)
1526  // Emits: (MOVCCr:i32 GPR:i32:$false, GPR:i32:$true, (imm:i32):$cc)
1527  // Pattern complexity = 6  cost = 1  size = 0
1528  //
1529  // Pattern: (ARMcmov:i32 GPR:i32:$false, GPR:i32:$true, (imm:i32):$cc)
1530  // Emits: (tMOVCCr:i32 GPR:i32:$false, GPR:i32:$true, (imm:i32):$cc)
1531  // Pattern complexity = 6  cost = 11  size = 0
1532  //
1533  // Also FCPYScc and FCPYDcc.
1534  SDValue Tmp2 = CurDAG->getTargetConstant(CCVal, MVT::i32);
1535  SDValue Ops[] = { FalseVal, TrueVal, Tmp2, CCR, InFlag };
1536  unsigned Opc = 0;
1537  switch (VT.getSimpleVT().SimpleTy) {
1538  default: assert(false && "Illegal conditional move type!");
1539    break;
1540  case MVT::i32:
1541    Opc = Subtarget->isThumb()
1542      ? (Subtarget->hasThumb2() ? ARM::t2MOVCCr : ARM::tMOVCCr_pseudo)
1543      : ARM::MOVCCr;
1544    break;
1545  case MVT::f32:
1546    Opc = ARM::VMOVScc;
1547    break;
1548  case MVT::f64:
1549    Opc = ARM::VMOVDcc;
1550    break;
1551  }
1552  return CurDAG->SelectNodeTo(N, Opc, VT, Ops, 5);
1553}
1554
1555SDNode *ARMDAGToDAGISel::SelectConcatVector(SDNode *N) {
1556  // The only time a CONCAT_VECTORS operation can have legal types is when
1557  // two 64-bit vectors are concatenated to a 128-bit vector.
1558  EVT VT = N->getValueType(0);
1559  if (!VT.is128BitVector() || N->getNumOperands() != 2)
1560    llvm_unreachable("unexpected CONCAT_VECTORS");
1561  DebugLoc dl = N->getDebugLoc();
1562  SDValue V0 = N->getOperand(0);
1563  SDValue V1 = N->getOperand(1);
1564  SDValue SubReg0 = CurDAG->getTargetConstant(ARM::DSUBREG_0, MVT::i32);
1565  SDValue SubReg1 = CurDAG->getTargetConstant(ARM::DSUBREG_1, MVT::i32);
1566  const SDValue Ops[] = { V0, SubReg0, V1, SubReg1 };
1567  return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops, 4);
1568}
1569
1570SDNode *ARMDAGToDAGISel::Select(SDNode *N) {
1571  DebugLoc dl = N->getDebugLoc();
1572
1573  if (N->isMachineOpcode())
1574    return NULL;   // Already selected.
1575
1576  switch (N->getOpcode()) {
1577  default: break;
1578  case ISD::Constant: {
1579    unsigned Val = cast<ConstantSDNode>(N)->getZExtValue();
1580    bool UseCP = true;
1581    if (Subtarget->hasThumb2())
1582      // Thumb2-aware targets have the MOVT instruction, so all immediates can
1583      // be done with MOV + MOVT, at worst.
1584      UseCP = 0;
1585    else {
1586      if (Subtarget->isThumb()) {
1587        UseCP = (Val > 255 &&                          // MOV
1588                 ~Val > 255 &&                         // MOV + MVN
1589                 !ARM_AM::isThumbImmShiftedVal(Val));  // MOV + LSL
1590      } else
1591        UseCP = (ARM_AM::getSOImmVal(Val) == -1 &&     // MOV
1592                 ARM_AM::getSOImmVal(~Val) == -1 &&    // MVN
1593                 !ARM_AM::isSOImmTwoPartVal(Val));     // two instrs.
1594    }
1595
1596    if (UseCP) {
1597      SDValue CPIdx =
1598        CurDAG->getTargetConstantPool(ConstantInt::get(
1599                                  Type::getInt32Ty(*CurDAG->getContext()), Val),
1600                                      TLI.getPointerTy());
1601
1602      SDNode *ResNode;
1603      if (Subtarget->isThumb1Only()) {
1604        SDValue Pred = getAL(CurDAG);
1605        SDValue PredReg = CurDAG->getRegister(0, MVT::i32);
1606        SDValue Ops[] = { CPIdx, Pred, PredReg, CurDAG->getEntryNode() };
1607        ResNode = CurDAG->getMachineNode(ARM::tLDRcp, dl, MVT::i32, MVT::Other,
1608                                         Ops, 4);
1609      } else {
1610        SDValue Ops[] = {
1611          CPIdx,
1612          CurDAG->getRegister(0, MVT::i32),
1613          CurDAG->getTargetConstant(0, MVT::i32),
1614          getAL(CurDAG),
1615          CurDAG->getRegister(0, MVT::i32),
1616          CurDAG->getEntryNode()
1617        };
1618        ResNode=CurDAG->getMachineNode(ARM::LDRcp, dl, MVT::i32, MVT::Other,
1619                                       Ops, 6);
1620      }
1621      ReplaceUses(SDValue(N, 0), SDValue(ResNode, 0));
1622      return NULL;
1623    }
1624
1625    // Other cases are autogenerated.
1626    break;
1627  }
1628  case ISD::FrameIndex: {
1629    // Selects to ADDri FI, 0 which in turn will become ADDri SP, imm.
1630    int FI = cast<FrameIndexSDNode>(N)->getIndex();
1631    SDValue TFI = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
1632    if (Subtarget->isThumb1Only()) {
1633      return CurDAG->SelectNodeTo(N, ARM::tADDrSPi, MVT::i32, TFI,
1634                                  CurDAG->getTargetConstant(0, MVT::i32));
1635    } else {
1636      unsigned Opc = ((Subtarget->isThumb() && Subtarget->hasThumb2()) ?
1637                      ARM::t2ADDri : ARM::ADDri);
1638      SDValue Ops[] = { TFI, CurDAG->getTargetConstant(0, MVT::i32),
1639                        getAL(CurDAG), CurDAG->getRegister(0, MVT::i32),
1640                        CurDAG->getRegister(0, MVT::i32) };
1641      return CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops, 5);
1642    }
1643  }
1644  case ISD::SRL:
1645    if (SDNode *I = SelectV6T2BitfieldExtractOp(N, false))
1646      return I;
1647    break;
1648  case ISD::SRA:
1649    if (SDNode *I = SelectV6T2BitfieldExtractOp(N, true))
1650      return I;
1651    break;
1652  case ISD::MUL:
1653    if (Subtarget->isThumb1Only())
1654      break;
1655    if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(N->getOperand(1))) {
1656      unsigned RHSV = C->getZExtValue();
1657      if (!RHSV) break;
1658      if (isPowerOf2_32(RHSV-1)) {  // 2^n+1?
1659        unsigned ShImm = Log2_32(RHSV-1);
1660        if (ShImm >= 32)
1661          break;
1662        SDValue V = N->getOperand(0);
1663        ShImm = ARM_AM::getSORegOpc(ARM_AM::lsl, ShImm);
1664        SDValue ShImmOp = CurDAG->getTargetConstant(ShImm, MVT::i32);
1665        SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
1666        if (Subtarget->isThumb()) {
1667          SDValue Ops[] = { V, V, ShImmOp, getAL(CurDAG), Reg0, Reg0 };
1668          return CurDAG->SelectNodeTo(N, ARM::t2ADDrs, MVT::i32, Ops, 6);
1669        } else {
1670          SDValue Ops[] = { V, V, Reg0, ShImmOp, getAL(CurDAG), Reg0, Reg0 };
1671          return CurDAG->SelectNodeTo(N, ARM::ADDrs, MVT::i32, Ops, 7);
1672        }
1673      }
1674      if (isPowerOf2_32(RHSV+1)) {  // 2^n-1?
1675        unsigned ShImm = Log2_32(RHSV+1);
1676        if (ShImm >= 32)
1677          break;
1678        SDValue V = N->getOperand(0);
1679        ShImm = ARM_AM::getSORegOpc(ARM_AM::lsl, ShImm);
1680        SDValue ShImmOp = CurDAG->getTargetConstant(ShImm, MVT::i32);
1681        SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
1682        if (Subtarget->isThumb()) {
1683          SDValue Ops[] = { V, V, ShImmOp, getAL(CurDAG), Reg0 };
1684          return CurDAG->SelectNodeTo(N, ARM::t2RSBrs, MVT::i32, Ops, 5);
1685        } else {
1686          SDValue Ops[] = { V, V, Reg0, ShImmOp, getAL(CurDAG), Reg0, Reg0 };
1687          return CurDAG->SelectNodeTo(N, ARM::RSBrs, MVT::i32, Ops, 7);
1688        }
1689      }
1690    }
1691    break;
1692  case ISD::AND: {
1693    // Check for unsigned bitfield extract
1694    if (SDNode *I = SelectV6T2BitfieldExtractOp(N, false))
1695      return I;
1696
1697    // (and (or x, c2), c1) and top 16-bits of c1 and c2 match, lower 16-bits
1698    // of c1 are 0xffff, and lower 16-bit of c2 are 0. That is, the top 16-bits
1699    // are entirely contributed by c2 and lower 16-bits are entirely contributed
1700    // by x. That's equal to (or (and x, 0xffff), (and c1, 0xffff0000)).
1701    // Select it to: "movt x, ((c1 & 0xffff) >> 16)
1702    EVT VT = N->getValueType(0);
1703    if (VT != MVT::i32)
1704      break;
1705    unsigned Opc = (Subtarget->isThumb() && Subtarget->hasThumb2())
1706      ? ARM::t2MOVTi16
1707      : (Subtarget->hasV6T2Ops() ? ARM::MOVTi16 : 0);
1708    if (!Opc)
1709      break;
1710    SDValue N0 = N->getOperand(0), N1 = N->getOperand(1);
1711    ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
1712    if (!N1C)
1713      break;
1714    if (N0.getOpcode() == ISD::OR && N0.getNode()->hasOneUse()) {
1715      SDValue N2 = N0.getOperand(1);
1716      ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(N2);
1717      if (!N2C)
1718        break;
1719      unsigned N1CVal = N1C->getZExtValue();
1720      unsigned N2CVal = N2C->getZExtValue();
1721      if ((N1CVal & 0xffff0000U) == (N2CVal & 0xffff0000U) &&
1722          (N1CVal & 0xffffU) == 0xffffU &&
1723          (N2CVal & 0xffffU) == 0x0U) {
1724        SDValue Imm16 = CurDAG->getTargetConstant((N2CVal & 0xFFFF0000U) >> 16,
1725                                                  MVT::i32);
1726        SDValue Ops[] = { N0.getOperand(0), Imm16,
1727                          getAL(CurDAG), CurDAG->getRegister(0, MVT::i32) };
1728        return CurDAG->getMachineNode(Opc, dl, VT, Ops, 4);
1729      }
1730    }
1731    break;
1732  }
1733  case ARMISD::VMOVRRD:
1734    return CurDAG->getMachineNode(ARM::VMOVRRD, dl, MVT::i32, MVT::i32,
1735                                  N->getOperand(0), getAL(CurDAG),
1736                                  CurDAG->getRegister(0, MVT::i32));
1737  case ISD::UMUL_LOHI: {
1738    if (Subtarget->isThumb1Only())
1739      break;
1740    if (Subtarget->isThumb()) {
1741      SDValue Ops[] = { N->getOperand(0), N->getOperand(1),
1742                        getAL(CurDAG), CurDAG->getRegister(0, MVT::i32),
1743                        CurDAG->getRegister(0, MVT::i32) };
1744      return CurDAG->getMachineNode(ARM::t2UMULL, dl, MVT::i32, MVT::i32, Ops,4);
1745    } else {
1746      SDValue Ops[] = { N->getOperand(0), N->getOperand(1),
1747                        getAL(CurDAG), CurDAG->getRegister(0, MVT::i32),
1748                        CurDAG->getRegister(0, MVT::i32) };
1749      return CurDAG->getMachineNode(ARM::UMULL, dl, MVT::i32, MVT::i32, Ops, 5);
1750    }
1751  }
1752  case ISD::SMUL_LOHI: {
1753    if (Subtarget->isThumb1Only())
1754      break;
1755    if (Subtarget->isThumb()) {
1756      SDValue Ops[] = { N->getOperand(0), N->getOperand(1),
1757                        getAL(CurDAG), CurDAG->getRegister(0, MVT::i32) };
1758      return CurDAG->getMachineNode(ARM::t2SMULL, dl, MVT::i32, MVT::i32, Ops,4);
1759    } else {
1760      SDValue Ops[] = { N->getOperand(0), N->getOperand(1),
1761                        getAL(CurDAG), CurDAG->getRegister(0, MVT::i32),
1762                        CurDAG->getRegister(0, MVT::i32) };
1763      return CurDAG->getMachineNode(ARM::SMULL, dl, MVT::i32, MVT::i32, Ops, 5);
1764    }
1765  }
1766  case ISD::LOAD: {
1767    SDNode *ResNode = 0;
1768    if (Subtarget->isThumb() && Subtarget->hasThumb2())
1769      ResNode = SelectT2IndexedLoad(N);
1770    else
1771      ResNode = SelectARMIndexedLoad(N);
1772    if (ResNode)
1773      return ResNode;
1774
1775    // VLDMQ must be custom-selected for "v2f64 load" to set the AM5Opc value.
1776    if (Subtarget->hasVFP2() &&
1777        N->getValueType(0).getSimpleVT().SimpleTy == MVT::v2f64) {
1778      SDValue Chain = N->getOperand(0);
1779      SDValue AM5Opc =
1780        CurDAG->getTargetConstant(ARM_AM::getAM5Opc(ARM_AM::ia, 4), MVT::i32);
1781      SDValue Pred = getAL(CurDAG);
1782      SDValue PredReg = CurDAG->getRegister(0, MVT::i32);
1783      SDValue Ops[] = { N->getOperand(1), AM5Opc, Pred, PredReg, Chain };
1784      return CurDAG->getMachineNode(ARM::VLDMQ, dl, MVT::v2f64, MVT::Other,
1785                                    Ops, 5);
1786    }
1787    // Other cases are autogenerated.
1788    break;
1789  }
1790  case ISD::STORE: {
1791    // VSTMQ must be custom-selected for "v2f64 store" to set the AM5Opc value.
1792    if (Subtarget->hasVFP2() &&
1793        N->getOperand(1).getValueType().getSimpleVT().SimpleTy == MVT::v2f64) {
1794      SDValue Chain = N->getOperand(0);
1795      SDValue AM5Opc =
1796        CurDAG->getTargetConstant(ARM_AM::getAM5Opc(ARM_AM::ia, 4), MVT::i32);
1797      SDValue Pred = getAL(CurDAG);
1798      SDValue PredReg = CurDAG->getRegister(0, MVT::i32);
1799      SDValue Ops[] = { N->getOperand(1), N->getOperand(2),
1800                        AM5Opc, Pred, PredReg, Chain };
1801      return CurDAG->getMachineNode(ARM::VSTMQ, dl, MVT::Other, Ops, 6);
1802    }
1803    // Other cases are autogenerated.
1804    break;
1805  }
1806  case ARMISD::BRCOND: {
1807    // Pattern: (ARMbrcond:void (bb:Other):$dst, (imm:i32):$cc)
1808    // Emits: (Bcc:void (bb:Other):$dst, (imm:i32):$cc)
1809    // Pattern complexity = 6  cost = 1  size = 0
1810
1811    // Pattern: (ARMbrcond:void (bb:Other):$dst, (imm:i32):$cc)
1812    // Emits: (tBcc:void (bb:Other):$dst, (imm:i32):$cc)
1813    // Pattern complexity = 6  cost = 1  size = 0
1814
1815    // Pattern: (ARMbrcond:void (bb:Other):$dst, (imm:i32):$cc)
1816    // Emits: (t2Bcc:void (bb:Other):$dst, (imm:i32):$cc)
1817    // Pattern complexity = 6  cost = 1  size = 0
1818
1819    unsigned Opc = Subtarget->isThumb() ?
1820      ((Subtarget->hasThumb2()) ? ARM::t2Bcc : ARM::tBcc) : ARM::Bcc;
1821    SDValue Chain = N->getOperand(0);
1822    SDValue N1 = N->getOperand(1);
1823    SDValue N2 = N->getOperand(2);
1824    SDValue N3 = N->getOperand(3);
1825    SDValue InFlag = N->getOperand(4);
1826    assert(N1.getOpcode() == ISD::BasicBlock);
1827    assert(N2.getOpcode() == ISD::Constant);
1828    assert(N3.getOpcode() == ISD::Register);
1829
1830    SDValue Tmp2 = CurDAG->getTargetConstant(((unsigned)
1831                               cast<ConstantSDNode>(N2)->getZExtValue()),
1832                               MVT::i32);
1833    SDValue Ops[] = { N1, Tmp2, N3, Chain, InFlag };
1834    SDNode *ResNode = CurDAG->getMachineNode(Opc, dl, MVT::Other,
1835                                             MVT::Flag, Ops, 5);
1836    Chain = SDValue(ResNode, 0);
1837    if (N->getNumValues() == 2) {
1838      InFlag = SDValue(ResNode, 1);
1839      ReplaceUses(SDValue(N, 1), InFlag);
1840    }
1841    ReplaceUses(SDValue(N, 0),
1842                SDValue(Chain.getNode(), Chain.getResNo()));
1843    return NULL;
1844  }
1845  case ARMISD::CMOV:
1846    return SelectCMOVOp(N);
1847  case ARMISD::CNEG: {
1848    EVT VT = N->getValueType(0);
1849    SDValue N0 = N->getOperand(0);
1850    SDValue N1 = N->getOperand(1);
1851    SDValue N2 = N->getOperand(2);
1852    SDValue N3 = N->getOperand(3);
1853    SDValue InFlag = N->getOperand(4);
1854    assert(N2.getOpcode() == ISD::Constant);
1855    assert(N3.getOpcode() == ISD::Register);
1856
1857    SDValue Tmp2 = CurDAG->getTargetConstant(((unsigned)
1858                               cast<ConstantSDNode>(N2)->getZExtValue()),
1859                               MVT::i32);
1860    SDValue Ops[] = { N0, N1, Tmp2, N3, InFlag };
1861    unsigned Opc = 0;
1862    switch (VT.getSimpleVT().SimpleTy) {
1863    default: assert(false && "Illegal conditional move type!");
1864      break;
1865    case MVT::f32:
1866      Opc = ARM::VNEGScc;
1867      break;
1868    case MVT::f64:
1869      Opc = ARM::VNEGDcc;
1870      break;
1871    }
1872    return CurDAG->SelectNodeTo(N, Opc, VT, Ops, 5);
1873  }
1874
1875  case ARMISD::VZIP: {
1876    unsigned Opc = 0;
1877    EVT VT = N->getValueType(0);
1878    switch (VT.getSimpleVT().SimpleTy) {
1879    default: return NULL;
1880    case MVT::v8i8:  Opc = ARM::VZIPd8; break;
1881    case MVT::v4i16: Opc = ARM::VZIPd16; break;
1882    case MVT::v2f32:
1883    case MVT::v2i32: Opc = ARM::VZIPd32; break;
1884    case MVT::v16i8: Opc = ARM::VZIPq8; break;
1885    case MVT::v8i16: Opc = ARM::VZIPq16; break;
1886    case MVT::v4f32:
1887    case MVT::v4i32: Opc = ARM::VZIPq32; break;
1888    }
1889    SDValue Pred = getAL(CurDAG);
1890    SDValue PredReg = CurDAG->getRegister(0, MVT::i32);
1891    SDValue Ops[] = { N->getOperand(0), N->getOperand(1), Pred, PredReg };
1892    return CurDAG->getMachineNode(Opc, dl, VT, VT, Ops, 4);
1893  }
1894  case ARMISD::VUZP: {
1895    unsigned Opc = 0;
1896    EVT VT = N->getValueType(0);
1897    switch (VT.getSimpleVT().SimpleTy) {
1898    default: return NULL;
1899    case MVT::v8i8:  Opc = ARM::VUZPd8; break;
1900    case MVT::v4i16: Opc = ARM::VUZPd16; break;
1901    case MVT::v2f32:
1902    case MVT::v2i32: Opc = ARM::VUZPd32; break;
1903    case MVT::v16i8: Opc = ARM::VUZPq8; break;
1904    case MVT::v8i16: Opc = ARM::VUZPq16; break;
1905    case MVT::v4f32:
1906    case MVT::v4i32: Opc = ARM::VUZPq32; break;
1907    }
1908    SDValue Pred = getAL(CurDAG);
1909    SDValue PredReg = CurDAG->getRegister(0, MVT::i32);
1910    SDValue Ops[] = { N->getOperand(0), N->getOperand(1), Pred, PredReg };
1911    return CurDAG->getMachineNode(Opc, dl, VT, VT, Ops, 4);
1912  }
1913  case ARMISD::VTRN: {
1914    unsigned Opc = 0;
1915    EVT VT = N->getValueType(0);
1916    switch (VT.getSimpleVT().SimpleTy) {
1917    default: return NULL;
1918    case MVT::v8i8:  Opc = ARM::VTRNd8; break;
1919    case MVT::v4i16: Opc = ARM::VTRNd16; break;
1920    case MVT::v2f32:
1921    case MVT::v2i32: Opc = ARM::VTRNd32; break;
1922    case MVT::v16i8: Opc = ARM::VTRNq8; break;
1923    case MVT::v8i16: Opc = ARM::VTRNq16; break;
1924    case MVT::v4f32:
1925    case MVT::v4i32: Opc = ARM::VTRNq32; break;
1926    }
1927    SDValue Pred = getAL(CurDAG);
1928    SDValue PredReg = CurDAG->getRegister(0, MVT::i32);
1929    SDValue Ops[] = { N->getOperand(0), N->getOperand(1), Pred, PredReg };
1930    return CurDAG->getMachineNode(Opc, dl, VT, VT, Ops, 4);
1931  }
1932
1933  case ISD::INTRINSIC_VOID:
1934  case ISD::INTRINSIC_W_CHAIN: {
1935    unsigned IntNo = cast<ConstantSDNode>(N->getOperand(1))->getZExtValue();
1936    switch (IntNo) {
1937    default:
1938      break;
1939
1940    case Intrinsic::arm_neon_vld1: {
1941      unsigned DOpcodes[] = { ARM::VLD1d8, ARM::VLD1d16,
1942                              ARM::VLD1d32, ARM::VLD1d64 };
1943      unsigned QOpcodes[] = { ARM::VLD1q8, ARM::VLD1q16,
1944                              ARM::VLD1q32, ARM::VLD1q64 };
1945      return SelectVLD(N, 1, DOpcodes, QOpcodes, 0);
1946    }
1947
1948    case Intrinsic::arm_neon_vld2: {
1949      unsigned DOpcodes[] = { ARM::VLD2d8, ARM::VLD2d16,
1950                              ARM::VLD2d32, ARM::VLD1q64 };
1951      unsigned QOpcodes[] = { ARM::VLD2q8, ARM::VLD2q16, ARM::VLD2q32 };
1952      return SelectVLD(N, 2, DOpcodes, QOpcodes, 0);
1953    }
1954
1955    case Intrinsic::arm_neon_vld3: {
1956      unsigned DOpcodes[] = { ARM::VLD3d8, ARM::VLD3d16,
1957                              ARM::VLD3d32, ARM::VLD1d64T };
1958      unsigned QOpcodes0[] = { ARM::VLD3q8_UPD,
1959                               ARM::VLD3q16_UPD,
1960                               ARM::VLD3q32_UPD };
1961      unsigned QOpcodes1[] = { ARM::VLD3q8odd_UPD,
1962                               ARM::VLD3q16odd_UPD,
1963                               ARM::VLD3q32odd_UPD };
1964      return SelectVLD(N, 3, DOpcodes, QOpcodes0, QOpcodes1);
1965    }
1966
1967    case Intrinsic::arm_neon_vld4: {
1968      unsigned DOpcodes[] = { ARM::VLD4d8, ARM::VLD4d16,
1969                              ARM::VLD4d32, ARM::VLD1d64Q };
1970      unsigned QOpcodes0[] = { ARM::VLD4q8_UPD,
1971                               ARM::VLD4q16_UPD,
1972                               ARM::VLD4q32_UPD };
1973      unsigned QOpcodes1[] = { ARM::VLD4q8odd_UPD,
1974                               ARM::VLD4q16odd_UPD,
1975                               ARM::VLD4q32odd_UPD };
1976      return SelectVLD(N, 4, DOpcodes, QOpcodes0, QOpcodes1);
1977    }
1978
1979    case Intrinsic::arm_neon_vld2lane: {
1980      unsigned DOpcodes[] = { ARM::VLD2LNd8, ARM::VLD2LNd16, ARM::VLD2LNd32 };
1981      unsigned QOpcodes0[] = { ARM::VLD2LNq16, ARM::VLD2LNq32 };
1982      unsigned QOpcodes1[] = { ARM::VLD2LNq16odd, ARM::VLD2LNq32odd };
1983      return SelectVLDSTLane(N, true, 2, DOpcodes, QOpcodes0, QOpcodes1);
1984    }
1985
1986    case Intrinsic::arm_neon_vld3lane: {
1987      unsigned DOpcodes[] = { ARM::VLD3LNd8, ARM::VLD3LNd16, ARM::VLD3LNd32 };
1988      unsigned QOpcodes0[] = { ARM::VLD3LNq16, ARM::VLD3LNq32 };
1989      unsigned QOpcodes1[] = { ARM::VLD3LNq16odd, ARM::VLD3LNq32odd };
1990      return SelectVLDSTLane(N, true, 3, DOpcodes, QOpcodes0, QOpcodes1);
1991    }
1992
1993    case Intrinsic::arm_neon_vld4lane: {
1994      unsigned DOpcodes[] = { ARM::VLD4LNd8, ARM::VLD4LNd16, ARM::VLD4LNd32 };
1995      unsigned QOpcodes0[] = { ARM::VLD4LNq16, ARM::VLD4LNq32 };
1996      unsigned QOpcodes1[] = { ARM::VLD4LNq16odd, ARM::VLD4LNq32odd };
1997      return SelectVLDSTLane(N, true, 4, DOpcodes, QOpcodes0, QOpcodes1);
1998    }
1999
2000    case Intrinsic::arm_neon_vst1: {
2001      unsigned DOpcodes[] = { ARM::VST1d8, ARM::VST1d16,
2002                              ARM::VST1d32, ARM::VST1d64 };
2003      unsigned QOpcodes[] = { ARM::VST1q8, ARM::VST1q16,
2004                              ARM::VST1q32, ARM::VST1q64 };
2005      return SelectVST(N, 1, DOpcodes, QOpcodes, 0);
2006    }
2007
2008    case Intrinsic::arm_neon_vst2: {
2009      unsigned DOpcodes[] = { ARM::VST2d8, ARM::VST2d16,
2010                              ARM::VST2d32, ARM::VST1q64 };
2011      unsigned QOpcodes[] = { ARM::VST2q8, ARM::VST2q16, ARM::VST2q32 };
2012      return SelectVST(N, 2, DOpcodes, QOpcodes, 0);
2013    }
2014
2015    case Intrinsic::arm_neon_vst3: {
2016      unsigned DOpcodes[] = { ARM::VST3d8, ARM::VST3d16,
2017                              ARM::VST3d32, ARM::VST1d64T };
2018      unsigned QOpcodes0[] = { ARM::VST3q8_UPD,
2019                               ARM::VST3q16_UPD,
2020                               ARM::VST3q32_UPD };
2021      unsigned QOpcodes1[] = { ARM::VST3q8odd_UPD,
2022                               ARM::VST3q16odd_UPD,
2023                               ARM::VST3q32odd_UPD };
2024      return SelectVST(N, 3, DOpcodes, QOpcodes0, QOpcodes1);
2025    }
2026
2027    case Intrinsic::arm_neon_vst4: {
2028      unsigned DOpcodes[] = { ARM::VST4d8, ARM::VST4d16,
2029                              ARM::VST4d32, ARM::VST1d64Q };
2030      unsigned QOpcodes0[] = { ARM::VST4q8_UPD,
2031                               ARM::VST4q16_UPD,
2032                               ARM::VST4q32_UPD };
2033      unsigned QOpcodes1[] = { ARM::VST4q8odd_UPD,
2034                               ARM::VST4q16odd_UPD,
2035                               ARM::VST4q32odd_UPD };
2036      return SelectVST(N, 4, DOpcodes, QOpcodes0, QOpcodes1);
2037    }
2038
2039    case Intrinsic::arm_neon_vst2lane: {
2040      unsigned DOpcodes[] = { ARM::VST2LNd8, ARM::VST2LNd16, ARM::VST2LNd32 };
2041      unsigned QOpcodes0[] = { ARM::VST2LNq16, ARM::VST2LNq32 };
2042      unsigned QOpcodes1[] = { ARM::VST2LNq16odd, ARM::VST2LNq32odd };
2043      return SelectVLDSTLane(N, false, 2, DOpcodes, QOpcodes0, QOpcodes1);
2044    }
2045
2046    case Intrinsic::arm_neon_vst3lane: {
2047      unsigned DOpcodes[] = { ARM::VST3LNd8, ARM::VST3LNd16, ARM::VST3LNd32 };
2048      unsigned QOpcodes0[] = { ARM::VST3LNq16, ARM::VST3LNq32 };
2049      unsigned QOpcodes1[] = { ARM::VST3LNq16odd, ARM::VST3LNq32odd };
2050      return SelectVLDSTLane(N, false, 3, DOpcodes, QOpcodes0, QOpcodes1);
2051    }
2052
2053    case Intrinsic::arm_neon_vst4lane: {
2054      unsigned DOpcodes[] = { ARM::VST4LNd8, ARM::VST4LNd16, ARM::VST4LNd32 };
2055      unsigned QOpcodes0[] = { ARM::VST4LNq16, ARM::VST4LNq32 };
2056      unsigned QOpcodes1[] = { ARM::VST4LNq16odd, ARM::VST4LNq32odd };
2057      return SelectVLDSTLane(N, false, 4, DOpcodes, QOpcodes0, QOpcodes1);
2058    }
2059    }
2060    break;
2061  }
2062
2063  case ISD::CONCAT_VECTORS:
2064    return SelectConcatVector(N);
2065  }
2066
2067  return SelectCode(N);
2068}
2069
2070bool ARMDAGToDAGISel::
2071SelectInlineAsmMemoryOperand(const SDValue &Op, char ConstraintCode,
2072                             std::vector<SDValue> &OutOps) {
2073  assert(ConstraintCode == 'm' && "unexpected asm memory constraint");
2074  // Require the address to be in a register.  That is safe for all ARM
2075  // variants and it is hard to do anything much smarter without knowing
2076  // how the operand is used.
2077  OutOps.push_back(Op);
2078  return false;
2079}
2080
2081/// createARMISelDag - This pass converts a legalized DAG into a
2082/// ARM-specific DAG, ready for instruction scheduling.
2083///
2084FunctionPass *llvm::createARMISelDag(ARMBaseTargetMachine &TM,
2085                                     CodeGenOpt::Level OptLevel) {
2086  return new ARMDAGToDAGISel(TM, OptLevel);
2087}
2088
2089/// ModelWithRegSequence - Return true if isel should use REG_SEQUENCE to model
2090/// operations involving sub-registers.
2091bool llvm::ModelWithRegSequence() {
2092  return UseRegSeq;
2093}
2094