ARMISelDAGToDAG.cpp revision de8aa4ed9c8d3654e08eda3973e0500ddc7ac0fd 
1//===-- ARMISelDAGToDAG.cpp - A dag to dag inst selector for ARM ----------===//
2//
3//                     The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10// This file defines an instruction selector for the ARM target.
11//
12//===----------------------------------------------------------------------===//
13
14#include "ARM.h"
15#include "ARMAddressingModes.h"
16#include "ARMTargetMachine.h"
17#include "llvm/CallingConv.h"
18#include "llvm/Constants.h"
19#include "llvm/DerivedTypes.h"
20#include "llvm/Function.h"
21#include "llvm/Intrinsics.h"
22#include "llvm/LLVMContext.h"
23#include "llvm/CodeGen/MachineFrameInfo.h"
24#include "llvm/CodeGen/MachineFunction.h"
25#include "llvm/CodeGen/MachineInstrBuilder.h"
26#include "llvm/CodeGen/SelectionDAG.h"
27#include "llvm/CodeGen/SelectionDAGISel.h"
28#include "llvm/Target/TargetLowering.h"
29#include "llvm/Target/TargetOptions.h"
30#include "llvm/Support/CommandLine.h"
31#include "llvm/Support/Compiler.h"
32#include "llvm/Support/Debug.h"
33#include "llvm/Support/ErrorHandling.h"
34#include "llvm/Support/raw_ostream.h"
35
36using namespace llvm;
37
38static cl::opt<bool>
39UseRegSeq("neon-reg-sequence", cl::Hidden,
40          cl::desc("Use reg_sequence to model ld / st of multiple neon regs"));
41
42//===--------------------------------------------------------------------===//
43/// ARMDAGToDAGISel - ARM specific code to select ARM machine
44/// instructions for SelectionDAG operations.
45///
46namespace {
47class ARMDAGToDAGISel : public SelectionDAGISel {
48  ARMBaseTargetMachine &TM;
49
50  /// Subtarget - Keep a pointer to the ARMSubtarget around so that we can
51  /// make the right decision when generating code for different targets.
52  const ARMSubtarget *Subtarget;
53
54public:
55  explicit ARMDAGToDAGISel(ARMBaseTargetMachine &tm,
56                           CodeGenOpt::Level OptLevel)
57    : SelectionDAGISel(tm, OptLevel), TM(tm),
58    Subtarget(&TM.getSubtarget<ARMSubtarget>()) {
59  }
60
61  virtual const char *getPassName() const {
62    return "ARM Instruction Selection";
63  }
64
65  /// getI32Imm - Return a target constant of type i32 with the specified
66  /// value.
67  inline SDValue getI32Imm(unsigned Imm) {
68    return CurDAG->getTargetConstant(Imm, MVT::i32);
69  }
70
71  SDNode *Select(SDNode *N);
72
73  bool SelectShifterOperandReg(SDNode *Op, SDValue N, SDValue &A,
74                               SDValue &B, SDValue &C);
75  bool SelectAddrMode2(SDNode *Op, SDValue N, SDValue &Base,
76                       SDValue &Offset, SDValue &Opc);
77  bool SelectAddrMode2Offset(SDNode *Op, SDValue N,
78                             SDValue &Offset, SDValue &Opc);
79  bool SelectAddrMode3(SDNode *Op, SDValue N, SDValue &Base,
80                       SDValue &Offset, SDValue &Opc);
81  bool SelectAddrMode3Offset(SDNode *Op, SDValue N,
82                             SDValue &Offset, SDValue &Opc);
83  bool SelectAddrMode4(SDNode *Op, SDValue N, SDValue &Addr,
84                       SDValue &Mode);
85  bool SelectAddrMode5(SDNode *Op, SDValue N, SDValue &Base,
86                       SDValue &Offset);
87  bool SelectAddrMode6(SDNode *Op, SDValue N, SDValue &Addr, SDValue &Align);
88
89  bool SelectAddrModePC(SDNode *Op, SDValue N, SDValue &Offset,
90                        SDValue &Label);
91
92  bool SelectThumbAddrModeRR(SDNode *Op, SDValue N, SDValue &Base,
93                             SDValue &Offset);
94  bool SelectThumbAddrModeRI5(SDNode *Op, SDValue N, unsigned Scale,
95                              SDValue &Base, SDValue &OffImm,
96                              SDValue &Offset);
97  bool SelectThumbAddrModeS1(SDNode *Op, SDValue N, SDValue &Base,
98                             SDValue &OffImm, SDValue &Offset);
99  bool SelectThumbAddrModeS2(SDNode *Op, SDValue N, SDValue &Base,
100                             SDValue &OffImm, SDValue &Offset);
101  bool SelectThumbAddrModeS4(SDNode *Op, SDValue N, SDValue &Base,
102                             SDValue &OffImm, SDValue &Offset);
103  bool SelectThumbAddrModeSP(SDNode *Op, SDValue N, SDValue &Base,
104                             SDValue &OffImm);
105
106  bool SelectT2ShifterOperandReg(SDNode *Op, SDValue N,
107                                 SDValue &BaseReg, SDValue &Opc);
108  bool SelectT2AddrModeImm12(SDNode *Op, SDValue N, SDValue &Base,
109                             SDValue &OffImm);
110  bool SelectT2AddrModeImm8(SDNode *Op, SDValue N, SDValue &Base,
111                            SDValue &OffImm);
112  bool SelectT2AddrModeImm8Offset(SDNode *Op, SDValue N,
113                                 SDValue &OffImm);
114  bool SelectT2AddrModeImm8s4(SDNode *Op, SDValue N, SDValue &Base,
115                              SDValue &OffImm);
116  bool SelectT2AddrModeSoReg(SDNode *Op, SDValue N, SDValue &Base,
117                             SDValue &OffReg, SDValue &ShImm);
118
119  // Include the pieces autogenerated from the target description.
120#include "ARMGenDAGISel.inc"
121
122private:
123  /// SelectARMIndexedLoad - Indexed (pre/post inc/dec) load matching code for
124  /// ARM.
125  SDNode *SelectARMIndexedLoad(SDNode *N);
126  SDNode *SelectT2IndexedLoad(SDNode *N);
127
128  /// SelectVLD - Select NEON load intrinsics.  NumVecs should be
129  /// 1, 2, 3 or 4.  The opcode arrays specify the instructions used for
130  /// loads of D registers and even subregs and odd subregs of Q registers.
131  /// For NumVecs <= 2, QOpcodes1 is not used.
132  SDNode *SelectVLD(SDNode *N, unsigned NumVecs, unsigned *DOpcodes,
133                    unsigned *QOpcodes0, unsigned *QOpcodes1);
134
135  /// SelectVST - Select NEON store intrinsics.  NumVecs should
136  /// be 1, 2, 3 or 4.  The opcode arrays specify the instructions used for
137  /// stores of D registers and even subregs and odd subregs of Q registers.
138  /// For NumVecs <= 2, QOpcodes1 is not used.
139  SDNode *SelectVST(SDNode *N, unsigned NumVecs, unsigned *DOpcodes,
140                    unsigned *QOpcodes0, unsigned *QOpcodes1);
141
142  /// SelectVLDSTLane - Select NEON load/store lane intrinsics.  NumVecs should
143  /// be 2, 3 or 4.  The opcode arrays specify the instructions used for
144  /// load/store of D registers and even subregs and odd subregs of Q registers.
145  SDNode *SelectVLDSTLane(SDNode *N, bool IsLoad, unsigned NumVecs,
146                          unsigned *DOpcodes, unsigned *QOpcodes0,
147                          unsigned *QOpcodes1);
148
149  /// SelectV6T2BitfieldExtractOp - Select SBFX/UBFX instructions for ARM.
150  SDNode *SelectV6T2BitfieldExtractOp(SDNode *N, bool isSigned);
151
152  /// SelectCMOVOp - Select CMOV instructions for ARM.
153  SDNode *SelectCMOVOp(SDNode *N);
154  SDNode *SelectT2CMOVShiftOp(SDNode *N, SDValue FalseVal, SDValue TrueVal,
155                              ARMCC::CondCodes CCVal, SDValue CCR,
156                              SDValue InFlag);
157  SDNode *SelectARMCMOVShiftOp(SDNode *N, SDValue FalseVal, SDValue TrueVal,
158                               ARMCC::CondCodes CCVal, SDValue CCR,
159                               SDValue InFlag);
160  SDNode *SelectT2CMOVSoImmOp(SDNode *N, SDValue FalseVal, SDValue TrueVal,
161                              ARMCC::CondCodes CCVal, SDValue CCR,
162                              SDValue InFlag);
163  SDNode *SelectARMCMOVSoImmOp(SDNode *N, SDValue FalseVal, SDValue TrueVal,
164                               ARMCC::CondCodes CCVal, SDValue CCR,
165                               SDValue InFlag);
166
167  SDNode *SelectConcatVector(SDNode *N);
168
169  /// SelectInlineAsmMemoryOperand - Implement addressing mode selection for
170  /// inline asm expressions.
171  virtual bool SelectInlineAsmMemoryOperand(const SDValue &Op,
172                                            char ConstraintCode,
173                                            std::vector<SDValue> &OutOps);
174
175  /// PairDRegs - Insert a pair of double registers into an implicit def to
176  /// form a quad register.
177  SDNode *PairDRegs(EVT VT, SDValue V0, SDValue V1);
178};
179}
180
181/// isInt32Immediate - This method tests to see if the node is a 32-bit constant
182/// operand. If so Imm will receive the 32-bit value.
183static bool isInt32Immediate(SDNode *N, unsigned &Imm) {
184  if (N->getOpcode() == ISD::Constant && N->getValueType(0) == MVT::i32) {
185    Imm = cast<ConstantSDNode>(N)->getZExtValue();
186    return true;
187  }
188  return false;
189}
190
191// isInt32Immediate - This method tests to see if a constant operand.
192// If so Imm will receive the 32 bit value.
193static bool isInt32Immediate(SDValue N, unsigned &Imm) {
194  return isInt32Immediate(N.getNode(), Imm);
195}
196
197// isOpcWithIntImmediate - This method tests to see if the node is a specific
198// opcode and that it has a immediate integer right operand.
199// If so Imm will receive the 32 bit value.
200static bool isOpcWithIntImmediate(SDNode *N, unsigned Opc, unsigned& Imm) {
201  return N->getOpcode() == Opc &&
202         isInt32Immediate(N->getOperand(1).getNode(), Imm);
203}
204
205
206bool ARMDAGToDAGISel::SelectShifterOperandReg(SDNode *Op,
207                                              SDValue N,
208                                              SDValue &BaseReg,
209                                              SDValue &ShReg,
210                                              SDValue &Opc) {
211  ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(N);
212
213  // Don't match base register only case. That is matched to a separate
214  // lower complexity pattern with explicit register operand.
215  if (ShOpcVal == ARM_AM::no_shift) return false;
216
217  BaseReg = N.getOperand(0);
218  unsigned ShImmVal = 0;
219  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
220    ShReg = CurDAG->getRegister(0, MVT::i32);
221    ShImmVal = RHS->getZExtValue() & 31;
222  } else {
223    ShReg = N.getOperand(1);
224  }
225  Opc = CurDAG->getTargetConstant(ARM_AM::getSORegOpc(ShOpcVal, ShImmVal),
226                                  MVT::i32);
227  return true;
228}
229
230bool ARMDAGToDAGISel::SelectAddrMode2(SDNode *Op, SDValue N,
231                                      SDValue &Base, SDValue &Offset,
232                                      SDValue &Opc) {
233  if (N.getOpcode() == ISD::MUL) {
234    if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
235      // X * [3,5,9] -> X + X * [2,4,8] etc.
236      int RHSC = (int)RHS->getZExtValue();
237      if (RHSC & 1) {
238        RHSC = RHSC & ~1;
239        ARM_AM::AddrOpc AddSub = ARM_AM::add;
240        if (RHSC < 0) {
241          AddSub = ARM_AM::sub;
242          RHSC = - RHSC;
243        }
244        if (isPowerOf2_32(RHSC)) {
245          unsigned ShAmt = Log2_32(RHSC);
246          Base = Offset = N.getOperand(0);
247          Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, ShAmt,
248                                                            ARM_AM::lsl),
249                                          MVT::i32);
250          return true;
251        }
252      }
253    }
254  }
255
256  if (N.getOpcode() != ISD::ADD && N.getOpcode() != ISD::SUB) {
257    Base = N;
258    if (N.getOpcode() == ISD::FrameIndex) {
259      int FI = cast<FrameIndexSDNode>(N)->getIndex();
260      Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
261    } else if (N.getOpcode() == ARMISD::Wrapper &&
262               !(Subtarget->useMovt() &&
263                 N.getOperand(0).getOpcode() == ISD::TargetGlobalAddress)) {
264      Base = N.getOperand(0);
265    }
266    Offset = CurDAG->getRegister(0, MVT::i32);
267    Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(ARM_AM::add, 0,
268                                                      ARM_AM::no_shift),
269                                    MVT::i32);
270    return true;
271  }
272
273  // Match simple R +/- imm12 operands.
274  if (N.getOpcode() == ISD::ADD)
275    if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
276      int RHSC = (int)RHS->getZExtValue();
277      if ((RHSC >= 0 && RHSC < 0x1000) ||
278          (RHSC < 0 && RHSC > -0x1000)) { // 12 bits.
279        Base = N.getOperand(0);
280        if (Base.getOpcode() == ISD::FrameIndex) {
281          int FI = cast<FrameIndexSDNode>(Base)->getIndex();
282          Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
283        }
284        Offset = CurDAG->getRegister(0, MVT::i32);
285
286        ARM_AM::AddrOpc AddSub = ARM_AM::add;
287        if (RHSC < 0) {
288          AddSub = ARM_AM::sub;
289          RHSC = - RHSC;
290        }
291        Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, RHSC,
292                                                          ARM_AM::no_shift),
293                                        MVT::i32);
294        return true;
295      }
296    }
297
298  // Otherwise this is R +/- [possibly shifted] R.
299  ARM_AM::AddrOpc AddSub = N.getOpcode() == ISD::ADD ? ARM_AM::add:ARM_AM::sub;
300  ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(N.getOperand(1));
301  unsigned ShAmt = 0;
302
303  Base   = N.getOperand(0);
304  Offset = N.getOperand(1);
305
306  if (ShOpcVal != ARM_AM::no_shift) {
307    // Check to see if the RHS of the shift is a constant, if not, we can't fold
308    // it.
309    if (ConstantSDNode *Sh =
310           dyn_cast<ConstantSDNode>(N.getOperand(1).getOperand(1))) {
311      ShAmt = Sh->getZExtValue();
312      Offset = N.getOperand(1).getOperand(0);
313    } else {
314      ShOpcVal = ARM_AM::no_shift;
315    }
316  }
317
318  // Try matching (R shl C) + (R).
319  if (N.getOpcode() == ISD::ADD && ShOpcVal == ARM_AM::no_shift) {
320    ShOpcVal = ARM_AM::getShiftOpcForNode(N.getOperand(0));
321    if (ShOpcVal != ARM_AM::no_shift) {
322      // Check to see if the RHS of the shift is a constant, if not, we can't
323      // fold it.
324      if (ConstantSDNode *Sh =
325          dyn_cast<ConstantSDNode>(N.getOperand(0).getOperand(1))) {
326        ShAmt = Sh->getZExtValue();
327        Offset = N.getOperand(0).getOperand(0);
328        Base = N.getOperand(1);
329      } else {
330        ShOpcVal = ARM_AM::no_shift;
331      }
332    }
333  }
334
335  Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, ShAmt, ShOpcVal),
336                                  MVT::i32);
337  return true;
338}
339
340bool ARMDAGToDAGISel::SelectAddrMode2Offset(SDNode *Op, SDValue N,
341                                            SDValue &Offset, SDValue &Opc) {
342  unsigned Opcode = Op->getOpcode();
343  ISD::MemIndexedMode AM = (Opcode == ISD::LOAD)
344    ? cast<LoadSDNode>(Op)->getAddressingMode()
345    : cast<StoreSDNode>(Op)->getAddressingMode();
346  ARM_AM::AddrOpc AddSub = (AM == ISD::PRE_INC || AM == ISD::POST_INC)
347    ? ARM_AM::add : ARM_AM::sub;
348  if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(N)) {
349    int Val = (int)C->getZExtValue();
350    if (Val >= 0 && Val < 0x1000) { // 12 bits.
351      Offset = CurDAG->getRegister(0, MVT::i32);
352      Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, Val,
353                                                        ARM_AM::no_shift),
354                                      MVT::i32);
355      return true;
356    }
357  }
358
359  Offset = N;
360  ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(N);
361  unsigned ShAmt = 0;
362  if (ShOpcVal != ARM_AM::no_shift) {
363    // Check to see if the RHS of the shift is a constant, if not, we can't fold
364    // it.
365    if (ConstantSDNode *Sh = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
366      ShAmt = Sh->getZExtValue();
367      Offset = N.getOperand(0);
368    } else {
369      ShOpcVal = ARM_AM::no_shift;
370    }
371  }
372
373  Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, ShAmt, ShOpcVal),
374                                  MVT::i32);
375  return true;
376}
377
378
379bool ARMDAGToDAGISel::SelectAddrMode3(SDNode *Op, SDValue N,
380                                      SDValue &Base, SDValue &Offset,
381                                      SDValue &Opc) {
382  if (N.getOpcode() == ISD::SUB) {
383    // X - C  is canonicalize to X + -C, no need to handle it here.
384    Base = N.getOperand(0);
385    Offset = N.getOperand(1);
386    Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(ARM_AM::sub, 0),MVT::i32);
387    return true;
388  }
389
390  if (N.getOpcode() != ISD::ADD) {
391    Base = N;
392    if (N.getOpcode() == ISD::FrameIndex) {
393      int FI = cast<FrameIndexSDNode>(N)->getIndex();
394      Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
395    }
396    Offset = CurDAG->getRegister(0, MVT::i32);
397    Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(ARM_AM::add, 0),MVT::i32);
398    return true;
399  }
400
401  // If the RHS is +/- imm8, fold into addr mode.
402  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
403    int RHSC = (int)RHS->getZExtValue();
404    if ((RHSC >= 0 && RHSC < 256) ||
405        (RHSC < 0 && RHSC > -256)) { // note -256 itself isn't allowed.
406      Base = N.getOperand(0);
407      if (Base.getOpcode() == ISD::FrameIndex) {
408        int FI = cast<FrameIndexSDNode>(Base)->getIndex();
409        Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
410      }
411      Offset = CurDAG->getRegister(0, MVT::i32);
412
413      ARM_AM::AddrOpc AddSub = ARM_AM::add;
414      if (RHSC < 0) {
415        AddSub = ARM_AM::sub;
416        RHSC = - RHSC;
417      }
418      Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(AddSub, RHSC),MVT::i32);
419      return true;
420    }
421  }
422
423  Base = N.getOperand(0);
424  Offset = N.getOperand(1);
425  Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(ARM_AM::add, 0), MVT::i32);
426  return true;
427}
428
429bool ARMDAGToDAGISel::SelectAddrMode3Offset(SDNode *Op, SDValue N,
430                                            SDValue &Offset, SDValue &Opc) {
431  unsigned Opcode = Op->getOpcode();
432  ISD::MemIndexedMode AM = (Opcode == ISD::LOAD)
433    ? cast<LoadSDNode>(Op)->getAddressingMode()
434    : cast<StoreSDNode>(Op)->getAddressingMode();
435  ARM_AM::AddrOpc AddSub = (AM == ISD::PRE_INC || AM == ISD::POST_INC)
436    ? ARM_AM::add : ARM_AM::sub;
437  if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(N)) {
438    int Val = (int)C->getZExtValue();
439    if (Val >= 0 && Val < 256) {
440      Offset = CurDAG->getRegister(0, MVT::i32);
441      Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(AddSub, Val), MVT::i32);
442      return true;
443    }
444  }
445
446  Offset = N;
447  Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(AddSub, 0), MVT::i32);
448  return true;
449}
450
451bool ARMDAGToDAGISel::SelectAddrMode4(SDNode *Op, SDValue N,
452                                      SDValue &Addr, SDValue &Mode) {
453  Addr = N;
454  Mode = CurDAG->getTargetConstant(0, MVT::i32);
455  return true;
456}
457
458bool ARMDAGToDAGISel::SelectAddrMode5(SDNode *Op, SDValue N,
459                                      SDValue &Base, SDValue &Offset) {
460  if (N.getOpcode() != ISD::ADD) {
461    Base = N;
462    if (N.getOpcode() == ISD::FrameIndex) {
463      int FI = cast<FrameIndexSDNode>(N)->getIndex();
464      Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
465    } else if (N.getOpcode() == ARMISD::Wrapper &&
466               !(Subtarget->useMovt() &&
467                 N.getOperand(0).getOpcode() == ISD::TargetGlobalAddress)) {
468      Base = N.getOperand(0);
469    }
470    Offset = CurDAG->getTargetConstant(ARM_AM::getAM5Opc(ARM_AM::add, 0),
471                                       MVT::i32);
472    return true;
473  }
474
475  // If the RHS is +/- imm8, fold into addr mode.
476  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
477    int RHSC = (int)RHS->getZExtValue();
478    if ((RHSC & 3) == 0) {  // The constant is implicitly multiplied by 4.
479      RHSC >>= 2;
480      if ((RHSC >= 0 && RHSC < 256) ||
481          (RHSC < 0 && RHSC > -256)) { // note -256 itself isn't allowed.
482        Base = N.getOperand(0);
483        if (Base.getOpcode() == ISD::FrameIndex) {
484          int FI = cast<FrameIndexSDNode>(Base)->getIndex();
485          Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
486        }
487
488        ARM_AM::AddrOpc AddSub = ARM_AM::add;
489        if (RHSC < 0) {
490          AddSub = ARM_AM::sub;
491          RHSC = - RHSC;
492        }
493        Offset = CurDAG->getTargetConstant(ARM_AM::getAM5Opc(AddSub, RHSC),
494                                           MVT::i32);
495        return true;
496      }
497    }
498  }
499
500  Base = N;
501  Offset = CurDAG->getTargetConstant(ARM_AM::getAM5Opc(ARM_AM::add, 0),
502                                     MVT::i32);
503  return true;
504}
505
506bool ARMDAGToDAGISel::SelectAddrMode6(SDNode *Op, SDValue N,
507                                      SDValue &Addr, SDValue &Align) {
508  Addr = N;
509  // Default to no alignment.
510  Align = CurDAG->getTargetConstant(0, MVT::i32);
511  return true;
512}
513
514bool ARMDAGToDAGISel::SelectAddrModePC(SDNode *Op, SDValue N,
515                                       SDValue &Offset, SDValue &Label) {
516  if (N.getOpcode() == ARMISD::PIC_ADD && N.hasOneUse()) {
517    Offset = N.getOperand(0);
518    SDValue N1 = N.getOperand(1);
519    Label  = CurDAG->getTargetConstant(cast<ConstantSDNode>(N1)->getZExtValue(),
520                                       MVT::i32);
521    return true;
522  }
523  return false;
524}
525
526bool ARMDAGToDAGISel::SelectThumbAddrModeRR(SDNode *Op, SDValue N,
527                                            SDValue &Base, SDValue &Offset){
528  // FIXME dl should come from the parent load or store, not the address
529  DebugLoc dl = Op->getDebugLoc();
530  if (N.getOpcode() != ISD::ADD) {
531    ConstantSDNode *NC = dyn_cast<ConstantSDNode>(N);
532    if (!NC || NC->getZExtValue() != 0)
533      return false;
534
535    Base = Offset = N;
536    return true;
537  }
538
539  Base = N.getOperand(0);
540  Offset = N.getOperand(1);
541  return true;
542}
543
544bool
545ARMDAGToDAGISel::SelectThumbAddrModeRI5(SDNode *Op, SDValue N,
546                                        unsigned Scale, SDValue &Base,
547                                        SDValue &OffImm, SDValue &Offset) {
548  if (Scale == 4) {
549    SDValue TmpBase, TmpOffImm;
550    if (SelectThumbAddrModeSP(Op, N, TmpBase, TmpOffImm))
551      return false;  // We want to select tLDRspi / tSTRspi instead.
552    if (N.getOpcode() == ARMISD::Wrapper &&
553        N.getOperand(0).getOpcode() == ISD::TargetConstantPool)
554      return false;  // We want to select tLDRpci instead.
555  }
556
557  if (N.getOpcode() != ISD::ADD) {
558    if (N.getOpcode() == ARMISD::Wrapper &&
559        !(Subtarget->useMovt() &&
560          N.getOperand(0).getOpcode() == ISD::TargetGlobalAddress)) {
561      Base = N.getOperand(0);
562    } else
563      Base = N;
564
565    Offset = CurDAG->getRegister(0, MVT::i32);
566    OffImm = CurDAG->getTargetConstant(0, MVT::i32);
567    return true;
568  }
569
570  // Thumb does not have [sp, r] address mode.
571  RegisterSDNode *LHSR = dyn_cast<RegisterSDNode>(N.getOperand(0));
572  RegisterSDNode *RHSR = dyn_cast<RegisterSDNode>(N.getOperand(1));
573  if ((LHSR && LHSR->getReg() == ARM::SP) ||
574      (RHSR && RHSR->getReg() == ARM::SP)) {
575    Base = N;
576    Offset = CurDAG->getRegister(0, MVT::i32);
577    OffImm = CurDAG->getTargetConstant(0, MVT::i32);
578    return true;
579  }
580
581  // If the RHS is + imm5 * scale, fold into addr mode.
582  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
583    int RHSC = (int)RHS->getZExtValue();
584    if ((RHSC & (Scale-1)) == 0) {  // The constant is implicitly multiplied.
585      RHSC /= Scale;
586      if (RHSC >= 0 && RHSC < 32) {
587        Base = N.getOperand(0);
588        Offset = CurDAG->getRegister(0, MVT::i32);
589        OffImm = CurDAG->getTargetConstant(RHSC, MVT::i32);
590        return true;
591      }
592    }
593  }
594
595  Base = N.getOperand(0);
596  Offset = N.getOperand(1);
597  OffImm = CurDAG->getTargetConstant(0, MVT::i32);
598  return true;
599}
600
601bool ARMDAGToDAGISel::SelectThumbAddrModeS1(SDNode *Op, SDValue N,
602                                            SDValue &Base, SDValue &OffImm,
603                                            SDValue &Offset) {
604  return SelectThumbAddrModeRI5(Op, N, 1, Base, OffImm, Offset);
605}
606
607bool ARMDAGToDAGISel::SelectThumbAddrModeS2(SDNode *Op, SDValue N,
608                                            SDValue &Base, SDValue &OffImm,
609                                            SDValue &Offset) {
610  return SelectThumbAddrModeRI5(Op, N, 2, Base, OffImm, Offset);
611}
612
613bool ARMDAGToDAGISel::SelectThumbAddrModeS4(SDNode *Op, SDValue N,
614                                            SDValue &Base, SDValue &OffImm,
615                                            SDValue &Offset) {
616  return SelectThumbAddrModeRI5(Op, N, 4, Base, OffImm, Offset);
617}
618
619bool ARMDAGToDAGISel::SelectThumbAddrModeSP(SDNode *Op, SDValue N,
620                                           SDValue &Base, SDValue &OffImm) {
621  if (N.getOpcode() == ISD::FrameIndex) {
622    int FI = cast<FrameIndexSDNode>(N)->getIndex();
623    Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
624    OffImm = CurDAG->getTargetConstant(0, MVT::i32);
625    return true;
626  }
627
628  if (N.getOpcode() != ISD::ADD)
629    return false;
630
631  RegisterSDNode *LHSR = dyn_cast<RegisterSDNode>(N.getOperand(0));
632  if (N.getOperand(0).getOpcode() == ISD::FrameIndex ||
633      (LHSR && LHSR->getReg() == ARM::SP)) {
634    // If the RHS is + imm8 * scale, fold into addr mode.
635    if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
636      int RHSC = (int)RHS->getZExtValue();
637      if ((RHSC & 3) == 0) {  // The constant is implicitly multiplied.
638        RHSC >>= 2;
639        if (RHSC >= 0 && RHSC < 256) {
640          Base = N.getOperand(0);
641          if (Base.getOpcode() == ISD::FrameIndex) {
642            int FI = cast<FrameIndexSDNode>(Base)->getIndex();
643            Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
644          }
645          OffImm = CurDAG->getTargetConstant(RHSC, MVT::i32);
646          return true;
647        }
648      }
649    }
650  }
651
652  return false;
653}
654
655bool ARMDAGToDAGISel::SelectT2ShifterOperandReg(SDNode *Op, SDValue N,
656                                                SDValue &BaseReg,
657                                                SDValue &Opc) {
658  ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(N);
659
660  // Don't match base register only case. That is matched to a separate
661  // lower complexity pattern with explicit register operand.
662  if (ShOpcVal == ARM_AM::no_shift) return false;
663
664  BaseReg = N.getOperand(0);
665  unsigned ShImmVal = 0;
666  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
667    ShImmVal = RHS->getZExtValue() & 31;
668    Opc = getI32Imm(ARM_AM::getSORegOpc(ShOpcVal, ShImmVal));
669    return true;
670  }
671
672  return false;
673}
674
675bool ARMDAGToDAGISel::SelectT2AddrModeImm12(SDNode *Op, SDValue N,
676                                            SDValue &Base, SDValue &OffImm) {
677  // Match simple R + imm12 operands.
678
679  // Base only.
680  if (N.getOpcode() != ISD::ADD && N.getOpcode() != ISD::SUB) {
681    if (N.getOpcode() == ISD::FrameIndex) {
682      // Match frame index...
683      int FI = cast<FrameIndexSDNode>(N)->getIndex();
684      Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
685      OffImm  = CurDAG->getTargetConstant(0, MVT::i32);
686      return true;
687    } else if (N.getOpcode() == ARMISD::Wrapper &&
688               !(Subtarget->useMovt() &&
689                 N.getOperand(0).getOpcode() == ISD::TargetGlobalAddress)) {
690      Base = N.getOperand(0);
691      if (Base.getOpcode() == ISD::TargetConstantPool)
692        return false;  // We want to select t2LDRpci instead.
693    } else
694      Base = N;
695    OffImm  = CurDAG->getTargetConstant(0, MVT::i32);
696    return true;
697  }
698
699  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
700    if (SelectT2AddrModeImm8(Op, N, Base, OffImm))
701      // Let t2LDRi8 handle (R - imm8).
702      return false;
703
704    int RHSC = (int)RHS->getZExtValue();
705    if (N.getOpcode() == ISD::SUB)
706      RHSC = -RHSC;
707
708    if (RHSC >= 0 && RHSC < 0x1000) { // 12 bits (unsigned)
709      Base   = N.getOperand(0);
710      if (Base.getOpcode() == ISD::FrameIndex) {
711        int FI = cast<FrameIndexSDNode>(Base)->getIndex();
712        Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
713      }
714      OffImm = CurDAG->getTargetConstant(RHSC, MVT::i32);
715      return true;
716    }
717  }
718
719  // Base only.
720  Base = N;
721  OffImm  = CurDAG->getTargetConstant(0, MVT::i32);
722  return true;
723}
724
725bool ARMDAGToDAGISel::SelectT2AddrModeImm8(SDNode *Op, SDValue N,
726                                           SDValue &Base, SDValue &OffImm) {
727  // Match simple R - imm8 operands.
728  if (N.getOpcode() == ISD::ADD || N.getOpcode() == ISD::SUB) {
729    if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
730      int RHSC = (int)RHS->getSExtValue();
731      if (N.getOpcode() == ISD::SUB)
732        RHSC = -RHSC;
733
734      if ((RHSC >= -255) && (RHSC < 0)) { // 8 bits (always negative)
735        Base = N.getOperand(0);
736        if (Base.getOpcode() == ISD::FrameIndex) {
737          int FI = cast<FrameIndexSDNode>(Base)->getIndex();
738          Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
739        }
740        OffImm = CurDAG->getTargetConstant(RHSC, MVT::i32);
741        return true;
742      }
743    }
744  }
745
746  return false;
747}
748
749bool ARMDAGToDAGISel::SelectT2AddrModeImm8Offset(SDNode *Op, SDValue N,
750                                                 SDValue &OffImm){
751  unsigned Opcode = Op->getOpcode();
752  ISD::MemIndexedMode AM = (Opcode == ISD::LOAD)
753    ? cast<LoadSDNode>(Op)->getAddressingMode()
754    : cast<StoreSDNode>(Op)->getAddressingMode();
755  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N)) {
756    int RHSC = (int)RHS->getZExtValue();
757    if (RHSC >= 0 && RHSC < 0x100) { // 8 bits.
758      OffImm = ((AM == ISD::PRE_INC) || (AM == ISD::POST_INC))
759        ? CurDAG->getTargetConstant(RHSC, MVT::i32)
760        : CurDAG->getTargetConstant(-RHSC, MVT::i32);
761      return true;
762    }
763  }
764
765  return false;
766}
767
768bool ARMDAGToDAGISel::SelectT2AddrModeImm8s4(SDNode *Op, SDValue N,
769                                             SDValue &Base, SDValue &OffImm) {
770  if (N.getOpcode() == ISD::ADD) {
771    if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
772      int RHSC = (int)RHS->getZExtValue();
773      if (((RHSC & 0x3) == 0) &&
774          ((RHSC >= 0 && RHSC < 0x400) || (RHSC < 0 && RHSC > -0x400))) { // 8 bits.
775        Base   = N.getOperand(0);
776        OffImm = CurDAG->getTargetConstant(RHSC, MVT::i32);
777        return true;
778      }
779    }
780  } else if (N.getOpcode() == ISD::SUB) {
781    if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
782      int RHSC = (int)RHS->getZExtValue();
783      if (((RHSC & 0x3) == 0) && (RHSC >= 0 && RHSC < 0x400)) { // 8 bits.
784        Base   = N.getOperand(0);
785        OffImm = CurDAG->getTargetConstant(-RHSC, MVT::i32);
786        return true;
787      }
788    }
789  }
790
791  return false;
792}
793
794bool ARMDAGToDAGISel::SelectT2AddrModeSoReg(SDNode *Op, SDValue N,
795                                            SDValue &Base,
796                                            SDValue &OffReg, SDValue &ShImm) {
797  // (R - imm8) should be handled by t2LDRi8. The rest are handled by t2LDRi12.
798  if (N.getOpcode() != ISD::ADD)
799    return false;
800
801  // Leave (R + imm12) for t2LDRi12, (R - imm8) for t2LDRi8.
802  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
803    int RHSC = (int)RHS->getZExtValue();
804    if (RHSC >= 0 && RHSC < 0x1000) // 12 bits (unsigned)
805      return false;
806    else if (RHSC < 0 && RHSC >= -255) // 8 bits
807      return false;
808  }
809
810  // Look for (R + R) or (R + (R << [1,2,3])).
811  unsigned ShAmt = 0;
812  Base   = N.getOperand(0);
813  OffReg = N.getOperand(1);
814
815  // Swap if it is ((R << c) + R).
816  ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(OffReg);
817  if (ShOpcVal != ARM_AM::lsl) {
818    ShOpcVal = ARM_AM::getShiftOpcForNode(Base);
819    if (ShOpcVal == ARM_AM::lsl)
820      std::swap(Base, OffReg);
821  }
822
823  if (ShOpcVal == ARM_AM::lsl) {
824    // Check to see if the RHS of the shift is a constant, if not, we can't fold
825    // it.
826    if (ConstantSDNode *Sh = dyn_cast<ConstantSDNode>(OffReg.getOperand(1))) {
827      ShAmt = Sh->getZExtValue();
828      if (ShAmt >= 4) {
829        ShAmt = 0;
830        ShOpcVal = ARM_AM::no_shift;
831      } else
832        OffReg = OffReg.getOperand(0);
833    } else {
834      ShOpcVal = ARM_AM::no_shift;
835    }
836  }
837
838  ShImm = CurDAG->getTargetConstant(ShAmt, MVT::i32);
839
840  return true;
841}
842
843//===--------------------------------------------------------------------===//
844
845/// getAL - Returns a ARMCC::AL immediate node.
846static inline SDValue getAL(SelectionDAG *CurDAG) {
847  return CurDAG->getTargetConstant((uint64_t)ARMCC::AL, MVT::i32);
848}
849
850SDNode *ARMDAGToDAGISel::SelectARMIndexedLoad(SDNode *N) {
851  LoadSDNode *LD = cast<LoadSDNode>(N);
852  ISD::MemIndexedMode AM = LD->getAddressingMode();
853  if (AM == ISD::UNINDEXED)
854    return NULL;
855
856  EVT LoadedVT = LD->getMemoryVT();
857  SDValue Offset, AMOpc;
858  bool isPre = (AM == ISD::PRE_INC) || (AM == ISD::PRE_DEC);
859  unsigned Opcode = 0;
860  bool Match = false;
861  if (LoadedVT == MVT::i32 &&
862      SelectAddrMode2Offset(N, LD->getOffset(), Offset, AMOpc)) {
863    Opcode = isPre ? ARM::LDR_PRE : ARM::LDR_POST;
864    Match = true;
865  } else if (LoadedVT == MVT::i16 &&
866             SelectAddrMode3Offset(N, LD->getOffset(), Offset, AMOpc)) {
867    Match = true;
868    Opcode = (LD->getExtensionType() == ISD::SEXTLOAD)
869      ? (isPre ? ARM::LDRSH_PRE : ARM::LDRSH_POST)
870      : (isPre ? ARM::LDRH_PRE : ARM::LDRH_POST);
871  } else if (LoadedVT == MVT::i8 || LoadedVT == MVT::i1) {
872    if (LD->getExtensionType() == ISD::SEXTLOAD) {
873      if (SelectAddrMode3Offset(N, LD->getOffset(), Offset, AMOpc)) {
874        Match = true;
875        Opcode = isPre ? ARM::LDRSB_PRE : ARM::LDRSB_POST;
876      }
877    } else {
878      if (SelectAddrMode2Offset(N, LD->getOffset(), Offset, AMOpc)) {
879        Match = true;
880        Opcode = isPre ? ARM::LDRB_PRE : ARM::LDRB_POST;
881      }
882    }
883  }
884
885  if (Match) {
886    SDValue Chain = LD->getChain();
887    SDValue Base = LD->getBasePtr();
888    SDValue Ops[]= { Base, Offset, AMOpc, getAL(CurDAG),
889                     CurDAG->getRegister(0, MVT::i32), Chain };
890    return CurDAG->getMachineNode(Opcode, N->getDebugLoc(), MVT::i32, MVT::i32,
891                                  MVT::Other, Ops, 6);
892  }
893
894  return NULL;
895}
896
897SDNode *ARMDAGToDAGISel::SelectT2IndexedLoad(SDNode *N) {
898  LoadSDNode *LD = cast<LoadSDNode>(N);
899  ISD::MemIndexedMode AM = LD->getAddressingMode();
900  if (AM == ISD::UNINDEXED)
901    return NULL;
902
903  EVT LoadedVT = LD->getMemoryVT();
904  bool isSExtLd = LD->getExtensionType() == ISD::SEXTLOAD;
905  SDValue Offset;
906  bool isPre = (AM == ISD::PRE_INC) || (AM == ISD::PRE_DEC);
907  unsigned Opcode = 0;
908  bool Match = false;
909  if (SelectT2AddrModeImm8Offset(N, LD->getOffset(), Offset)) {
910    switch (LoadedVT.getSimpleVT().SimpleTy) {
911    case MVT::i32:
912      Opcode = isPre ? ARM::t2LDR_PRE : ARM::t2LDR_POST;
913      break;
914    case MVT::i16:
915      if (isSExtLd)
916        Opcode = isPre ? ARM::t2LDRSH_PRE : ARM::t2LDRSH_POST;
917      else
918        Opcode = isPre ? ARM::t2LDRH_PRE : ARM::t2LDRH_POST;
919      break;
920    case MVT::i8:
921    case MVT::i1:
922      if (isSExtLd)
923        Opcode = isPre ? ARM::t2LDRSB_PRE : ARM::t2LDRSB_POST;
924      else
925        Opcode = isPre ? ARM::t2LDRB_PRE : ARM::t2LDRB_POST;
926      break;
927    default:
928      return NULL;
929    }
930    Match = true;
931  }
932
933  if (Match) {
934    SDValue Chain = LD->getChain();
935    SDValue Base = LD->getBasePtr();
936    SDValue Ops[]= { Base, Offset, getAL(CurDAG),
937                     CurDAG->getRegister(0, MVT::i32), Chain };
938    return CurDAG->getMachineNode(Opcode, N->getDebugLoc(), MVT::i32, MVT::i32,
939                                  MVT::Other, Ops, 5);
940  }
941
942  return NULL;
943}
944
945/// PairDRegs - Insert a pair of double registers into an implicit def to
946/// form a quad register.
947SDNode *ARMDAGToDAGISel::PairDRegs(EVT VT, SDValue V0, SDValue V1) {
948  DebugLoc dl = V0.getNode()->getDebugLoc();
949  SDValue SubReg0 = CurDAG->getTargetConstant(ARM::DSUBREG_0, MVT::i32);
950  SDValue SubReg1 = CurDAG->getTargetConstant(ARM::DSUBREG_1, MVT::i32);
951  if (llvm::ModelWithRegSequence()) {
952    const SDValue Ops[] = { V0, SubReg0, V1, SubReg1 };
953    return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops, 4);
954  }
955  SDValue Undef =
956    SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, dl, VT), 0);
957  SDNode *Pair = CurDAG->getMachineNode(TargetOpcode::INSERT_SUBREG, dl,
958                                        VT, Undef, V0, SubReg0);
959  return CurDAG->getMachineNode(TargetOpcode::INSERT_SUBREG, dl,
960                                VT, SDValue(Pair, 0), V1, SubReg1);
961}
962
963/// GetNEONSubregVT - Given a type for a 128-bit NEON vector, return the type
964/// for a 64-bit subregister of the vector.
965static EVT GetNEONSubregVT(EVT VT) {
966  switch (VT.getSimpleVT().SimpleTy) {
967  default: llvm_unreachable("unhandled NEON type");
968  case MVT::v16i8: return MVT::v8i8;
969  case MVT::v8i16: return MVT::v4i16;
970  case MVT::v4f32: return MVT::v2f32;
971  case MVT::v4i32: return MVT::v2i32;
972  case MVT::v2i64: return MVT::v1i64;
973  }
974}
975
976SDNode *ARMDAGToDAGISel::SelectVLD(SDNode *N, unsigned NumVecs,
977                                   unsigned *DOpcodes, unsigned *QOpcodes0,
978                                   unsigned *QOpcodes1) {
979  assert(NumVecs >= 1 && NumVecs <= 4 && "VLD NumVecs out-of-range");
980  DebugLoc dl = N->getDebugLoc();
981
982  SDValue MemAddr, Align;
983  if (!SelectAddrMode6(N, N->getOperand(2), MemAddr, Align))
984    return NULL;
985
986  SDValue Chain = N->getOperand(0);
987  EVT VT = N->getValueType(0);
988  bool is64BitVector = VT.is64BitVector();
989
990  unsigned OpcodeIndex;
991  switch (VT.getSimpleVT().SimpleTy) {
992  default: llvm_unreachable("unhandled vld type");
993    // Double-register operations:
994  case MVT::v8i8:  OpcodeIndex = 0; break;
995  case MVT::v4i16: OpcodeIndex = 1; break;
996  case MVT::v2f32:
997  case MVT::v2i32: OpcodeIndex = 2; break;
998  case MVT::v1i64: OpcodeIndex = 3; break;
999    // Quad-register operations:
1000  case MVT::v16i8: OpcodeIndex = 0; break;
1001  case MVT::v8i16: OpcodeIndex = 1; break;
1002  case MVT::v4f32:
1003  case MVT::v4i32: OpcodeIndex = 2; break;
1004  case MVT::v2i64: OpcodeIndex = 3;
1005    assert(NumVecs == 1 && "v2i64 type only supported for VLD1");
1006    break;
1007  }
1008
1009  SDValue Pred = getAL(CurDAG);
1010  SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
1011  if (is64BitVector) {
1012    unsigned Opc = DOpcodes[OpcodeIndex];
1013    const SDValue Ops[] = { MemAddr, Align, Pred, Reg0, Chain };
1014    std::vector<EVT> ResTys(NumVecs, VT);
1015    ResTys.push_back(MVT::Other);
1016    return CurDAG->getMachineNode(Opc, dl, ResTys, Ops, 5);
1017  }
1018
1019  EVT RegVT = GetNEONSubregVT(VT);
1020  if (NumVecs <= 2) {
1021    // Quad registers are directly supported for VLD1 and VLD2,
1022    // loading pairs of D regs.
1023    unsigned Opc = QOpcodes0[OpcodeIndex];
1024    const SDValue Ops[] = { MemAddr, Align, Pred, Reg0, Chain };
1025    std::vector<EVT> ResTys(2 * NumVecs, RegVT);
1026    ResTys.push_back(MVT::Other);
1027    SDNode *VLd = CurDAG->getMachineNode(Opc, dl, ResTys, Ops, 5);
1028    Chain = SDValue(VLd, 2 * NumVecs);
1029
1030    // Combine the even and odd subregs to produce the result.
1031    for (unsigned Vec = 0; Vec < NumVecs; ++Vec) {
1032      SDNode *Q = PairDRegs(VT, SDValue(VLd, 2*Vec), SDValue(VLd, 2*Vec+1));
1033      ReplaceUses(SDValue(N, Vec), SDValue(Q, 0));
1034    }
1035  } else {
1036    // Otherwise, quad registers are loaded with two separate instructions,
1037    // where one loads the even registers and the other loads the odd registers.
1038
1039    std::vector<EVT> ResTys(NumVecs, RegVT);
1040    ResTys.push_back(MemAddr.getValueType());
1041    ResTys.push_back(MVT::Other);
1042
1043    // Load the even subregs.
1044    unsigned Opc = QOpcodes0[OpcodeIndex];
1045    const SDValue OpsA[] = { MemAddr, Align, Reg0, Pred, Reg0, Chain };
1046    SDNode *VLdA = CurDAG->getMachineNode(Opc, dl, ResTys, OpsA, 6);
1047    Chain = SDValue(VLdA, NumVecs+1);
1048
1049    // Load the odd subregs.
1050    Opc = QOpcodes1[OpcodeIndex];
1051    const SDValue OpsB[] = { SDValue(VLdA, NumVecs),
1052                             Align, Reg0, Pred, Reg0, Chain };
1053    SDNode *VLdB = CurDAG->getMachineNode(Opc, dl, ResTys, OpsB, 6);
1054    Chain = SDValue(VLdB, NumVecs+1);
1055
1056    // Combine the even and odd subregs to produce the result.
1057    for (unsigned Vec = 0; Vec < NumVecs; ++Vec) {
1058      SDNode *Q = PairDRegs(VT, SDValue(VLdA, Vec), SDValue(VLdB, Vec));
1059      ReplaceUses(SDValue(N, Vec), SDValue(Q, 0));
1060    }
1061  }
1062  ReplaceUses(SDValue(N, NumVecs), Chain);
1063  return NULL;
1064}
1065
1066SDNode *ARMDAGToDAGISel::SelectVST(SDNode *N, unsigned NumVecs,
1067                                   unsigned *DOpcodes, unsigned *QOpcodes0,
1068                                   unsigned *QOpcodes1) {
1069  assert(NumVecs >=1 && NumVecs <= 4 && "VST NumVecs out-of-range");
1070  DebugLoc dl = N->getDebugLoc();
1071
1072  SDValue MemAddr, Align;
1073  if (!SelectAddrMode6(N, N->getOperand(2), MemAddr, Align))
1074    return NULL;
1075
1076  SDValue Chain = N->getOperand(0);
1077  EVT VT = N->getOperand(3).getValueType();
1078  bool is64BitVector = VT.is64BitVector();
1079
1080  unsigned OpcodeIndex;
1081  switch (VT.getSimpleVT().SimpleTy) {
1082  default: llvm_unreachable("unhandled vst type");
1083    // Double-register operations:
1084  case MVT::v8i8:  OpcodeIndex = 0; break;
1085  case MVT::v4i16: OpcodeIndex = 1; break;
1086  case MVT::v2f32:
1087  case MVT::v2i32: OpcodeIndex = 2; break;
1088  case MVT::v1i64: OpcodeIndex = 3; break;
1089    // Quad-register operations:
1090  case MVT::v16i8: OpcodeIndex = 0; break;
1091  case MVT::v8i16: OpcodeIndex = 1; break;
1092  case MVT::v4f32:
1093  case MVT::v4i32: OpcodeIndex = 2; break;
1094  case MVT::v2i64: OpcodeIndex = 3;
1095    assert(NumVecs == 1 && "v2i64 type only supported for VST1");
1096    break;
1097  }
1098
1099  SDValue Pred = getAL(CurDAG);
1100  SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
1101
1102  SmallVector<SDValue, 10> Ops;
1103  Ops.push_back(MemAddr);
1104  Ops.push_back(Align);
1105
1106  if (is64BitVector) {
1107    unsigned Opc = DOpcodes[OpcodeIndex];
1108    for (unsigned Vec = 0; Vec < NumVecs; ++Vec)
1109      Ops.push_back(N->getOperand(Vec+3));
1110    Ops.push_back(Pred);
1111    Ops.push_back(Reg0); // predicate register
1112    Ops.push_back(Chain);
1113    return CurDAG->getMachineNode(Opc, dl, MVT::Other, Ops.data(), NumVecs+5);
1114  }
1115
1116  EVT RegVT = GetNEONSubregVT(VT);
1117  if (NumVecs <= 2) {
1118    // Quad registers are directly supported for VST1 and VST2,
1119    // storing pairs of D regs.
1120    unsigned Opc = QOpcodes0[OpcodeIndex];
1121    for (unsigned Vec = 0; Vec < NumVecs; ++Vec) {
1122      Ops.push_back(CurDAG->getTargetExtractSubreg(ARM::DSUBREG_0, dl, RegVT,
1123                                                   N->getOperand(Vec+3)));
1124      Ops.push_back(CurDAG->getTargetExtractSubreg(ARM::DSUBREG_1, dl, RegVT,
1125                                                   N->getOperand(Vec+3)));
1126    }
1127    Ops.push_back(Pred);
1128    Ops.push_back(Reg0); // predicate register
1129    Ops.push_back(Chain);
1130    return CurDAG->getMachineNode(Opc, dl, MVT::Other, Ops.data(),
1131                                  5 + 2 * NumVecs);
1132  }
1133
1134  // Otherwise, quad registers are stored with two separate instructions,
1135  // where one stores the even registers and the other stores the odd registers.
1136
1137  Ops.push_back(Reg0); // post-access address offset
1138
1139  // Store the even subregs.
1140  for (unsigned Vec = 0; Vec < NumVecs; ++Vec)
1141    Ops.push_back(CurDAG->getTargetExtractSubreg(ARM::DSUBREG_0, dl, RegVT,
1142                                                 N->getOperand(Vec+3)));
1143  Ops.push_back(Pred);
1144  Ops.push_back(Reg0); // predicate register
1145  Ops.push_back(Chain);
1146  unsigned Opc = QOpcodes0[OpcodeIndex];
1147  SDNode *VStA = CurDAG->getMachineNode(Opc, dl, MemAddr.getValueType(),
1148                                        MVT::Other, Ops.data(), NumVecs+6);
1149  Chain = SDValue(VStA, 1);
1150
1151  // Store the odd subregs.
1152  Ops[0] = SDValue(VStA, 0); // MemAddr
1153  for (unsigned Vec = 0; Vec < NumVecs; ++Vec)
1154    Ops[Vec+3] = CurDAG->getTargetExtractSubreg(ARM::DSUBREG_1, dl, RegVT,
1155                                                N->getOperand(Vec+3));
1156  Ops[NumVecs+5] = Chain;
1157  Opc = QOpcodes1[OpcodeIndex];
1158  SDNode *VStB = CurDAG->getMachineNode(Opc, dl, MemAddr.getValueType(),
1159                                        MVT::Other, Ops.data(), NumVecs+6);
1160  Chain = SDValue(VStB, 1);
1161  ReplaceUses(SDValue(N, 0), Chain);
1162  return NULL;
1163}
1164
1165SDNode *ARMDAGToDAGISel::SelectVLDSTLane(SDNode *N, bool IsLoad,
1166                                         unsigned NumVecs, unsigned *DOpcodes,
1167                                         unsigned *QOpcodes0,
1168                                         unsigned *QOpcodes1) {
1169  assert(NumVecs >=2 && NumVecs <= 4 && "VLDSTLane NumVecs out-of-range");
1170  DebugLoc dl = N->getDebugLoc();
1171
1172  SDValue MemAddr, Align;
1173  if (!SelectAddrMode6(N, N->getOperand(2), MemAddr, Align))
1174    return NULL;
1175
1176  SDValue Chain = N->getOperand(0);
1177  unsigned Lane =
1178    cast<ConstantSDNode>(N->getOperand(NumVecs+3))->getZExtValue();
1179  EVT VT = IsLoad ? N->getValueType(0) : N->getOperand(3).getValueType();
1180  bool is64BitVector = VT.is64BitVector();
1181
1182  // Quad registers are handled by load/store of subregs. Find the subreg info.
1183  unsigned NumElts = 0;
1184  int SubregIdx = 0;
1185  EVT RegVT = VT;
1186  if (!is64BitVector) {
1187    RegVT = GetNEONSubregVT(VT);
1188    NumElts = RegVT.getVectorNumElements();
1189    SubregIdx = (Lane < NumElts) ? ARM::DSUBREG_0 : ARM::DSUBREG_1;
1190  }
1191
1192  unsigned OpcodeIndex;
1193  switch (VT.getSimpleVT().SimpleTy) {
1194  default: llvm_unreachable("unhandled vld/vst lane type");
1195    // Double-register operations:
1196  case MVT::v8i8:  OpcodeIndex = 0; break;
1197  case MVT::v4i16: OpcodeIndex = 1; break;
1198  case MVT::v2f32:
1199  case MVT::v2i32: OpcodeIndex = 2; break;
1200    // Quad-register operations:
1201  case MVT::v8i16: OpcodeIndex = 0; break;
1202  case MVT::v4f32:
1203  case MVT::v4i32: OpcodeIndex = 1; break;
1204  }
1205
1206  SDValue Pred = getAL(CurDAG);
1207  SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
1208
1209  SmallVector<SDValue, 10> Ops;
1210  Ops.push_back(MemAddr);
1211  Ops.push_back(Align);
1212
1213  unsigned Opc = 0;
1214  if (is64BitVector) {
1215    Opc = DOpcodes[OpcodeIndex];
1216    for (unsigned Vec = 0; Vec < NumVecs; ++Vec)
1217      Ops.push_back(N->getOperand(Vec+3));
1218  } else {
1219    // Check if this is loading the even or odd subreg of a Q register.
1220    if (Lane < NumElts) {
1221      Opc = QOpcodes0[OpcodeIndex];
1222    } else {
1223      Lane -= NumElts;
1224      Opc = QOpcodes1[OpcodeIndex];
1225    }
1226    // Extract the subregs of the input vector.
1227    for (unsigned Vec = 0; Vec < NumVecs; ++Vec)
1228      Ops.push_back(CurDAG->getTargetExtractSubreg(SubregIdx, dl, RegVT,
1229                                                   N->getOperand(Vec+3)));
1230  }
1231  Ops.push_back(getI32Imm(Lane));
1232  Ops.push_back(Pred);
1233  Ops.push_back(Reg0);
1234  Ops.push_back(Chain);
1235
1236  if (!IsLoad)
1237    return CurDAG->getMachineNode(Opc, dl, MVT::Other, Ops.data(), NumVecs+6);
1238
1239  std::vector<EVT> ResTys(NumVecs, RegVT);
1240  ResTys.push_back(MVT::Other);
1241  SDNode *VLdLn =
1242    CurDAG->getMachineNode(Opc, dl, ResTys, Ops.data(), NumVecs+6);
1243  // For a 64-bit vector load to D registers, nothing more needs to be done.
1244  if (is64BitVector)
1245    return VLdLn;
1246
1247  // For 128-bit vectors, take the 64-bit results of the load and insert them
1248  // as subregs into the result.
1249  for (unsigned Vec = 0; Vec < NumVecs; ++Vec) {
1250    SDValue QuadVec = CurDAG->getTargetInsertSubreg(SubregIdx, dl, VT,
1251                                                    N->getOperand(Vec+3),
1252                                                    SDValue(VLdLn, Vec));
1253    ReplaceUses(SDValue(N, Vec), QuadVec);
1254  }
1255
1256  Chain = SDValue(VLdLn, NumVecs);
1257  ReplaceUses(SDValue(N, NumVecs), Chain);
1258  return NULL;
1259}
1260
1261SDNode *ARMDAGToDAGISel::SelectV6T2BitfieldExtractOp(SDNode *N,
1262                                                     bool isSigned) {
1263  if (!Subtarget->hasV6T2Ops())
1264    return NULL;
1265
1266  unsigned Opc = isSigned ? (Subtarget->isThumb() ? ARM::t2SBFX : ARM::SBFX)
1267    : (Subtarget->isThumb() ? ARM::t2UBFX : ARM::UBFX);
1268
1269
1270  // For unsigned extracts, check for a shift right and mask
1271  unsigned And_imm = 0;
1272  if (N->getOpcode() == ISD::AND) {
1273    if (isOpcWithIntImmediate(N, ISD::AND, And_imm)) {
1274
1275      // The immediate is a mask of the low bits iff imm & (imm+1) == 0
1276      if (And_imm & (And_imm + 1))
1277        return NULL;
1278
1279      unsigned Srl_imm = 0;
1280      if (isOpcWithIntImmediate(N->getOperand(0).getNode(), ISD::SRL,
1281                                Srl_imm)) {
1282        assert(Srl_imm > 0 && Srl_imm < 32 && "bad amount in shift node!");
1283
1284        unsigned Width = CountTrailingOnes_32(And_imm);
1285        unsigned LSB = Srl_imm;
1286        SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
1287        SDValue Ops[] = { N->getOperand(0).getOperand(0),
1288                          CurDAG->getTargetConstant(LSB, MVT::i32),
1289                          CurDAG->getTargetConstant(Width, MVT::i32),
1290          getAL(CurDAG), Reg0 };
1291        return CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops, 5);
1292      }
1293    }
1294    return NULL;
1295  }
1296
1297  // Otherwise, we're looking for a shift of a shift
1298  unsigned Shl_imm = 0;
1299  if (isOpcWithIntImmediate(N->getOperand(0).getNode(), ISD::SHL, Shl_imm)) {
1300    assert(Shl_imm > 0 && Shl_imm < 32 && "bad amount in shift node!");
1301    unsigned Srl_imm = 0;
1302    if (isInt32Immediate(N->getOperand(1), Srl_imm)) {
1303      assert(Srl_imm > 0 && Srl_imm < 32 && "bad amount in shift node!");
1304      unsigned Width = 32 - Srl_imm;
1305      int LSB = Srl_imm - Shl_imm;
1306      if (LSB < 0)
1307        return NULL;
1308      SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
1309      SDValue Ops[] = { N->getOperand(0).getOperand(0),
1310                        CurDAG->getTargetConstant(LSB, MVT::i32),
1311                        CurDAG->getTargetConstant(Width, MVT::i32),
1312                        getAL(CurDAG), Reg0 };
1313      return CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops, 5);
1314    }
1315  }
1316  return NULL;
1317}
1318
1319SDNode *ARMDAGToDAGISel::
1320SelectT2CMOVShiftOp(SDNode *N, SDValue FalseVal, SDValue TrueVal,
1321                    ARMCC::CondCodes CCVal, SDValue CCR, SDValue InFlag) {
1322  SDValue CPTmp0;
1323  SDValue CPTmp1;
1324  if (SelectT2ShifterOperandReg(N, TrueVal, CPTmp0, CPTmp1)) {
1325    unsigned SOVal = cast<ConstantSDNode>(CPTmp1)->getZExtValue();
1326    unsigned SOShOp = ARM_AM::getSORegShOp(SOVal);
1327    unsigned Opc = 0;
1328    switch (SOShOp) {
1329    case ARM_AM::lsl: Opc = ARM::t2MOVCClsl; break;
1330    case ARM_AM::lsr: Opc = ARM::t2MOVCClsr; break;
1331    case ARM_AM::asr: Opc = ARM::t2MOVCCasr; break;
1332    case ARM_AM::ror: Opc = ARM::t2MOVCCror; break;
1333    default:
1334      llvm_unreachable("Unknown so_reg opcode!");
1335      break;
1336    }
1337    SDValue SOShImm =
1338      CurDAG->getTargetConstant(ARM_AM::getSORegOffset(SOVal), MVT::i32);
1339    SDValue CC = CurDAG->getTargetConstant(CCVal, MVT::i32);
1340    SDValue Ops[] = { FalseVal, CPTmp0, SOShImm, CC, CCR, InFlag };
1341    return CurDAG->SelectNodeTo(N, Opc, MVT::i32,Ops, 6);
1342  }
1343  return 0;
1344}
1345
1346SDNode *ARMDAGToDAGISel::
1347SelectARMCMOVShiftOp(SDNode *N, SDValue FalseVal, SDValue TrueVal,
1348                     ARMCC::CondCodes CCVal, SDValue CCR, SDValue InFlag) {
1349  SDValue CPTmp0;
1350  SDValue CPTmp1;
1351  SDValue CPTmp2;
1352  if (SelectShifterOperandReg(N, TrueVal, CPTmp0, CPTmp1, CPTmp2)) {
1353    SDValue CC = CurDAG->getTargetConstant(CCVal, MVT::i32);
1354    SDValue Ops[] = { FalseVal, CPTmp0, CPTmp1, CPTmp2, CC, CCR, InFlag };
1355    return CurDAG->SelectNodeTo(N, ARM::MOVCCs, MVT::i32, Ops, 7);
1356  }
1357  return 0;
1358}
1359
1360SDNode *ARMDAGToDAGISel::
1361SelectT2CMOVSoImmOp(SDNode *N, SDValue FalseVal, SDValue TrueVal,
1362                    ARMCC::CondCodes CCVal, SDValue CCR, SDValue InFlag) {
1363  ConstantSDNode *T = dyn_cast<ConstantSDNode>(TrueVal);
1364  if (!T)
1365    return 0;
1366
1367  if (Predicate_t2_so_imm(TrueVal.getNode())) {
1368    SDValue True = CurDAG->getTargetConstant(T->getZExtValue(), MVT::i32);
1369    SDValue CC = CurDAG->getTargetConstant(CCVal, MVT::i32);
1370    SDValue Ops[] = { FalseVal, True, CC, CCR, InFlag };
1371    return CurDAG->SelectNodeTo(N,
1372                                ARM::t2MOVCCi, MVT::i32, Ops, 5);
1373  }
1374  return 0;
1375}
1376
1377SDNode *ARMDAGToDAGISel::
1378SelectARMCMOVSoImmOp(SDNode *N, SDValue FalseVal, SDValue TrueVal,
1379                     ARMCC::CondCodes CCVal, SDValue CCR, SDValue InFlag) {
1380  ConstantSDNode *T = dyn_cast<ConstantSDNode>(TrueVal);
1381  if (!T)
1382    return 0;
1383
1384  if (Predicate_so_imm(TrueVal.getNode())) {
1385    SDValue True = CurDAG->getTargetConstant(T->getZExtValue(), MVT::i32);
1386    SDValue CC = CurDAG->getTargetConstant(CCVal, MVT::i32);
1387    SDValue Ops[] = { FalseVal, True, CC, CCR, InFlag };
1388    return CurDAG->SelectNodeTo(N,
1389                                ARM::MOVCCi, MVT::i32, Ops, 5);
1390  }
1391  return 0;
1392}
1393
1394SDNode *ARMDAGToDAGISel::SelectCMOVOp(SDNode *N) {
1395  EVT VT = N->getValueType(0);
1396  SDValue FalseVal = N->getOperand(0);
1397  SDValue TrueVal  = N->getOperand(1);
1398  SDValue CC = N->getOperand(2);
1399  SDValue CCR = N->getOperand(3);
1400  SDValue InFlag = N->getOperand(4);
1401  assert(CC.getOpcode() == ISD::Constant);
1402  assert(CCR.getOpcode() == ISD::Register);
1403  ARMCC::CondCodes CCVal =
1404    (ARMCC::CondCodes)cast<ConstantSDNode>(CC)->getZExtValue();
1405
1406  if (!Subtarget->isThumb1Only() && VT == MVT::i32) {
1407    // Pattern: (ARMcmov:i32 GPR:i32:$false, so_reg:i32:$true, (imm:i32):$cc)
1408    // Emits: (MOVCCs:i32 GPR:i32:$false, so_reg:i32:$true, (imm:i32):$cc)
1409    // Pattern complexity = 18  cost = 1  size = 0
1410    SDValue CPTmp0;
1411    SDValue CPTmp1;
1412    SDValue CPTmp2;
1413    if (Subtarget->isThumb()) {
1414      SDNode *Res = SelectT2CMOVShiftOp(N, FalseVal, TrueVal,
1415                                        CCVal, CCR, InFlag);
1416      if (!Res)
1417        Res = SelectT2CMOVShiftOp(N, TrueVal, FalseVal,
1418                               ARMCC::getOppositeCondition(CCVal), CCR, InFlag);
1419      if (Res)
1420        return Res;
1421    } else {
1422      SDNode *Res = SelectARMCMOVShiftOp(N, FalseVal, TrueVal,
1423                                         CCVal, CCR, InFlag);
1424      if (!Res)
1425        Res = SelectARMCMOVShiftOp(N, TrueVal, FalseVal,
1426                               ARMCC::getOppositeCondition(CCVal), CCR, InFlag);
1427      if (Res)
1428        return Res;
1429    }
1430
1431    // Pattern: (ARMcmov:i32 GPR:i32:$false,
1432    //             (imm:i32)<<P:Predicate_so_imm>>:$true,
1433    //             (imm:i32):$cc)
1434    // Emits: (MOVCCi:i32 GPR:i32:$false,
1435    //           (so_imm:i32 (imm:i32):$true), (imm:i32):$cc)
1436    // Pattern complexity = 10  cost = 1  size = 0
1437    if (Subtarget->isThumb()) {
1438      SDNode *Res = SelectT2CMOVSoImmOp(N, FalseVal, TrueVal,
1439                                        CCVal, CCR, InFlag);
1440      if (!Res)
1441        Res = SelectT2CMOVSoImmOp(N, TrueVal, FalseVal,
1442                               ARMCC::getOppositeCondition(CCVal), CCR, InFlag);
1443      if (Res)
1444        return Res;
1445    } else {
1446      SDNode *Res = SelectARMCMOVSoImmOp(N, FalseVal, TrueVal,
1447                                         CCVal, CCR, InFlag);
1448      if (!Res)
1449        Res = SelectARMCMOVSoImmOp(N, TrueVal, FalseVal,
1450                               ARMCC::getOppositeCondition(CCVal), CCR, InFlag);
1451      if (Res)
1452        return Res;
1453    }
1454  }
1455
1456  // Pattern: (ARMcmov:i32 GPR:i32:$false, GPR:i32:$true, (imm:i32):$cc)
1457  // Emits: (MOVCCr:i32 GPR:i32:$false, GPR:i32:$true, (imm:i32):$cc)
1458  // Pattern complexity = 6  cost = 1  size = 0
1459  //
1460  // Pattern: (ARMcmov:i32 GPR:i32:$false, GPR:i32:$true, (imm:i32):$cc)
1461  // Emits: (tMOVCCr:i32 GPR:i32:$false, GPR:i32:$true, (imm:i32):$cc)
1462  // Pattern complexity = 6  cost = 11  size = 0
1463  //
1464  // Also FCPYScc and FCPYDcc.
1465  SDValue Tmp2 = CurDAG->getTargetConstant(CCVal, MVT::i32);
1466  SDValue Ops[] = { FalseVal, TrueVal, Tmp2, CCR, InFlag };
1467  unsigned Opc = 0;
1468  switch (VT.getSimpleVT().SimpleTy) {
1469  default: assert(false && "Illegal conditional move type!");
1470    break;
1471  case MVT::i32:
1472    Opc = Subtarget->isThumb()
1473      ? (Subtarget->hasThumb2() ? ARM::t2MOVCCr : ARM::tMOVCCr_pseudo)
1474      : ARM::MOVCCr;
1475    break;
1476  case MVT::f32:
1477    Opc = ARM::VMOVScc;
1478    break;
1479  case MVT::f64:
1480    Opc = ARM::VMOVDcc;
1481    break;
1482  }
1483  return CurDAG->SelectNodeTo(N, Opc, VT, Ops, 5);
1484}
1485
1486SDNode *ARMDAGToDAGISel::SelectConcatVector(SDNode *N) {
1487  // The only time a CONCAT_VECTORS operation can have legal types is when
1488  // two 64-bit vectors are concatenated to a 128-bit vector.
1489  EVT VT = N->getValueType(0);
1490  if (!VT.is128BitVector() || N->getNumOperands() != 2)
1491    llvm_unreachable("unexpected CONCAT_VECTORS");
1492  DebugLoc dl = N->getDebugLoc();
1493  SDValue V0 = N->getOperand(0);
1494  SDValue V1 = N->getOperand(1);
1495  SDValue SubReg0 = CurDAG->getTargetConstant(ARM::DSUBREG_0, MVT::i32);
1496  SDValue SubReg1 = CurDAG->getTargetConstant(ARM::DSUBREG_1, MVT::i32);
1497  const SDValue Ops[] = { V0, SubReg0, V1, SubReg1 };
1498  return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops, 4);
1499}
1500
1501SDNode *ARMDAGToDAGISel::Select(SDNode *N) {
1502  DebugLoc dl = N->getDebugLoc();
1503
1504  if (N->isMachineOpcode())
1505    return NULL;   // Already selected.
1506
1507  switch (N->getOpcode()) {
1508  default: break;
1509  case ISD::Constant: {
1510    unsigned Val = cast<ConstantSDNode>(N)->getZExtValue();
1511    bool UseCP = true;
1512    if (Subtarget->hasThumb2())
1513      // Thumb2-aware targets have the MOVT instruction, so all immediates can
1514      // be done with MOV + MOVT, at worst.
1515      UseCP = 0;
1516    else {
1517      if (Subtarget->isThumb()) {
1518        UseCP = (Val > 255 &&                          // MOV
1519                 ~Val > 255 &&                         // MOV + MVN
1520                 !ARM_AM::isThumbImmShiftedVal(Val));  // MOV + LSL
1521      } else
1522        UseCP = (ARM_AM::getSOImmVal(Val) == -1 &&     // MOV
1523                 ARM_AM::getSOImmVal(~Val) == -1 &&    // MVN
1524                 !ARM_AM::isSOImmTwoPartVal(Val));     // two instrs.
1525    }
1526
1527    if (UseCP) {
1528      SDValue CPIdx =
1529        CurDAG->getTargetConstantPool(ConstantInt::get(
1530                                  Type::getInt32Ty(*CurDAG->getContext()), Val),
1531                                      TLI.getPointerTy());
1532
1533      SDNode *ResNode;
1534      if (Subtarget->isThumb1Only()) {
1535        SDValue Pred = getAL(CurDAG);
1536        SDValue PredReg = CurDAG->getRegister(0, MVT::i32);
1537        SDValue Ops[] = { CPIdx, Pred, PredReg, CurDAG->getEntryNode() };
1538        ResNode = CurDAG->getMachineNode(ARM::tLDRcp, dl, MVT::i32, MVT::Other,
1539                                         Ops, 4);
1540      } else {
1541        SDValue Ops[] = {
1542          CPIdx,
1543          CurDAG->getRegister(0, MVT::i32),
1544          CurDAG->getTargetConstant(0, MVT::i32),
1545          getAL(CurDAG),
1546          CurDAG->getRegister(0, MVT::i32),
1547          CurDAG->getEntryNode()
1548        };
1549        ResNode=CurDAG->getMachineNode(ARM::LDRcp, dl, MVT::i32, MVT::Other,
1550                                       Ops, 6);
1551      }
1552      ReplaceUses(SDValue(N, 0), SDValue(ResNode, 0));
1553      return NULL;
1554    }
1555
1556    // Other cases are autogenerated.
1557    break;
1558  }
1559  case ISD::FrameIndex: {
1560    // Selects to ADDri FI, 0 which in turn will become ADDri SP, imm.
1561    int FI = cast<FrameIndexSDNode>(N)->getIndex();
1562    SDValue TFI = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
1563    if (Subtarget->isThumb1Only()) {
1564      return CurDAG->SelectNodeTo(N, ARM::tADDrSPi, MVT::i32, TFI,
1565                                  CurDAG->getTargetConstant(0, MVT::i32));
1566    } else {
1567      unsigned Opc = ((Subtarget->isThumb() && Subtarget->hasThumb2()) ?
1568                      ARM::t2ADDri : ARM::ADDri);
1569      SDValue Ops[] = { TFI, CurDAG->getTargetConstant(0, MVT::i32),
1570                        getAL(CurDAG), CurDAG->getRegister(0, MVT::i32),
1571                        CurDAG->getRegister(0, MVT::i32) };
1572      return CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops, 5);
1573    }
1574  }
1575  case ISD::SRL:
1576    if (SDNode *I = SelectV6T2BitfieldExtractOp(N, false))
1577      return I;
1578    break;
1579  case ISD::SRA:
1580    if (SDNode *I = SelectV6T2BitfieldExtractOp(N, true))
1581      return I;
1582    break;
1583  case ISD::MUL:
1584    if (Subtarget->isThumb1Only())
1585      break;
1586    if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(N->getOperand(1))) {
1587      unsigned RHSV = C->getZExtValue();
1588      if (!RHSV) break;
1589      if (isPowerOf2_32(RHSV-1)) {  // 2^n+1?
1590        unsigned ShImm = Log2_32(RHSV-1);
1591        if (ShImm >= 32)
1592          break;
1593        SDValue V = N->getOperand(0);
1594        ShImm = ARM_AM::getSORegOpc(ARM_AM::lsl, ShImm);
1595        SDValue ShImmOp = CurDAG->getTargetConstant(ShImm, MVT::i32);
1596        SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
1597        if (Subtarget->isThumb()) {
1598          SDValue Ops[] = { V, V, ShImmOp, getAL(CurDAG), Reg0, Reg0 };
1599          return CurDAG->SelectNodeTo(N, ARM::t2ADDrs, MVT::i32, Ops, 6);
1600        } else {
1601          SDValue Ops[] = { V, V, Reg0, ShImmOp, getAL(CurDAG), Reg0, Reg0 };
1602          return CurDAG->SelectNodeTo(N, ARM::ADDrs, MVT::i32, Ops, 7);
1603        }
1604      }
1605      if (isPowerOf2_32(RHSV+1)) {  // 2^n-1?
1606        unsigned ShImm = Log2_32(RHSV+1);
1607        if (ShImm >= 32)
1608          break;
1609        SDValue V = N->getOperand(0);
1610        ShImm = ARM_AM::getSORegOpc(ARM_AM::lsl, ShImm);
1611        SDValue ShImmOp = CurDAG->getTargetConstant(ShImm, MVT::i32);
1612        SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
1613        if (Subtarget->isThumb()) {
1614          SDValue Ops[] = { V, V, ShImmOp, getAL(CurDAG), Reg0 };
1615          return CurDAG->SelectNodeTo(N, ARM::t2RSBrs, MVT::i32, Ops, 5);
1616        } else {
1617          SDValue Ops[] = { V, V, Reg0, ShImmOp, getAL(CurDAG), Reg0, Reg0 };
1618          return CurDAG->SelectNodeTo(N, ARM::RSBrs, MVT::i32, Ops, 7);
1619        }
1620      }
1621    }
1622    break;
1623  case ISD::AND: {
1624    // Check for unsigned bitfield extract
1625    if (SDNode *I = SelectV6T2BitfieldExtractOp(N, false))
1626      return I;
1627
1628    // (and (or x, c2), c1) and top 16-bits of c1 and c2 match, lower 16-bits
1629    // of c1 are 0xffff, and lower 16-bit of c2 are 0. That is, the top 16-bits
1630    // are entirely contributed by c2 and lower 16-bits are entirely contributed
1631    // by x. That's equal to (or (and x, 0xffff), (and c1, 0xffff0000)).
1632    // Select it to: "movt x, ((c1 & 0xffff) >> 16)
1633    EVT VT = N->getValueType(0);
1634    if (VT != MVT::i32)
1635      break;
1636    unsigned Opc = (Subtarget->isThumb() && Subtarget->hasThumb2())
1637      ? ARM::t2MOVTi16
1638      : (Subtarget->hasV6T2Ops() ? ARM::MOVTi16 : 0);
1639    if (!Opc)
1640      break;
1641    SDValue N0 = N->getOperand(0), N1 = N->getOperand(1);
1642    ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
1643    if (!N1C)
1644      break;
1645    if (N0.getOpcode() == ISD::OR && N0.getNode()->hasOneUse()) {
1646      SDValue N2 = N0.getOperand(1);
1647      ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(N2);
1648      if (!N2C)
1649        break;
1650      unsigned N1CVal = N1C->getZExtValue();
1651      unsigned N2CVal = N2C->getZExtValue();
1652      if ((N1CVal & 0xffff0000U) == (N2CVal & 0xffff0000U) &&
1653          (N1CVal & 0xffffU) == 0xffffU &&
1654          (N2CVal & 0xffffU) == 0x0U) {
1655        SDValue Imm16 = CurDAG->getTargetConstant((N2CVal & 0xFFFF0000U) >> 16,
1656                                                  MVT::i32);
1657        SDValue Ops[] = { N0.getOperand(0), Imm16,
1658                          getAL(CurDAG), CurDAG->getRegister(0, MVT::i32) };
1659        return CurDAG->getMachineNode(Opc, dl, VT, Ops, 4);
1660      }
1661    }
1662    break;
1663  }
1664  case ARMISD::VMOVRRD:
1665    return CurDAG->getMachineNode(ARM::VMOVRRD, dl, MVT::i32, MVT::i32,
1666                                  N->getOperand(0), getAL(CurDAG),
1667                                  CurDAG->getRegister(0, MVT::i32));
1668  case ISD::UMUL_LOHI: {
1669    if (Subtarget->isThumb1Only())
1670      break;
1671    if (Subtarget->isThumb()) {
1672      SDValue Ops[] = { N->getOperand(0), N->getOperand(1),
1673                        getAL(CurDAG), CurDAG->getRegister(0, MVT::i32),
1674                        CurDAG->getRegister(0, MVT::i32) };
1675      return CurDAG->getMachineNode(ARM::t2UMULL, dl, MVT::i32, MVT::i32, Ops,4);
1676    } else {
1677      SDValue Ops[] = { N->getOperand(0), N->getOperand(1),
1678                        getAL(CurDAG), CurDAG->getRegister(0, MVT::i32),
1679                        CurDAG->getRegister(0, MVT::i32) };
1680      return CurDAG->getMachineNode(ARM::UMULL, dl, MVT::i32, MVT::i32, Ops, 5);
1681    }
1682  }
1683  case ISD::SMUL_LOHI: {
1684    if (Subtarget->isThumb1Only())
1685      break;
1686    if (Subtarget->isThumb()) {
1687      SDValue Ops[] = { N->getOperand(0), N->getOperand(1),
1688                        getAL(CurDAG), CurDAG->getRegister(0, MVT::i32) };
1689      return CurDAG->getMachineNode(ARM::t2SMULL, dl, MVT::i32, MVT::i32, Ops,4);
1690    } else {
1691      SDValue Ops[] = { N->getOperand(0), N->getOperand(1),
1692                        getAL(CurDAG), CurDAG->getRegister(0, MVT::i32),
1693                        CurDAG->getRegister(0, MVT::i32) };
1694      return CurDAG->getMachineNode(ARM::SMULL, dl, MVT::i32, MVT::i32, Ops, 5);
1695    }
1696  }
1697  case ISD::LOAD: {
1698    SDNode *ResNode = 0;
1699    if (Subtarget->isThumb() && Subtarget->hasThumb2())
1700      ResNode = SelectT2IndexedLoad(N);
1701    else
1702      ResNode = SelectARMIndexedLoad(N);
1703    if (ResNode)
1704      return ResNode;
1705
1706    // VLDMQ must be custom-selected for "v2f64 load" to set the AM5Opc value.
1707    if (Subtarget->hasVFP2() &&
1708        N->getValueType(0).getSimpleVT().SimpleTy == MVT::v2f64) {
1709      SDValue Chain = N->getOperand(0);
1710      SDValue AM5Opc =
1711        CurDAG->getTargetConstant(ARM_AM::getAM5Opc(ARM_AM::ia, 4), MVT::i32);
1712      SDValue Pred = getAL(CurDAG);
1713      SDValue PredReg = CurDAG->getRegister(0, MVT::i32);
1714      SDValue Ops[] = { N->getOperand(1), AM5Opc, Pred, PredReg, Chain };
1715      return CurDAG->getMachineNode(ARM::VLDMQ, dl, MVT::v2f64, MVT::Other,
1716                                    Ops, 5);
1717    }
1718    // Other cases are autogenerated.
1719    break;
1720  }
1721  case ISD::STORE: {
1722    // VSTMQ must be custom-selected for "v2f64 store" to set the AM5Opc value.
1723    if (Subtarget->hasVFP2() &&
1724        N->getOperand(1).getValueType().getSimpleVT().SimpleTy == MVT::v2f64) {
1725      SDValue Chain = N->getOperand(0);
1726      SDValue AM5Opc =
1727        CurDAG->getTargetConstant(ARM_AM::getAM5Opc(ARM_AM::ia, 4), MVT::i32);
1728      SDValue Pred = getAL(CurDAG);
1729      SDValue PredReg = CurDAG->getRegister(0, MVT::i32);
1730      SDValue Ops[] = { N->getOperand(1), N->getOperand(2),
1731                        AM5Opc, Pred, PredReg, Chain };
1732      return CurDAG->getMachineNode(ARM::VSTMQ, dl, MVT::Other, Ops, 6);
1733    }
1734    // Other cases are autogenerated.
1735    break;
1736  }
1737  case ARMISD::BRCOND: {
1738    // Pattern: (ARMbrcond:void (bb:Other):$dst, (imm:i32):$cc)
1739    // Emits: (Bcc:void (bb:Other):$dst, (imm:i32):$cc)
1740    // Pattern complexity = 6  cost = 1  size = 0
1741
1742    // Pattern: (ARMbrcond:void (bb:Other):$dst, (imm:i32):$cc)
1743    // Emits: (tBcc:void (bb:Other):$dst, (imm:i32):$cc)
1744    // Pattern complexity = 6  cost = 1  size = 0
1745
1746    // Pattern: (ARMbrcond:void (bb:Other):$dst, (imm:i32):$cc)
1747    // Emits: (t2Bcc:void (bb:Other):$dst, (imm:i32):$cc)
1748    // Pattern complexity = 6  cost = 1  size = 0
1749
1750    unsigned Opc = Subtarget->isThumb() ?
1751      ((Subtarget->hasThumb2()) ? ARM::t2Bcc : ARM::tBcc) : ARM::Bcc;
1752    SDValue Chain = N->getOperand(0);
1753    SDValue N1 = N->getOperand(1);
1754    SDValue N2 = N->getOperand(2);
1755    SDValue N3 = N->getOperand(3);
1756    SDValue InFlag = N->getOperand(4);
1757    assert(N1.getOpcode() == ISD::BasicBlock);
1758    assert(N2.getOpcode() == ISD::Constant);
1759    assert(N3.getOpcode() == ISD::Register);
1760
1761    SDValue Tmp2 = CurDAG->getTargetConstant(((unsigned)
1762                               cast<ConstantSDNode>(N2)->getZExtValue()),
1763                               MVT::i32);
1764    SDValue Ops[] = { N1, Tmp2, N3, Chain, InFlag };
1765    SDNode *ResNode = CurDAG->getMachineNode(Opc, dl, MVT::Other,
1766                                             MVT::Flag, Ops, 5);
1767    Chain = SDValue(ResNode, 0);
1768    if (N->getNumValues() == 2) {
1769      InFlag = SDValue(ResNode, 1);
1770      ReplaceUses(SDValue(N, 1), InFlag);
1771    }
1772    ReplaceUses(SDValue(N, 0),
1773                SDValue(Chain.getNode(), Chain.getResNo()));
1774    return NULL;
1775  }
1776  case ARMISD::CMOV:
1777    return SelectCMOVOp(N);
1778  case ARMISD::CNEG: {
1779    EVT VT = N->getValueType(0);
1780    SDValue N0 = N->getOperand(0);
1781    SDValue N1 = N->getOperand(1);
1782    SDValue N2 = N->getOperand(2);
1783    SDValue N3 = N->getOperand(3);
1784    SDValue InFlag = N->getOperand(4);
1785    assert(N2.getOpcode() == ISD::Constant);
1786    assert(N3.getOpcode() == ISD::Register);
1787
1788    SDValue Tmp2 = CurDAG->getTargetConstant(((unsigned)
1789                               cast<ConstantSDNode>(N2)->getZExtValue()),
1790                               MVT::i32);
1791    SDValue Ops[] = { N0, N1, Tmp2, N3, InFlag };
1792    unsigned Opc = 0;
1793    switch (VT.getSimpleVT().SimpleTy) {
1794    default: assert(false && "Illegal conditional move type!");
1795      break;
1796    case MVT::f32:
1797      Opc = ARM::VNEGScc;
1798      break;
1799    case MVT::f64:
1800      Opc = ARM::VNEGDcc;
1801      break;
1802    }
1803    return CurDAG->SelectNodeTo(N, Opc, VT, Ops, 5);
1804  }
1805
1806  case ARMISD::VZIP: {
1807    unsigned Opc = 0;
1808    EVT VT = N->getValueType(0);
1809    switch (VT.getSimpleVT().SimpleTy) {
1810    default: return NULL;
1811    case MVT::v8i8:  Opc = ARM::VZIPd8; break;
1812    case MVT::v4i16: Opc = ARM::VZIPd16; break;
1813    case MVT::v2f32:
1814    case MVT::v2i32: Opc = ARM::VZIPd32; break;
1815    case MVT::v16i8: Opc = ARM::VZIPq8; break;
1816    case MVT::v8i16: Opc = ARM::VZIPq16; break;
1817    case MVT::v4f32:
1818    case MVT::v4i32: Opc = ARM::VZIPq32; break;
1819    }
1820    SDValue Pred = getAL(CurDAG);
1821    SDValue PredReg = CurDAG->getRegister(0, MVT::i32);
1822    SDValue Ops[] = { N->getOperand(0), N->getOperand(1), Pred, PredReg };
1823    return CurDAG->getMachineNode(Opc, dl, VT, VT, Ops, 4);
1824  }
1825  case ARMISD::VUZP: {
1826    unsigned Opc = 0;
1827    EVT VT = N->getValueType(0);
1828    switch (VT.getSimpleVT().SimpleTy) {
1829    default: return NULL;
1830    case MVT::v8i8:  Opc = ARM::VUZPd8; break;
1831    case MVT::v4i16: Opc = ARM::VUZPd16; break;
1832    case MVT::v2f32:
1833    case MVT::v2i32: Opc = ARM::VUZPd32; break;
1834    case MVT::v16i8: Opc = ARM::VUZPq8; break;
1835    case MVT::v8i16: Opc = ARM::VUZPq16; break;
1836    case MVT::v4f32:
1837    case MVT::v4i32: Opc = ARM::VUZPq32; break;
1838    }
1839    SDValue Pred = getAL(CurDAG);
1840    SDValue PredReg = CurDAG->getRegister(0, MVT::i32);
1841    SDValue Ops[] = { N->getOperand(0), N->getOperand(1), Pred, PredReg };
1842    return CurDAG->getMachineNode(Opc, dl, VT, VT, Ops, 4);
1843  }
1844  case ARMISD::VTRN: {
1845    unsigned Opc = 0;
1846    EVT VT = N->getValueType(0);
1847    switch (VT.getSimpleVT().SimpleTy) {
1848    default: return NULL;
1849    case MVT::v8i8:  Opc = ARM::VTRNd8; break;
1850    case MVT::v4i16: Opc = ARM::VTRNd16; break;
1851    case MVT::v2f32:
1852    case MVT::v2i32: Opc = ARM::VTRNd32; break;
1853    case MVT::v16i8: Opc = ARM::VTRNq8; break;
1854    case MVT::v8i16: Opc = ARM::VTRNq16; break;
1855    case MVT::v4f32:
1856    case MVT::v4i32: Opc = ARM::VTRNq32; break;
1857    }
1858    SDValue Pred = getAL(CurDAG);
1859    SDValue PredReg = CurDAG->getRegister(0, MVT::i32);
1860    SDValue Ops[] = { N->getOperand(0), N->getOperand(1), Pred, PredReg };
1861    return CurDAG->getMachineNode(Opc, dl, VT, VT, Ops, 4);
1862  }
1863
1864  case ISD::INTRINSIC_VOID:
1865  case ISD::INTRINSIC_W_CHAIN: {
1866    unsigned IntNo = cast<ConstantSDNode>(N->getOperand(1))->getZExtValue();
1867    switch (IntNo) {
1868    default:
1869      break;
1870
1871    case Intrinsic::arm_neon_vld1: {
1872      unsigned DOpcodes[] = { ARM::VLD1d8, ARM::VLD1d16,
1873                              ARM::VLD1d32, ARM::VLD1d64 };
1874      unsigned QOpcodes[] = { ARM::VLD1q8, ARM::VLD1q16,
1875                              ARM::VLD1q32, ARM::VLD1q64 };
1876      return SelectVLD(N, 1, DOpcodes, QOpcodes, 0);
1877    }
1878
1879    case Intrinsic::arm_neon_vld2: {
1880      unsigned DOpcodes[] = { ARM::VLD2d8, ARM::VLD2d16,
1881                              ARM::VLD2d32, ARM::VLD1q64 };
1882      unsigned QOpcodes[] = { ARM::VLD2q8, ARM::VLD2q16, ARM::VLD2q32 };
1883      return SelectVLD(N, 2, DOpcodes, QOpcodes, 0);
1884    }
1885
1886    case Intrinsic::arm_neon_vld3: {
1887      unsigned DOpcodes[] = { ARM::VLD3d8, ARM::VLD3d16,
1888                              ARM::VLD3d32, ARM::VLD1d64T };
1889      unsigned QOpcodes0[] = { ARM::VLD3q8_UPD,
1890                               ARM::VLD3q16_UPD,
1891                               ARM::VLD3q32_UPD };
1892      unsigned QOpcodes1[] = { ARM::VLD3q8odd_UPD,
1893                               ARM::VLD3q16odd_UPD,
1894                               ARM::VLD3q32odd_UPD };
1895      return SelectVLD(N, 3, DOpcodes, QOpcodes0, QOpcodes1);
1896    }
1897
1898    case Intrinsic::arm_neon_vld4: {
1899      unsigned DOpcodes[] = { ARM::VLD4d8, ARM::VLD4d16,
1900                              ARM::VLD4d32, ARM::VLD1d64Q };
1901      unsigned QOpcodes0[] = { ARM::VLD4q8_UPD,
1902                               ARM::VLD4q16_UPD,
1903                               ARM::VLD4q32_UPD };
1904      unsigned QOpcodes1[] = { ARM::VLD4q8odd_UPD,
1905                               ARM::VLD4q16odd_UPD,
1906                               ARM::VLD4q32odd_UPD };
1907      return SelectVLD(N, 4, DOpcodes, QOpcodes0, QOpcodes1);
1908    }
1909
1910    case Intrinsic::arm_neon_vld2lane: {
1911      unsigned DOpcodes[] = { ARM::VLD2LNd8, ARM::VLD2LNd16, ARM::VLD2LNd32 };
1912      unsigned QOpcodes0[] = { ARM::VLD2LNq16, ARM::VLD2LNq32 };
1913      unsigned QOpcodes1[] = { ARM::VLD2LNq16odd, ARM::VLD2LNq32odd };
1914      return SelectVLDSTLane(N, true, 2, DOpcodes, QOpcodes0, QOpcodes1);
1915    }
1916
1917    case Intrinsic::arm_neon_vld3lane: {
1918      unsigned DOpcodes[] = { ARM::VLD3LNd8, ARM::VLD3LNd16, ARM::VLD3LNd32 };
1919      unsigned QOpcodes0[] = { ARM::VLD3LNq16, ARM::VLD3LNq32 };
1920      unsigned QOpcodes1[] = { ARM::VLD3LNq16odd, ARM::VLD3LNq32odd };
1921      return SelectVLDSTLane(N, true, 3, DOpcodes, QOpcodes0, QOpcodes1);
1922    }
1923
1924    case Intrinsic::arm_neon_vld4lane: {
1925      unsigned DOpcodes[] = { ARM::VLD4LNd8, ARM::VLD4LNd16, ARM::VLD4LNd32 };
1926      unsigned QOpcodes0[] = { ARM::VLD4LNq16, ARM::VLD4LNq32 };
1927      unsigned QOpcodes1[] = { ARM::VLD4LNq16odd, ARM::VLD4LNq32odd };
1928      return SelectVLDSTLane(N, true, 4, DOpcodes, QOpcodes0, QOpcodes1);
1929    }
1930
1931    case Intrinsic::arm_neon_vst1: {
1932      unsigned DOpcodes[] = { ARM::VST1d8, ARM::VST1d16,
1933                              ARM::VST1d32, ARM::VST1d64 };
1934      unsigned QOpcodes[] = { ARM::VST1q8, ARM::VST1q16,
1935                              ARM::VST1q32, ARM::VST1q64 };
1936      return SelectVST(N, 1, DOpcodes, QOpcodes, 0);
1937    }
1938
1939    case Intrinsic::arm_neon_vst2: {
1940      unsigned DOpcodes[] = { ARM::VST2d8, ARM::VST2d16,
1941                              ARM::VST2d32, ARM::VST1q64 };
1942      unsigned QOpcodes[] = { ARM::VST2q8, ARM::VST2q16, ARM::VST2q32 };
1943      return SelectVST(N, 2, DOpcodes, QOpcodes, 0);
1944    }
1945
1946    case Intrinsic::arm_neon_vst3: {
1947      unsigned DOpcodes[] = { ARM::VST3d8, ARM::VST3d16,
1948                              ARM::VST3d32, ARM::VST1d64T };
1949      unsigned QOpcodes0[] = { ARM::VST3q8_UPD,
1950                               ARM::VST3q16_UPD,
1951                               ARM::VST3q32_UPD };
1952      unsigned QOpcodes1[] = { ARM::VST3q8odd_UPD,
1953                               ARM::VST3q16odd_UPD,
1954                               ARM::VST3q32odd_UPD };
1955      return SelectVST(N, 3, DOpcodes, QOpcodes0, QOpcodes1);
1956    }
1957
1958    case Intrinsic::arm_neon_vst4: {
1959      unsigned DOpcodes[] = { ARM::VST4d8, ARM::VST4d16,
1960                              ARM::VST4d32, ARM::VST1d64Q };
1961      unsigned QOpcodes0[] = { ARM::VST4q8_UPD,
1962                               ARM::VST4q16_UPD,
1963                               ARM::VST4q32_UPD };
1964      unsigned QOpcodes1[] = { ARM::VST4q8odd_UPD,
1965                               ARM::VST4q16odd_UPD,
1966                               ARM::VST4q32odd_UPD };
1967      return SelectVST(N, 4, DOpcodes, QOpcodes0, QOpcodes1);
1968    }
1969
1970    case Intrinsic::arm_neon_vst2lane: {
1971      unsigned DOpcodes[] = { ARM::VST2LNd8, ARM::VST2LNd16, ARM::VST2LNd32 };
1972      unsigned QOpcodes0[] = { ARM::VST2LNq16, ARM::VST2LNq32 };
1973      unsigned QOpcodes1[] = { ARM::VST2LNq16odd, ARM::VST2LNq32odd };
1974      return SelectVLDSTLane(N, false, 2, DOpcodes, QOpcodes0, QOpcodes1);
1975    }
1976
1977    case Intrinsic::arm_neon_vst3lane: {
1978      unsigned DOpcodes[] = { ARM::VST3LNd8, ARM::VST3LNd16, ARM::VST3LNd32 };
1979      unsigned QOpcodes0[] = { ARM::VST3LNq16, ARM::VST3LNq32 };
1980      unsigned QOpcodes1[] = { ARM::VST3LNq16odd, ARM::VST3LNq32odd };
1981      return SelectVLDSTLane(N, false, 3, DOpcodes, QOpcodes0, QOpcodes1);
1982    }
1983
1984    case Intrinsic::arm_neon_vst4lane: {
1985      unsigned DOpcodes[] = { ARM::VST4LNd8, ARM::VST4LNd16, ARM::VST4LNd32 };
1986      unsigned QOpcodes0[] = { ARM::VST4LNq16, ARM::VST4LNq32 };
1987      unsigned QOpcodes1[] = { ARM::VST4LNq16odd, ARM::VST4LNq32odd };
1988      return SelectVLDSTLane(N, false, 4, DOpcodes, QOpcodes0, QOpcodes1);
1989    }
1990    }
1991  }
1992
1993  case ISD::CONCAT_VECTORS: {
1994    return SelectConcatVector(N);
1995  }
1996  }
1997
1998  return SelectCode(N);
1999}
2000
2001bool ARMDAGToDAGISel::
2002SelectInlineAsmMemoryOperand(const SDValue &Op, char ConstraintCode,
2003                             std::vector<SDValue> &OutOps) {
2004  assert(ConstraintCode == 'm' && "unexpected asm memory constraint");
2005  // Require the address to be in a register.  That is safe for all ARM
2006  // variants and it is hard to do anything much smarter without knowing
2007  // how the operand is used.
2008  OutOps.push_back(Op);
2009  return false;
2010}
2011
2012/// createARMISelDag - This pass converts a legalized DAG into a
2013/// ARM-specific DAG, ready for instruction scheduling.
2014///
2015FunctionPass *llvm::createARMISelDag(ARMBaseTargetMachine &TM,
2016                                     CodeGenOpt::Level OptLevel) {
2017  return new ARMDAGToDAGISel(TM, OptLevel);
2018}
2019
2020/// ModelWithRegSequence - Return true if isel should use REG_SEQUENCE to model
2021/// operations involving sub-registers.
2022bool llvm::ModelWithRegSequence() {
2023  return UseRegSeq;
2024}
2025