ARMISelDAGToDAG.cpp revision af4a891273b82798e97ad3e59359d93f3c8d6c19 
1//===-- ARMISelDAGToDAG.cpp - A dag to dag inst selector for ARM ----------===//
2//
3//                     The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10// This file defines an instruction selector for the ARM target.
11//
12//===----------------------------------------------------------------------===//
13
14#include "ARM.h"
15#include "ARMAddressingModes.h"
16#include "ARMConstantPoolValue.h"
17#include "ARMISelLowering.h"
18#include "ARMTargetMachine.h"
19#include "llvm/CallingConv.h"
20#include "llvm/Constants.h"
21#include "llvm/DerivedTypes.h"
22#include "llvm/Function.h"
23#include "llvm/Intrinsics.h"
24#include "llvm/LLVMContext.h"
25#include "llvm/CodeGen/MachineFrameInfo.h"
26#include "llvm/CodeGen/MachineFunction.h"
27#include "llvm/CodeGen/MachineInstrBuilder.h"
28#include "llvm/CodeGen/SelectionDAG.h"
29#include "llvm/CodeGen/SelectionDAGISel.h"
30#include "llvm/Target/TargetLowering.h"
31#include "llvm/Target/TargetOptions.h"
32#include "llvm/Support/Compiler.h"
33#include "llvm/Support/Debug.h"
34#include "llvm/Support/ErrorHandling.h"
35#include "llvm/Support/raw_ostream.h"
36
37using namespace llvm;
38
39//===--------------------------------------------------------------------===//
40/// ARMDAGToDAGISel - ARM specific code to select ARM machine
41/// instructions for SelectionDAG operations.
42///
43namespace {
44class ARMDAGToDAGISel : public SelectionDAGISel {
45  ARMBaseTargetMachine &TM;
46
47  /// Subtarget - Keep a pointer to the ARMSubtarget around so that we can
48  /// make the right decision when generating code for different targets.
49  const ARMSubtarget *Subtarget;
50
51public:
52  explicit ARMDAGToDAGISel(ARMBaseTargetMachine &tm,
53                           CodeGenOpt::Level OptLevel)
54    : SelectionDAGISel(tm, OptLevel), TM(tm),
55    Subtarget(&TM.getSubtarget<ARMSubtarget>()) {
56  }
57
58  virtual const char *getPassName() const {
59    return "ARM Instruction Selection";
60  }
61
62  /// getI32Imm - Return a target constant of type i32 with the specified
63  /// value.
64  inline SDValue getI32Imm(unsigned Imm) {
65    return CurDAG->getTargetConstant(Imm, MVT::i32);
66  }
67
68  SDNode *Select(SDValue Op);
69  virtual void InstructionSelect();
70  bool SelectShifterOperandReg(SDValue Op, SDValue N, SDValue &A,
71                               SDValue &B, SDValue &C);
72  bool SelectAddrMode2(SDValue Op, SDValue N, SDValue &Base,
73                       SDValue &Offset, SDValue &Opc);
74  bool SelectAddrMode2Offset(SDValue Op, SDValue N,
75                             SDValue &Offset, SDValue &Opc);
76  bool SelectAddrMode3(SDValue Op, SDValue N, SDValue &Base,
77                       SDValue &Offset, SDValue &Opc);
78  bool SelectAddrMode3Offset(SDValue Op, SDValue N,
79                             SDValue &Offset, SDValue &Opc);
80  bool SelectAddrMode4(SDValue Op, SDValue N, SDValue &Addr,
81                       SDValue &Mode);
82  bool SelectAddrMode5(SDValue Op, SDValue N, SDValue &Base,
83                       SDValue &Offset);
84  bool SelectAddrMode6(SDValue Op, SDValue N, SDValue &Addr, SDValue &Update,
85                       SDValue &Opc);
86
87  bool SelectAddrModePC(SDValue Op, SDValue N, SDValue &Offset,
88                        SDValue &Label);
89
90  bool SelectThumbAddrModeRR(SDValue Op, SDValue N, SDValue &Base,
91                             SDValue &Offset);
92  bool SelectThumbAddrModeRI5(SDValue Op, SDValue N, unsigned Scale,
93                              SDValue &Base, SDValue &OffImm,
94                              SDValue &Offset);
95  bool SelectThumbAddrModeS1(SDValue Op, SDValue N, SDValue &Base,
96                             SDValue &OffImm, SDValue &Offset);
97  bool SelectThumbAddrModeS2(SDValue Op, SDValue N, SDValue &Base,
98                             SDValue &OffImm, SDValue &Offset);
99  bool SelectThumbAddrModeS4(SDValue Op, SDValue N, SDValue &Base,
100                             SDValue &OffImm, SDValue &Offset);
101  bool SelectThumbAddrModeSP(SDValue Op, SDValue N, SDValue &Base,
102                             SDValue &OffImm);
103
104  bool SelectT2ShifterOperandReg(SDValue Op, SDValue N,
105                                 SDValue &BaseReg, SDValue &Opc);
106  bool SelectT2AddrModeImm12(SDValue Op, SDValue N, SDValue &Base,
107                             SDValue &OffImm);
108  bool SelectT2AddrModeImm8(SDValue Op, SDValue N, SDValue &Base,
109                            SDValue &OffImm);
110  bool SelectT2AddrModeImm8Offset(SDValue Op, SDValue N,
111                                 SDValue &OffImm);
112  bool SelectT2AddrModeImm8s4(SDValue Op, SDValue N, SDValue &Base,
113                              SDValue &OffImm);
114  bool SelectT2AddrModeSoReg(SDValue Op, SDValue N, SDValue &Base,
115                             SDValue &OffReg, SDValue &ShImm);
116
117  // Include the pieces autogenerated from the target description.
118#include "ARMGenDAGISel.inc"
119
120private:
121  /// SelectARMIndexedLoad - Indexed (pre/post inc/dec) load matching code for
122  /// ARM.
123  SDNode *SelectARMIndexedLoad(SDValue Op);
124  SDNode *SelectT2IndexedLoad(SDValue Op);
125
126  /// SelectDYN_ALLOC - Select dynamic alloc for Thumb.
127  SDNode *SelectDYN_ALLOC(SDValue Op);
128
129  /// SelectInlineAsmMemoryOperand - Implement addressing mode selection for
130  /// inline asm expressions.
131  virtual bool SelectInlineAsmMemoryOperand(const SDValue &Op,
132                                            char ConstraintCode,
133                                            std::vector<SDValue> &OutOps);
134
135  /// PairDRegs - Insert a pair of double registers into an implicit def to
136  /// form a quad register.
137  SDNode *PairDRegs(EVT VT, SDValue V0, SDValue V1);
138};
139}
140
141void ARMDAGToDAGISel::InstructionSelect() {
142  DEBUG(BB->dump());
143
144  SelectRoot(*CurDAG);
145  CurDAG->RemoveDeadNodes();
146}
147
148bool ARMDAGToDAGISel::SelectShifterOperandReg(SDValue Op,
149                                              SDValue N,
150                                              SDValue &BaseReg,
151                                              SDValue &ShReg,
152                                              SDValue &Opc) {
153  ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(N);
154
155  // Don't match base register only case. That is matched to a separate
156  // lower complexity pattern with explicit register operand.
157  if (ShOpcVal == ARM_AM::no_shift) return false;
158
159  BaseReg = N.getOperand(0);
160  unsigned ShImmVal = 0;
161  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
162    ShReg = CurDAG->getRegister(0, MVT::i32);
163    ShImmVal = RHS->getZExtValue() & 31;
164  } else {
165    ShReg = N.getOperand(1);
166  }
167  Opc = CurDAG->getTargetConstant(ARM_AM::getSORegOpc(ShOpcVal, ShImmVal),
168                                  MVT::i32);
169  return true;
170}
171
172bool ARMDAGToDAGISel::SelectAddrMode2(SDValue Op, SDValue N,
173                                      SDValue &Base, SDValue &Offset,
174                                      SDValue &Opc) {
175  if (N.getOpcode() == ISD::MUL) {
176    if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
177      // X * [3,5,9] -> X + X * [2,4,8] etc.
178      int RHSC = (int)RHS->getZExtValue();
179      if (RHSC & 1) {
180        RHSC = RHSC & ~1;
181        ARM_AM::AddrOpc AddSub = ARM_AM::add;
182        if (RHSC < 0) {
183          AddSub = ARM_AM::sub;
184          RHSC = - RHSC;
185        }
186        if (isPowerOf2_32(RHSC)) {
187          unsigned ShAmt = Log2_32(RHSC);
188          Base = Offset = N.getOperand(0);
189          Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, ShAmt,
190                                                            ARM_AM::lsl),
191                                          MVT::i32);
192          return true;
193        }
194      }
195    }
196  }
197
198  if (N.getOpcode() != ISD::ADD && N.getOpcode() != ISD::SUB) {
199    Base = N;
200    if (N.getOpcode() == ISD::FrameIndex) {
201      int FI = cast<FrameIndexSDNode>(N)->getIndex();
202      Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
203    } else if (N.getOpcode() == ARMISD::Wrapper) {
204      Base = N.getOperand(0);
205    }
206    Offset = CurDAG->getRegister(0, MVT::i32);
207    Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(ARM_AM::add, 0,
208                                                      ARM_AM::no_shift),
209                                    MVT::i32);
210    return true;
211  }
212
213  // Match simple R +/- imm12 operands.
214  if (N.getOpcode() == ISD::ADD)
215    if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
216      int RHSC = (int)RHS->getZExtValue();
217      if ((RHSC >= 0 && RHSC < 0x1000) ||
218          (RHSC < 0 && RHSC > -0x1000)) { // 12 bits.
219        Base = N.getOperand(0);
220        if (Base.getOpcode() == ISD::FrameIndex) {
221          int FI = cast<FrameIndexSDNode>(Base)->getIndex();
222          Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
223        }
224        Offset = CurDAG->getRegister(0, MVT::i32);
225
226        ARM_AM::AddrOpc AddSub = ARM_AM::add;
227        if (RHSC < 0) {
228          AddSub = ARM_AM::sub;
229          RHSC = - RHSC;
230        }
231        Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, RHSC,
232                                                          ARM_AM::no_shift),
233                                        MVT::i32);
234        return true;
235      }
236    }
237
238  // Otherwise this is R +/- [possibly shifted] R
239  ARM_AM::AddrOpc AddSub = N.getOpcode() == ISD::ADD ? ARM_AM::add:ARM_AM::sub;
240  ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(N.getOperand(1));
241  unsigned ShAmt = 0;
242
243  Base   = N.getOperand(0);
244  Offset = N.getOperand(1);
245
246  if (ShOpcVal != ARM_AM::no_shift) {
247    // Check to see if the RHS of the shift is a constant, if not, we can't fold
248    // it.
249    if (ConstantSDNode *Sh =
250           dyn_cast<ConstantSDNode>(N.getOperand(1).getOperand(1))) {
251      ShAmt = Sh->getZExtValue();
252      Offset = N.getOperand(1).getOperand(0);
253    } else {
254      ShOpcVal = ARM_AM::no_shift;
255    }
256  }
257
258  // Try matching (R shl C) + (R).
259  if (N.getOpcode() == ISD::ADD && ShOpcVal == ARM_AM::no_shift) {
260    ShOpcVal = ARM_AM::getShiftOpcForNode(N.getOperand(0));
261    if (ShOpcVal != ARM_AM::no_shift) {
262      // Check to see if the RHS of the shift is a constant, if not, we can't
263      // fold it.
264      if (ConstantSDNode *Sh =
265          dyn_cast<ConstantSDNode>(N.getOperand(0).getOperand(1))) {
266        ShAmt = Sh->getZExtValue();
267        Offset = N.getOperand(0).getOperand(0);
268        Base = N.getOperand(1);
269      } else {
270        ShOpcVal = ARM_AM::no_shift;
271      }
272    }
273  }
274
275  Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, ShAmt, ShOpcVal),
276                                  MVT::i32);
277  return true;
278}
279
280bool ARMDAGToDAGISel::SelectAddrMode2Offset(SDValue Op, SDValue N,
281                                            SDValue &Offset, SDValue &Opc) {
282  unsigned Opcode = Op.getOpcode();
283  ISD::MemIndexedMode AM = (Opcode == ISD::LOAD)
284    ? cast<LoadSDNode>(Op)->getAddressingMode()
285    : cast<StoreSDNode>(Op)->getAddressingMode();
286  ARM_AM::AddrOpc AddSub = (AM == ISD::PRE_INC || AM == ISD::POST_INC)
287    ? ARM_AM::add : ARM_AM::sub;
288  if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(N)) {
289    int Val = (int)C->getZExtValue();
290    if (Val >= 0 && Val < 0x1000) { // 12 bits.
291      Offset = CurDAG->getRegister(0, MVT::i32);
292      Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, Val,
293                                                        ARM_AM::no_shift),
294                                      MVT::i32);
295      return true;
296    }
297  }
298
299  Offset = N;
300  ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(N);
301  unsigned ShAmt = 0;
302  if (ShOpcVal != ARM_AM::no_shift) {
303    // Check to see if the RHS of the shift is a constant, if not, we can't fold
304    // it.
305    if (ConstantSDNode *Sh = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
306      ShAmt = Sh->getZExtValue();
307      Offset = N.getOperand(0);
308    } else {
309      ShOpcVal = ARM_AM::no_shift;
310    }
311  }
312
313  Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, ShAmt, ShOpcVal),
314                                  MVT::i32);
315  return true;
316}
317
318
319bool ARMDAGToDAGISel::SelectAddrMode3(SDValue Op, SDValue N,
320                                      SDValue &Base, SDValue &Offset,
321                                      SDValue &Opc) {
322  if (N.getOpcode() == ISD::SUB) {
323    // X - C  is canonicalize to X + -C, no need to handle it here.
324    Base = N.getOperand(0);
325    Offset = N.getOperand(1);
326    Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(ARM_AM::sub, 0),MVT::i32);
327    return true;
328  }
329
330  if (N.getOpcode() != ISD::ADD) {
331    Base = N;
332    if (N.getOpcode() == ISD::FrameIndex) {
333      int FI = cast<FrameIndexSDNode>(N)->getIndex();
334      Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
335    }
336    Offset = CurDAG->getRegister(0, MVT::i32);
337    Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(ARM_AM::add, 0),MVT::i32);
338    return true;
339  }
340
341  // If the RHS is +/- imm8, fold into addr mode.
342  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
343    int RHSC = (int)RHS->getZExtValue();
344    if ((RHSC >= 0 && RHSC < 256) ||
345        (RHSC < 0 && RHSC > -256)) { // note -256 itself isn't allowed.
346      Base = N.getOperand(0);
347      if (Base.getOpcode() == ISD::FrameIndex) {
348        int FI = cast<FrameIndexSDNode>(Base)->getIndex();
349        Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
350      }
351      Offset = CurDAG->getRegister(0, MVT::i32);
352
353      ARM_AM::AddrOpc AddSub = ARM_AM::add;
354      if (RHSC < 0) {
355        AddSub = ARM_AM::sub;
356        RHSC = - RHSC;
357      }
358      Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(AddSub, RHSC),MVT::i32);
359      return true;
360    }
361  }
362
363  Base = N.getOperand(0);
364  Offset = N.getOperand(1);
365  Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(ARM_AM::add, 0), MVT::i32);
366  return true;
367}
368
369bool ARMDAGToDAGISel::SelectAddrMode3Offset(SDValue Op, SDValue N,
370                                            SDValue &Offset, SDValue &Opc) {
371  unsigned Opcode = Op.getOpcode();
372  ISD::MemIndexedMode AM = (Opcode == ISD::LOAD)
373    ? cast<LoadSDNode>(Op)->getAddressingMode()
374    : cast<StoreSDNode>(Op)->getAddressingMode();
375  ARM_AM::AddrOpc AddSub = (AM == ISD::PRE_INC || AM == ISD::POST_INC)
376    ? ARM_AM::add : ARM_AM::sub;
377  if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(N)) {
378    int Val = (int)C->getZExtValue();
379    if (Val >= 0 && Val < 256) {
380      Offset = CurDAG->getRegister(0, MVT::i32);
381      Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(AddSub, Val), MVT::i32);
382      return true;
383    }
384  }
385
386  Offset = N;
387  Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(AddSub, 0), MVT::i32);
388  return true;
389}
390
391bool ARMDAGToDAGISel::SelectAddrMode4(SDValue Op, SDValue N,
392                                      SDValue &Addr, SDValue &Mode) {
393  Addr = N;
394  Mode = CurDAG->getTargetConstant(0, MVT::i32);
395  return true;
396}
397
398bool ARMDAGToDAGISel::SelectAddrMode5(SDValue Op, SDValue N,
399                                      SDValue &Base, SDValue &Offset) {
400  if (N.getOpcode() != ISD::ADD) {
401    Base = N;
402    if (N.getOpcode() == ISD::FrameIndex) {
403      int FI = cast<FrameIndexSDNode>(N)->getIndex();
404      Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
405    } else if (N.getOpcode() == ARMISD::Wrapper) {
406      Base = N.getOperand(0);
407    }
408    Offset = CurDAG->getTargetConstant(ARM_AM::getAM5Opc(ARM_AM::add, 0),
409                                       MVT::i32);
410    return true;
411  }
412
413  // If the RHS is +/- imm8, fold into addr mode.
414  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
415    int RHSC = (int)RHS->getZExtValue();
416    if ((RHSC & 3) == 0) {  // The constant is implicitly multiplied by 4.
417      RHSC >>= 2;
418      if ((RHSC >= 0 && RHSC < 256) ||
419          (RHSC < 0 && RHSC > -256)) { // note -256 itself isn't allowed.
420        Base = N.getOperand(0);
421        if (Base.getOpcode() == ISD::FrameIndex) {
422          int FI = cast<FrameIndexSDNode>(Base)->getIndex();
423          Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
424        }
425
426        ARM_AM::AddrOpc AddSub = ARM_AM::add;
427        if (RHSC < 0) {
428          AddSub = ARM_AM::sub;
429          RHSC = - RHSC;
430        }
431        Offset = CurDAG->getTargetConstant(ARM_AM::getAM5Opc(AddSub, RHSC),
432                                           MVT::i32);
433        return true;
434      }
435    }
436  }
437
438  Base = N;
439  Offset = CurDAG->getTargetConstant(ARM_AM::getAM5Opc(ARM_AM::add, 0),
440                                     MVT::i32);
441  return true;
442}
443
444bool ARMDAGToDAGISel::SelectAddrMode6(SDValue Op, SDValue N,
445                                      SDValue &Addr, SDValue &Update,
446                                      SDValue &Opc) {
447  Addr = N;
448  // Default to no writeback.
449  Update = CurDAG->getRegister(0, MVT::i32);
450  Opc = CurDAG->getTargetConstant(ARM_AM::getAM6Opc(false), MVT::i32);
451  return true;
452}
453
454bool ARMDAGToDAGISel::SelectAddrModePC(SDValue Op, SDValue N,
455                                       SDValue &Offset, SDValue &Label) {
456  if (N.getOpcode() == ARMISD::PIC_ADD && N.hasOneUse()) {
457    Offset = N.getOperand(0);
458    SDValue N1 = N.getOperand(1);
459    Label  = CurDAG->getTargetConstant(cast<ConstantSDNode>(N1)->getZExtValue(),
460                                       MVT::i32);
461    return true;
462  }
463  return false;
464}
465
466bool ARMDAGToDAGISel::SelectThumbAddrModeRR(SDValue Op, SDValue N,
467                                            SDValue &Base, SDValue &Offset){
468  // FIXME dl should come from the parent load or store, not the address
469  DebugLoc dl = Op.getDebugLoc();
470  if (N.getOpcode() != ISD::ADD) {
471    ConstantSDNode *NC = dyn_cast<ConstantSDNode>(N);
472    if (!NC || NC->getZExtValue() != 0)
473      return false;
474
475    Base = Offset = N;
476    return true;
477  }
478
479  Base = N.getOperand(0);
480  Offset = N.getOperand(1);
481  return true;
482}
483
484bool
485ARMDAGToDAGISel::SelectThumbAddrModeRI5(SDValue Op, SDValue N,
486                                        unsigned Scale, SDValue &Base,
487                                        SDValue &OffImm, SDValue &Offset) {
488  if (Scale == 4) {
489    SDValue TmpBase, TmpOffImm;
490    if (SelectThumbAddrModeSP(Op, N, TmpBase, TmpOffImm))
491      return false;  // We want to select tLDRspi / tSTRspi instead.
492    if (N.getOpcode() == ARMISD::Wrapper &&
493        N.getOperand(0).getOpcode() == ISD::TargetConstantPool)
494      return false;  // We want to select tLDRpci instead.
495  }
496
497  if (N.getOpcode() != ISD::ADD) {
498    Base = (N.getOpcode() == ARMISD::Wrapper) ? N.getOperand(0) : N;
499    Offset = CurDAG->getRegister(0, MVT::i32);
500    OffImm = CurDAG->getTargetConstant(0, MVT::i32);
501    return true;
502  }
503
504  // Thumb does not have [sp, r] address mode.
505  RegisterSDNode *LHSR = dyn_cast<RegisterSDNode>(N.getOperand(0));
506  RegisterSDNode *RHSR = dyn_cast<RegisterSDNode>(N.getOperand(1));
507  if ((LHSR && LHSR->getReg() == ARM::SP) ||
508      (RHSR && RHSR->getReg() == ARM::SP)) {
509    Base = N;
510    Offset = CurDAG->getRegister(0, MVT::i32);
511    OffImm = CurDAG->getTargetConstant(0, MVT::i32);
512    return true;
513  }
514
515  // If the RHS is + imm5 * scale, fold into addr mode.
516  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
517    int RHSC = (int)RHS->getZExtValue();
518    if ((RHSC & (Scale-1)) == 0) {  // The constant is implicitly multiplied.
519      RHSC /= Scale;
520      if (RHSC >= 0 && RHSC < 32) {
521        Base = N.getOperand(0);
522        Offset = CurDAG->getRegister(0, MVT::i32);
523        OffImm = CurDAG->getTargetConstant(RHSC, MVT::i32);
524        return true;
525      }
526    }
527  }
528
529  Base = N.getOperand(0);
530  Offset = N.getOperand(1);
531  OffImm = CurDAG->getTargetConstant(0, MVT::i32);
532  return true;
533}
534
535bool ARMDAGToDAGISel::SelectThumbAddrModeS1(SDValue Op, SDValue N,
536                                            SDValue &Base, SDValue &OffImm,
537                                            SDValue &Offset) {
538  return SelectThumbAddrModeRI5(Op, N, 1, Base, OffImm, Offset);
539}
540
541bool ARMDAGToDAGISel::SelectThumbAddrModeS2(SDValue Op, SDValue N,
542                                            SDValue &Base, SDValue &OffImm,
543                                            SDValue &Offset) {
544  return SelectThumbAddrModeRI5(Op, N, 2, Base, OffImm, Offset);
545}
546
547bool ARMDAGToDAGISel::SelectThumbAddrModeS4(SDValue Op, SDValue N,
548                                            SDValue &Base, SDValue &OffImm,
549                                            SDValue &Offset) {
550  return SelectThumbAddrModeRI5(Op, N, 4, Base, OffImm, Offset);
551}
552
553bool ARMDAGToDAGISel::SelectThumbAddrModeSP(SDValue Op, SDValue N,
554                                           SDValue &Base, SDValue &OffImm) {
555  if (N.getOpcode() == ISD::FrameIndex) {
556    int FI = cast<FrameIndexSDNode>(N)->getIndex();
557    Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
558    OffImm = CurDAG->getTargetConstant(0, MVT::i32);
559    return true;
560  }
561
562  if (N.getOpcode() != ISD::ADD)
563    return false;
564
565  RegisterSDNode *LHSR = dyn_cast<RegisterSDNode>(N.getOperand(0));
566  if (N.getOperand(0).getOpcode() == ISD::FrameIndex ||
567      (LHSR && LHSR->getReg() == ARM::SP)) {
568    // If the RHS is + imm8 * scale, fold into addr mode.
569    if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
570      int RHSC = (int)RHS->getZExtValue();
571      if ((RHSC & 3) == 0) {  // The constant is implicitly multiplied.
572        RHSC >>= 2;
573        if (RHSC >= 0 && RHSC < 256) {
574          Base = N.getOperand(0);
575          if (Base.getOpcode() == ISD::FrameIndex) {
576            int FI = cast<FrameIndexSDNode>(Base)->getIndex();
577            Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
578          }
579          OffImm = CurDAG->getTargetConstant(RHSC, MVT::i32);
580          return true;
581        }
582      }
583    }
584  }
585
586  return false;
587}
588
589bool ARMDAGToDAGISel::SelectT2ShifterOperandReg(SDValue Op, SDValue N,
590                                                SDValue &BaseReg,
591                                                SDValue &Opc) {
592  ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(N);
593
594  // Don't match base register only case. That is matched to a separate
595  // lower complexity pattern with explicit register operand.
596  if (ShOpcVal == ARM_AM::no_shift) return false;
597
598  BaseReg = N.getOperand(0);
599  unsigned ShImmVal = 0;
600  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
601    ShImmVal = RHS->getZExtValue() & 31;
602    Opc = getI32Imm(ARM_AM::getSORegOpc(ShOpcVal, ShImmVal));
603    return true;
604  }
605
606  return false;
607}
608
609bool ARMDAGToDAGISel::SelectT2AddrModeImm12(SDValue Op, SDValue N,
610                                            SDValue &Base, SDValue &OffImm) {
611  // Match simple R + imm12 operands.
612
613  // Base only.
614  if (N.getOpcode() != ISD::ADD && N.getOpcode() != ISD::SUB) {
615    if (N.getOpcode() == ISD::FrameIndex) {
616      // Match frame index...
617      int FI = cast<FrameIndexSDNode>(N)->getIndex();
618      Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
619      OffImm  = CurDAG->getTargetConstant(0, MVT::i32);
620      return true;
621    } else if (N.getOpcode() == ARMISD::Wrapper) {
622      Base = N.getOperand(0);
623      if (Base.getOpcode() == ISD::TargetConstantPool)
624        return false;  // We want to select t2LDRpci instead.
625    } else
626      Base = N;
627    OffImm  = CurDAG->getTargetConstant(0, MVT::i32);
628    return true;
629  }
630
631  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
632    if (SelectT2AddrModeImm8(Op, N, Base, OffImm))
633      // Let t2LDRi8 handle (R - imm8).
634      return false;
635
636    int RHSC = (int)RHS->getZExtValue();
637    if (N.getOpcode() == ISD::SUB)
638      RHSC = -RHSC;
639
640    if (RHSC >= 0 && RHSC < 0x1000) { // 12 bits (unsigned)
641      Base   = N.getOperand(0);
642      if (Base.getOpcode() == ISD::FrameIndex) {
643        int FI = cast<FrameIndexSDNode>(Base)->getIndex();
644        Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
645      }
646      OffImm = CurDAG->getTargetConstant(RHSC, MVT::i32);
647      return true;
648    }
649  }
650
651  // Base only.
652  Base = N;
653  OffImm  = CurDAG->getTargetConstant(0, MVT::i32);
654  return true;
655}
656
657bool ARMDAGToDAGISel::SelectT2AddrModeImm8(SDValue Op, SDValue N,
658                                           SDValue &Base, SDValue &OffImm) {
659  // Match simple R - imm8 operands.
660  if (N.getOpcode() == ISD::ADD || N.getOpcode() == ISD::SUB) {
661    if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
662      int RHSC = (int)RHS->getSExtValue();
663      if (N.getOpcode() == ISD::SUB)
664        RHSC = -RHSC;
665
666      if ((RHSC >= -255) && (RHSC < 0)) { // 8 bits (always negative)
667        Base = N.getOperand(0);
668        if (Base.getOpcode() == ISD::FrameIndex) {
669          int FI = cast<FrameIndexSDNode>(Base)->getIndex();
670          Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
671        }
672        OffImm = CurDAG->getTargetConstant(RHSC, MVT::i32);
673        return true;
674      }
675    }
676  }
677
678  return false;
679}
680
681bool ARMDAGToDAGISel::SelectT2AddrModeImm8Offset(SDValue Op, SDValue N,
682                                                 SDValue &OffImm){
683  unsigned Opcode = Op.getOpcode();
684  ISD::MemIndexedMode AM = (Opcode == ISD::LOAD)
685    ? cast<LoadSDNode>(Op)->getAddressingMode()
686    : cast<StoreSDNode>(Op)->getAddressingMode();
687  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N)) {
688    int RHSC = (int)RHS->getZExtValue();
689    if (RHSC >= 0 && RHSC < 0x100) { // 8 bits.
690      OffImm = ((AM == ISD::PRE_INC) || (AM == ISD::POST_INC))
691        ? CurDAG->getTargetConstant(RHSC, MVT::i32)
692        : CurDAG->getTargetConstant(-RHSC, MVT::i32);
693      return true;
694    }
695  }
696
697  return false;
698}
699
700bool ARMDAGToDAGISel::SelectT2AddrModeImm8s4(SDValue Op, SDValue N,
701                                             SDValue &Base, SDValue &OffImm) {
702  if (N.getOpcode() == ISD::ADD) {
703    if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
704      int RHSC = (int)RHS->getZExtValue();
705      if (((RHSC & 0x3) == 0) &&
706          ((RHSC >= 0 && RHSC < 0x400) || (RHSC < 0 && RHSC > -0x400))) { // 8 bits.
707        Base   = N.getOperand(0);
708        OffImm = CurDAG->getTargetConstant(RHSC, MVT::i32);
709        return true;
710      }
711    }
712  } else if (N.getOpcode() == ISD::SUB) {
713    if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
714      int RHSC = (int)RHS->getZExtValue();
715      if (((RHSC & 0x3) == 0) && (RHSC >= 0 && RHSC < 0x400)) { // 8 bits.
716        Base   = N.getOperand(0);
717        OffImm = CurDAG->getTargetConstant(-RHSC, MVT::i32);
718        return true;
719      }
720    }
721  }
722
723  return false;
724}
725
726bool ARMDAGToDAGISel::SelectT2AddrModeSoReg(SDValue Op, SDValue N,
727                                            SDValue &Base,
728                                            SDValue &OffReg, SDValue &ShImm) {
729  // (R - imm8) should be handled by t2LDRi8. The rest are handled by t2LDRi12.
730  if (N.getOpcode() != ISD::ADD)
731    return false;
732
733  // Leave (R + imm12) for t2LDRi12, (R - imm8) for t2LDRi8.
734  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
735    int RHSC = (int)RHS->getZExtValue();
736    if (RHSC >= 0 && RHSC < 0x1000) // 12 bits (unsigned)
737      return false;
738    else if (RHSC < 0 && RHSC >= -255) // 8 bits
739      return false;
740  }
741
742  // Look for (R + R) or (R + (R << [1,2,3])).
743  unsigned ShAmt = 0;
744  Base   = N.getOperand(0);
745  OffReg = N.getOperand(1);
746
747  // Swap if it is ((R << c) + R).
748  ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(OffReg);
749  if (ShOpcVal != ARM_AM::lsl) {
750    ShOpcVal = ARM_AM::getShiftOpcForNode(Base);
751    if (ShOpcVal == ARM_AM::lsl)
752      std::swap(Base, OffReg);
753  }
754
755  if (ShOpcVal == ARM_AM::lsl) {
756    // Check to see if the RHS of the shift is a constant, if not, we can't fold
757    // it.
758    if (ConstantSDNode *Sh = dyn_cast<ConstantSDNode>(OffReg.getOperand(1))) {
759      ShAmt = Sh->getZExtValue();
760      if (ShAmt >= 4) {
761        ShAmt = 0;
762        ShOpcVal = ARM_AM::no_shift;
763      } else
764        OffReg = OffReg.getOperand(0);
765    } else {
766      ShOpcVal = ARM_AM::no_shift;
767    }
768  }
769
770  ShImm = CurDAG->getTargetConstant(ShAmt, MVT::i32);
771
772  return true;
773}
774
775//===--------------------------------------------------------------------===//
776
777/// getAL - Returns a ARMCC::AL immediate node.
778static inline SDValue getAL(SelectionDAG *CurDAG) {
779  return CurDAG->getTargetConstant((uint64_t)ARMCC::AL, MVT::i32);
780}
781
782SDNode *ARMDAGToDAGISel::SelectARMIndexedLoad(SDValue Op) {
783  LoadSDNode *LD = cast<LoadSDNode>(Op);
784  ISD::MemIndexedMode AM = LD->getAddressingMode();
785  if (AM == ISD::UNINDEXED)
786    return NULL;
787
788  EVT LoadedVT = LD->getMemoryVT();
789  SDValue Offset, AMOpc;
790  bool isPre = (AM == ISD::PRE_INC) || (AM == ISD::PRE_DEC);
791  unsigned Opcode = 0;
792  bool Match = false;
793  if (LoadedVT == MVT::i32 &&
794      SelectAddrMode2Offset(Op, LD->getOffset(), Offset, AMOpc)) {
795    Opcode = isPre ? ARM::LDR_PRE : ARM::LDR_POST;
796    Match = true;
797  } else if (LoadedVT == MVT::i16 &&
798             SelectAddrMode3Offset(Op, LD->getOffset(), Offset, AMOpc)) {
799    Match = true;
800    Opcode = (LD->getExtensionType() == ISD::SEXTLOAD)
801      ? (isPre ? ARM::LDRSH_PRE : ARM::LDRSH_POST)
802      : (isPre ? ARM::LDRH_PRE : ARM::LDRH_POST);
803  } else if (LoadedVT == MVT::i8 || LoadedVT == MVT::i1) {
804    if (LD->getExtensionType() == ISD::SEXTLOAD) {
805      if (SelectAddrMode3Offset(Op, LD->getOffset(), Offset, AMOpc)) {
806        Match = true;
807        Opcode = isPre ? ARM::LDRSB_PRE : ARM::LDRSB_POST;
808      }
809    } else {
810      if (SelectAddrMode2Offset(Op, LD->getOffset(), Offset, AMOpc)) {
811        Match = true;
812        Opcode = isPre ? ARM::LDRB_PRE : ARM::LDRB_POST;
813      }
814    }
815  }
816
817  if (Match) {
818    SDValue Chain = LD->getChain();
819    SDValue Base = LD->getBasePtr();
820    SDValue Ops[]= { Base, Offset, AMOpc, getAL(CurDAG),
821                     CurDAG->getRegister(0, MVT::i32), Chain };
822    return CurDAG->getMachineNode(Opcode, Op.getDebugLoc(), MVT::i32, MVT::i32,
823                                  MVT::Other, Ops, 6);
824  }
825
826  return NULL;
827}
828
829SDNode *ARMDAGToDAGISel::SelectT2IndexedLoad(SDValue Op) {
830  LoadSDNode *LD = cast<LoadSDNode>(Op);
831  ISD::MemIndexedMode AM = LD->getAddressingMode();
832  if (AM == ISD::UNINDEXED)
833    return NULL;
834
835  EVT LoadedVT = LD->getMemoryVT();
836  bool isSExtLd = LD->getExtensionType() == ISD::SEXTLOAD;
837  SDValue Offset;
838  bool isPre = (AM == ISD::PRE_INC) || (AM == ISD::PRE_DEC);
839  unsigned Opcode = 0;
840  bool Match = false;
841  if (SelectT2AddrModeImm8Offset(Op, LD->getOffset(), Offset)) {
842    switch (LoadedVT.getSimpleVT().SimpleTy) {
843    case MVT::i32:
844      Opcode = isPre ? ARM::t2LDR_PRE : ARM::t2LDR_POST;
845      break;
846    case MVT::i16:
847      if (isSExtLd)
848        Opcode = isPre ? ARM::t2LDRSH_PRE : ARM::t2LDRSH_POST;
849      else
850        Opcode = isPre ? ARM::t2LDRH_PRE : ARM::t2LDRH_POST;
851      break;
852    case MVT::i8:
853    case MVT::i1:
854      if (isSExtLd)
855        Opcode = isPre ? ARM::t2LDRSB_PRE : ARM::t2LDRSB_POST;
856      else
857        Opcode = isPre ? ARM::t2LDRB_PRE : ARM::t2LDRB_POST;
858      break;
859    default:
860      return NULL;
861    }
862    Match = true;
863  }
864
865  if (Match) {
866    SDValue Chain = LD->getChain();
867    SDValue Base = LD->getBasePtr();
868    SDValue Ops[]= { Base, Offset, getAL(CurDAG),
869                     CurDAG->getRegister(0, MVT::i32), Chain };
870    return CurDAG->getMachineNode(Opcode, Op.getDebugLoc(), MVT::i32, MVT::i32,
871                                  MVT::Other, Ops, 5);
872  }
873
874  return NULL;
875}
876
877SDNode *ARMDAGToDAGISel::SelectDYN_ALLOC(SDValue Op) {
878  SDNode *N = Op.getNode();
879  DebugLoc dl = N->getDebugLoc();
880  EVT VT = Op.getValueType();
881  SDValue Chain = Op.getOperand(0);
882  SDValue Size = Op.getOperand(1);
883  SDValue Align = Op.getOperand(2);
884  SDValue SP = CurDAG->getRegister(ARM::SP, MVT::i32);
885  int32_t AlignVal = cast<ConstantSDNode>(Align)->getSExtValue();
886  if (AlignVal < 0)
887    // We need to align the stack. Use Thumb1 tAND which is the only thumb
888    // instruction that can read and write SP. This matches to a pseudo
889    // instruction that has a chain to ensure the result is written back to
890    // the stack pointer.
891    SP = SDValue(CurDAG->getMachineNode(ARM::tANDsp, dl, VT, SP, Align), 0);
892
893  bool isC = isa<ConstantSDNode>(Size);
894  uint32_t C = isC ? cast<ConstantSDNode>(Size)->getZExtValue() : ~0UL;
895  // Handle the most common case for both Thumb1 and Thumb2:
896  // tSUBspi - immediate is between 0 ... 508 inclusive.
897  if (C <= 508 && ((C & 3) == 0))
898    // FIXME: tSUBspi encode scale 4 implicitly.
899    return CurDAG->SelectNodeTo(N, ARM::tSUBspi_, VT, MVT::Other, SP,
900                                CurDAG->getTargetConstant(C/4, MVT::i32),
901                                Chain);
902
903  if (Subtarget->isThumb1Only()) {
904    // Use tADDspr since Thumb1 does not have a sub r, sp, r. ARMISelLowering
905    // should have negated the size operand already. FIXME: We can't insert
906    // new target independent node at this stage so we are forced to negate
907    // it earlier. Is there a better solution?
908    return CurDAG->SelectNodeTo(N, ARM::tADDspr_, VT, MVT::Other, SP, Size,
909                                Chain);
910  } else if (Subtarget->isThumb2()) {
911    if (isC && Predicate_t2_so_imm(Size.getNode())) {
912      // t2SUBrSPi
913      SDValue Ops[] = { SP, CurDAG->getTargetConstant(C, MVT::i32), Chain };
914      return CurDAG->SelectNodeTo(N, ARM::t2SUBrSPi_, VT, MVT::Other, Ops, 3);
915    } else if (isC && Predicate_imm0_4095(Size.getNode())) {
916      // t2SUBrSPi12
917      SDValue Ops[] = { SP, CurDAG->getTargetConstant(C, MVT::i32), Chain };
918      return CurDAG->SelectNodeTo(N, ARM::t2SUBrSPi12_, VT, MVT::Other, Ops, 3);
919    } else {
920      // t2SUBrSPs
921      SDValue Ops[] = { SP, Size,
922                        getI32Imm(ARM_AM::getSORegOpc(ARM_AM::lsl,0)), Chain };
923      return CurDAG->SelectNodeTo(N, ARM::t2SUBrSPs_, VT, MVT::Other, Ops, 4);
924    }
925  }
926
927  // FIXME: Add ADD / SUB sp instructions for ARM.
928  return 0;
929}
930
931/// PairDRegs - Insert a pair of double registers into an implicit def to
932/// form a quad register.
933SDNode *ARMDAGToDAGISel::PairDRegs(EVT VT, SDValue V0, SDValue V1) {
934  DebugLoc dl = V0.getNode()->getDebugLoc();
935  SDValue Undef =
936    SDValue(CurDAG->getMachineNode(TargetInstrInfo::IMPLICIT_DEF, dl, VT), 0);
937  SDValue SubReg0 = CurDAG->getTargetConstant(ARM::DSUBREG_0, MVT::i32);
938  SDValue SubReg1 = CurDAG->getTargetConstant(ARM::DSUBREG_1, MVT::i32);
939  SDNode *Pair = CurDAG->getMachineNode(TargetInstrInfo::INSERT_SUBREG, dl,
940                                        VT, Undef, V0, SubReg0);
941  return CurDAG->getMachineNode(TargetInstrInfo::INSERT_SUBREG, dl,
942                                VT, SDValue(Pair, 0), V1, SubReg1);
943}
944
945SDNode *ARMDAGToDAGISel::Select(SDValue Op) {
946  SDNode *N = Op.getNode();
947  DebugLoc dl = N->getDebugLoc();
948
949  if (N->isMachineOpcode())
950    return NULL;   // Already selected.
951
952  switch (N->getOpcode()) {
953  default: break;
954  case ISD::Constant: {
955    unsigned Val = cast<ConstantSDNode>(N)->getZExtValue();
956    bool UseCP = true;
957    if (Subtarget->hasThumb2())
958      // Thumb2-aware targets have the MOVT instruction, so all immediates can
959      // be done with MOV + MOVT, at worst.
960      UseCP = 0;
961    else {
962      if (Subtarget->isThumb()) {
963        UseCP = (Val > 255 &&                          // MOV
964                 ~Val > 255 &&                         // MOV + MVN
965                 !ARM_AM::isThumbImmShiftedVal(Val));  // MOV + LSL
966      } else
967        UseCP = (ARM_AM::getSOImmVal(Val) == -1 &&     // MOV
968                 ARM_AM::getSOImmVal(~Val) == -1 &&    // MVN
969                 !ARM_AM::isSOImmTwoPartVal(Val));     // two instrs.
970    }
971
972    if (UseCP) {
973      SDValue CPIdx =
974        CurDAG->getTargetConstantPool(ConstantInt::get(
975                                  Type::getInt32Ty(*CurDAG->getContext()), Val),
976                                      TLI.getPointerTy());
977
978      SDNode *ResNode;
979      if (Subtarget->isThumb1Only()) {
980        SDValue Pred = CurDAG->getTargetConstant(0xEULL, MVT::i32);
981        SDValue PredReg = CurDAG->getRegister(0, MVT::i32);
982        SDValue Ops[] = { CPIdx, Pred, PredReg, CurDAG->getEntryNode() };
983        ResNode = CurDAG->getMachineNode(ARM::tLDRcp, dl, MVT::i32, MVT::Other,
984                                         Ops, 4);
985      } else {
986        SDValue Ops[] = {
987          CPIdx,
988          CurDAG->getRegister(0, MVT::i32),
989          CurDAG->getTargetConstant(0, MVT::i32),
990          getAL(CurDAG),
991          CurDAG->getRegister(0, MVT::i32),
992          CurDAG->getEntryNode()
993        };
994        ResNode=CurDAG->getMachineNode(ARM::LDRcp, dl, MVT::i32, MVT::Other,
995                                       Ops, 6);
996      }
997      ReplaceUses(Op, SDValue(ResNode, 0));
998      return NULL;
999    }
1000
1001    // Other cases are autogenerated.
1002    break;
1003  }
1004  case ISD::FrameIndex: {
1005    // Selects to ADDri FI, 0 which in turn will become ADDri SP, imm.
1006    int FI = cast<FrameIndexSDNode>(N)->getIndex();
1007    SDValue TFI = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
1008    if (Subtarget->isThumb1Only()) {
1009      return CurDAG->SelectNodeTo(N, ARM::tADDrSPi, MVT::i32, TFI,
1010                                  CurDAG->getTargetConstant(0, MVT::i32));
1011    } else {
1012      unsigned Opc = ((Subtarget->isThumb() && Subtarget->hasThumb2()) ?
1013                      ARM::t2ADDri : ARM::ADDri);
1014      SDValue Ops[] = { TFI, CurDAG->getTargetConstant(0, MVT::i32),
1015                        getAL(CurDAG), CurDAG->getRegister(0, MVT::i32),
1016                        CurDAG->getRegister(0, MVT::i32) };
1017      return CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops, 5);
1018    }
1019  }
1020  case ARMISD::DYN_ALLOC:
1021    return SelectDYN_ALLOC(Op);
1022  case ISD::MUL:
1023    if (Subtarget->isThumb1Only())
1024      break;
1025    if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
1026      unsigned RHSV = C->getZExtValue();
1027      if (!RHSV) break;
1028      if (isPowerOf2_32(RHSV-1)) {  // 2^n+1?
1029        unsigned ShImm = Log2_32(RHSV-1);
1030        if (ShImm >= 32)
1031          break;
1032        SDValue V = Op.getOperand(0);
1033        ShImm = ARM_AM::getSORegOpc(ARM_AM::lsl, ShImm);
1034        SDValue ShImmOp = CurDAG->getTargetConstant(ShImm, MVT::i32);
1035        SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
1036        if (Subtarget->isThumb()) {
1037          SDValue Ops[] = { V, V, ShImmOp, getAL(CurDAG), Reg0, Reg0 };
1038          return CurDAG->SelectNodeTo(N, ARM::t2ADDrs, MVT::i32, Ops, 6);
1039        } else {
1040          SDValue Ops[] = { V, V, Reg0, ShImmOp, getAL(CurDAG), Reg0, Reg0 };
1041          return CurDAG->SelectNodeTo(N, ARM::ADDrs, MVT::i32, Ops, 7);
1042        }
1043      }
1044      if (isPowerOf2_32(RHSV+1)) {  // 2^n-1?
1045        unsigned ShImm = Log2_32(RHSV+1);
1046        if (ShImm >= 32)
1047          break;
1048        SDValue V = Op.getOperand(0);
1049        ShImm = ARM_AM::getSORegOpc(ARM_AM::lsl, ShImm);
1050        SDValue ShImmOp = CurDAG->getTargetConstant(ShImm, MVT::i32);
1051        SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
1052        if (Subtarget->isThumb()) {
1053          SDValue Ops[] = { V, V, ShImmOp, getAL(CurDAG), Reg0 };
1054          return CurDAG->SelectNodeTo(N, ARM::t2RSBrs, MVT::i32, Ops, 5);
1055        } else {
1056          SDValue Ops[] = { V, V, Reg0, ShImmOp, getAL(CurDAG), Reg0, Reg0 };
1057          return CurDAG->SelectNodeTo(N, ARM::RSBrs, MVT::i32, Ops, 7);
1058        }
1059      }
1060    }
1061    break;
1062  case ARMISD::FMRRD:
1063    return CurDAG->getMachineNode(ARM::FMRRD, dl, MVT::i32, MVT::i32,
1064                                  Op.getOperand(0), getAL(CurDAG),
1065                                  CurDAG->getRegister(0, MVT::i32));
1066  case ISD::UMUL_LOHI: {
1067    if (Subtarget->isThumb1Only())
1068      break;
1069    if (Subtarget->isThumb()) {
1070      SDValue Ops[] = { Op.getOperand(0), Op.getOperand(1),
1071                        getAL(CurDAG), CurDAG->getRegister(0, MVT::i32),
1072                        CurDAG->getRegister(0, MVT::i32) };
1073      return CurDAG->getMachineNode(ARM::t2UMULL, dl, MVT::i32, MVT::i32, Ops,4);
1074    } else {
1075      SDValue Ops[] = { Op.getOperand(0), Op.getOperand(1),
1076                        getAL(CurDAG), CurDAG->getRegister(0, MVT::i32),
1077                        CurDAG->getRegister(0, MVT::i32) };
1078      return CurDAG->getMachineNode(ARM::UMULL, dl, MVT::i32, MVT::i32, Ops, 5);
1079    }
1080  }
1081  case ISD::SMUL_LOHI: {
1082    if (Subtarget->isThumb1Only())
1083      break;
1084    if (Subtarget->isThumb()) {
1085      SDValue Ops[] = { Op.getOperand(0), Op.getOperand(1),
1086                        getAL(CurDAG), CurDAG->getRegister(0, MVT::i32) };
1087      return CurDAG->getMachineNode(ARM::t2SMULL, dl, MVT::i32, MVT::i32, Ops,4);
1088    } else {
1089      SDValue Ops[] = { Op.getOperand(0), Op.getOperand(1),
1090                        getAL(CurDAG), CurDAG->getRegister(0, MVT::i32),
1091                        CurDAG->getRegister(0, MVT::i32) };
1092      return CurDAG->getMachineNode(ARM::SMULL, dl, MVT::i32, MVT::i32, Ops, 5);
1093    }
1094  }
1095  case ISD::LOAD: {
1096    SDNode *ResNode = 0;
1097    if (Subtarget->isThumb() && Subtarget->hasThumb2())
1098      ResNode = SelectT2IndexedLoad(Op);
1099    else
1100      ResNode = SelectARMIndexedLoad(Op);
1101    if (ResNode)
1102      return ResNode;
1103    // Other cases are autogenerated.
1104    break;
1105  }
1106  case ARMISD::BRCOND: {
1107    // Pattern: (ARMbrcond:void (bb:Other):$dst, (imm:i32):$cc)
1108    // Emits: (Bcc:void (bb:Other):$dst, (imm:i32):$cc)
1109    // Pattern complexity = 6  cost = 1  size = 0
1110
1111    // Pattern: (ARMbrcond:void (bb:Other):$dst, (imm:i32):$cc)
1112    // Emits: (tBcc:void (bb:Other):$dst, (imm:i32):$cc)
1113    // Pattern complexity = 6  cost = 1  size = 0
1114
1115    // Pattern: (ARMbrcond:void (bb:Other):$dst, (imm:i32):$cc)
1116    // Emits: (t2Bcc:void (bb:Other):$dst, (imm:i32):$cc)
1117    // Pattern complexity = 6  cost = 1  size = 0
1118
1119    unsigned Opc = Subtarget->isThumb() ?
1120      ((Subtarget->hasThumb2()) ? ARM::t2Bcc : ARM::tBcc) : ARM::Bcc;
1121    SDValue Chain = Op.getOperand(0);
1122    SDValue N1 = Op.getOperand(1);
1123    SDValue N2 = Op.getOperand(2);
1124    SDValue N3 = Op.getOperand(3);
1125    SDValue InFlag = Op.getOperand(4);
1126    assert(N1.getOpcode() == ISD::BasicBlock);
1127    assert(N2.getOpcode() == ISD::Constant);
1128    assert(N3.getOpcode() == ISD::Register);
1129
1130    SDValue Tmp2 = CurDAG->getTargetConstant(((unsigned)
1131                               cast<ConstantSDNode>(N2)->getZExtValue()),
1132                               MVT::i32);
1133    SDValue Ops[] = { N1, Tmp2, N3, Chain, InFlag };
1134    SDNode *ResNode = CurDAG->getMachineNode(Opc, dl, MVT::Other,
1135                                             MVT::Flag, Ops, 5);
1136    Chain = SDValue(ResNode, 0);
1137    if (Op.getNode()->getNumValues() == 2) {
1138      InFlag = SDValue(ResNode, 1);
1139      ReplaceUses(SDValue(Op.getNode(), 1), InFlag);
1140    }
1141    ReplaceUses(SDValue(Op.getNode(), 0), SDValue(Chain.getNode(), Chain.getResNo()));
1142    return NULL;
1143  }
1144  case ARMISD::CMOV: {
1145    EVT VT = Op.getValueType();
1146    SDValue N0 = Op.getOperand(0);
1147    SDValue N1 = Op.getOperand(1);
1148    SDValue N2 = Op.getOperand(2);
1149    SDValue N3 = Op.getOperand(3);
1150    SDValue InFlag = Op.getOperand(4);
1151    assert(N2.getOpcode() == ISD::Constant);
1152    assert(N3.getOpcode() == ISD::Register);
1153
1154    if (!Subtarget->isThumb1Only() && VT == MVT::i32) {
1155      // Pattern: (ARMcmov:i32 GPR:i32:$false, so_reg:i32:$true, (imm:i32):$cc)
1156      // Emits: (MOVCCs:i32 GPR:i32:$false, so_reg:i32:$true, (imm:i32):$cc)
1157      // Pattern complexity = 18  cost = 1  size = 0
1158      SDValue CPTmp0;
1159      SDValue CPTmp1;
1160      SDValue CPTmp2;
1161      if (Subtarget->isThumb()) {
1162        if (SelectT2ShifterOperandReg(Op, N1, CPTmp0, CPTmp1)) {
1163          unsigned SOVal = cast<ConstantSDNode>(CPTmp1)->getZExtValue();
1164          unsigned SOShOp = ARM_AM::getSORegShOp(SOVal);
1165          unsigned Opc = 0;
1166          switch (SOShOp) {
1167          case ARM_AM::lsl: Opc = ARM::t2MOVCClsl; break;
1168          case ARM_AM::lsr: Opc = ARM::t2MOVCClsr; break;
1169          case ARM_AM::asr: Opc = ARM::t2MOVCCasr; break;
1170          case ARM_AM::ror: Opc = ARM::t2MOVCCror; break;
1171          default:
1172            llvm_unreachable("Unknown so_reg opcode!");
1173            break;
1174          }
1175          SDValue SOShImm =
1176            CurDAG->getTargetConstant(ARM_AM::getSORegOffset(SOVal), MVT::i32);
1177          SDValue Tmp2 = CurDAG->getTargetConstant(((unsigned)
1178                                   cast<ConstantSDNode>(N2)->getZExtValue()),
1179                                   MVT::i32);
1180          SDValue Ops[] = { N0, CPTmp0, SOShImm, Tmp2, N3, InFlag };
1181          return CurDAG->SelectNodeTo(Op.getNode(), Opc, MVT::i32,Ops, 6);
1182        }
1183      } else {
1184        if (SelectShifterOperandReg(Op, N1, CPTmp0, CPTmp1, CPTmp2)) {
1185          SDValue Tmp2 = CurDAG->getTargetConstant(((unsigned)
1186                                   cast<ConstantSDNode>(N2)->getZExtValue()),
1187                                   MVT::i32);
1188          SDValue Ops[] = { N0, CPTmp0, CPTmp1, CPTmp2, Tmp2, N3, InFlag };
1189          return CurDAG->SelectNodeTo(Op.getNode(),
1190                                      ARM::MOVCCs, MVT::i32, Ops, 7);
1191        }
1192      }
1193
1194      // Pattern: (ARMcmov:i32 GPR:i32:$false,
1195      //             (imm:i32)<<P:Predicate_so_imm>>:$true,
1196      //             (imm:i32):$cc)
1197      // Emits: (MOVCCi:i32 GPR:i32:$false,
1198      //           (so_imm:i32 (imm:i32):$true), (imm:i32):$cc)
1199      // Pattern complexity = 10  cost = 1  size = 0
1200      if (N3.getOpcode() == ISD::Constant) {
1201        if (Subtarget->isThumb()) {
1202          if (Predicate_t2_so_imm(N3.getNode())) {
1203            SDValue Tmp1 = CurDAG->getTargetConstant(((unsigned)
1204                                     cast<ConstantSDNode>(N1)->getZExtValue()),
1205                                     MVT::i32);
1206            SDValue Tmp2 = CurDAG->getTargetConstant(((unsigned)
1207                                     cast<ConstantSDNode>(N2)->getZExtValue()),
1208                                     MVT::i32);
1209            SDValue Ops[] = { N0, Tmp1, Tmp2, N3, InFlag };
1210            return CurDAG->SelectNodeTo(Op.getNode(),
1211                                        ARM::t2MOVCCi, MVT::i32, Ops, 5);
1212          }
1213        } else {
1214          if (Predicate_so_imm(N3.getNode())) {
1215            SDValue Tmp1 = CurDAG->getTargetConstant(((unsigned)
1216                                     cast<ConstantSDNode>(N1)->getZExtValue()),
1217                                     MVT::i32);
1218            SDValue Tmp2 = CurDAG->getTargetConstant(((unsigned)
1219                                     cast<ConstantSDNode>(N2)->getZExtValue()),
1220                                     MVT::i32);
1221            SDValue Ops[] = { N0, Tmp1, Tmp2, N3, InFlag };
1222            return CurDAG->SelectNodeTo(Op.getNode(),
1223                                        ARM::MOVCCi, MVT::i32, Ops, 5);
1224          }
1225        }
1226      }
1227    }
1228
1229    // Pattern: (ARMcmov:i32 GPR:i32:$false, GPR:i32:$true, (imm:i32):$cc)
1230    // Emits: (MOVCCr:i32 GPR:i32:$false, GPR:i32:$true, (imm:i32):$cc)
1231    // Pattern complexity = 6  cost = 1  size = 0
1232    //
1233    // Pattern: (ARMcmov:i32 GPR:i32:$false, GPR:i32:$true, (imm:i32):$cc)
1234    // Emits: (tMOVCCr:i32 GPR:i32:$false, GPR:i32:$true, (imm:i32):$cc)
1235    // Pattern complexity = 6  cost = 11  size = 0
1236    //
1237    // Also FCPYScc and FCPYDcc.
1238    SDValue Tmp2 = CurDAG->getTargetConstant(((unsigned)
1239                               cast<ConstantSDNode>(N2)->getZExtValue()),
1240                               MVT::i32);
1241    SDValue Ops[] = { N0, N1, Tmp2, N3, InFlag };
1242    unsigned Opc = 0;
1243    switch (VT.getSimpleVT().SimpleTy) {
1244    default: assert(false && "Illegal conditional move type!");
1245      break;
1246    case MVT::i32:
1247      Opc = Subtarget->isThumb()
1248        ? (Subtarget->hasThumb2() ? ARM::t2MOVCCr : ARM::tMOVCCr_pseudo)
1249        : ARM::MOVCCr;
1250      break;
1251    case MVT::f32:
1252      Opc = ARM::FCPYScc;
1253      break;
1254    case MVT::f64:
1255      Opc = ARM::FCPYDcc;
1256      break;
1257    }
1258    return CurDAG->SelectNodeTo(Op.getNode(), Opc, VT, Ops, 5);
1259  }
1260  case ARMISD::CNEG: {
1261    EVT VT = Op.getValueType();
1262    SDValue N0 = Op.getOperand(0);
1263    SDValue N1 = Op.getOperand(1);
1264    SDValue N2 = Op.getOperand(2);
1265    SDValue N3 = Op.getOperand(3);
1266    SDValue InFlag = Op.getOperand(4);
1267    assert(N2.getOpcode() == ISD::Constant);
1268    assert(N3.getOpcode() == ISD::Register);
1269
1270    SDValue Tmp2 = CurDAG->getTargetConstant(((unsigned)
1271                               cast<ConstantSDNode>(N2)->getZExtValue()),
1272                               MVT::i32);
1273    SDValue Ops[] = { N0, N1, Tmp2, N3, InFlag };
1274    unsigned Opc = 0;
1275    switch (VT.getSimpleVT().SimpleTy) {
1276    default: assert(false && "Illegal conditional move type!");
1277      break;
1278    case MVT::f32:
1279      Opc = ARM::FNEGScc;
1280      break;
1281    case MVT::f64:
1282      Opc = ARM::FNEGDcc;
1283      break;
1284    }
1285    return CurDAG->SelectNodeTo(Op.getNode(), Opc, VT, Ops, 5);
1286  }
1287
1288  case ARMISD::VZIP: {
1289    unsigned Opc = 0;
1290    EVT VT = N->getValueType(0);
1291    switch (VT.getSimpleVT().SimpleTy) {
1292    default: return NULL;
1293    case MVT::v8i8:  Opc = ARM::VZIPd8; break;
1294    case MVT::v4i16: Opc = ARM::VZIPd16; break;
1295    case MVT::v2f32:
1296    case MVT::v2i32: Opc = ARM::VZIPd32; break;
1297    case MVT::v16i8: Opc = ARM::VZIPq8; break;
1298    case MVT::v8i16: Opc = ARM::VZIPq16; break;
1299    case MVT::v4f32:
1300    case MVT::v4i32: Opc = ARM::VZIPq32; break;
1301    }
1302    return CurDAG->getMachineNode(Opc, dl, VT, VT,
1303                                  N->getOperand(0), N->getOperand(1));
1304  }
1305  case ARMISD::VUZP: {
1306    unsigned Opc = 0;
1307    EVT VT = N->getValueType(0);
1308    switch (VT.getSimpleVT().SimpleTy) {
1309    default: return NULL;
1310    case MVT::v8i8:  Opc = ARM::VUZPd8; break;
1311    case MVT::v4i16: Opc = ARM::VUZPd16; break;
1312    case MVT::v2f32:
1313    case MVT::v2i32: Opc = ARM::VUZPd32; break;
1314    case MVT::v16i8: Opc = ARM::VUZPq8; break;
1315    case MVT::v8i16: Opc = ARM::VUZPq16; break;
1316    case MVT::v4f32:
1317    case MVT::v4i32: Opc = ARM::VUZPq32; break;
1318    }
1319    return CurDAG->getMachineNode(Opc, dl, VT, VT,
1320                                  N->getOperand(0), N->getOperand(1));
1321  }
1322  case ARMISD::VTRN: {
1323    unsigned Opc = 0;
1324    EVT VT = N->getValueType(0);
1325    switch (VT.getSimpleVT().SimpleTy) {
1326    default: return NULL;
1327    case MVT::v8i8:  Opc = ARM::VTRNd8; break;
1328    case MVT::v4i16: Opc = ARM::VTRNd16; break;
1329    case MVT::v2f32:
1330    case MVT::v2i32: Opc = ARM::VTRNd32; break;
1331    case MVT::v16i8: Opc = ARM::VTRNq8; break;
1332    case MVT::v8i16: Opc = ARM::VTRNq16; break;
1333    case MVT::v4f32:
1334    case MVT::v4i32: Opc = ARM::VTRNq32; break;
1335    }
1336    return CurDAG->getMachineNode(Opc, dl, VT, VT,
1337                                  N->getOperand(0), N->getOperand(1));
1338  }
1339
1340  case ISD::INTRINSIC_VOID:
1341  case ISD::INTRINSIC_W_CHAIN: {
1342    unsigned IntNo = cast<ConstantSDNode>(N->getOperand(1))->getZExtValue();
1343    EVT VT = N->getValueType(0);
1344    unsigned Opc = 0;
1345
1346    switch (IntNo) {
1347    default:
1348      break;
1349
1350    case Intrinsic::arm_neon_vld2: {
1351      SDValue MemAddr, MemUpdate, MemOpc;
1352      if (!SelectAddrMode6(Op, N->getOperand(2), MemAddr, MemUpdate, MemOpc))
1353        return NULL;
1354      if (VT.is64BitVector()) {
1355        switch (VT.getSimpleVT().SimpleTy) {
1356        default: llvm_unreachable("unhandled vld2 type");
1357        case MVT::v8i8:  Opc = ARM::VLD2d8; break;
1358        case MVT::v4i16: Opc = ARM::VLD2d16; break;
1359        case MVT::v2f32:
1360        case MVT::v2i32: Opc = ARM::VLD2d32; break;
1361        case MVT::v1i64: Opc = ARM::VLD2d64; break;
1362        }
1363        SDValue Chain = N->getOperand(0);
1364        const SDValue Ops[] = { MemAddr, MemUpdate, MemOpc, Chain };
1365        return CurDAG->getMachineNode(Opc, dl, VT, VT, MVT::Other, Ops, 4);
1366      }
1367      // Quad registers are loaded as pairs of double registers.
1368      EVT RegVT;
1369      switch (VT.getSimpleVT().SimpleTy) {
1370      default: llvm_unreachable("unhandled vld2 type");
1371      case MVT::v16i8: Opc = ARM::VLD2q8; RegVT = MVT::v8i8; break;
1372      case MVT::v8i16: Opc = ARM::VLD2q16; RegVT = MVT::v4i16; break;
1373      case MVT::v4f32: Opc = ARM::VLD2q32; RegVT = MVT::v2f32; break;
1374      case MVT::v4i32: Opc = ARM::VLD2q32; RegVT = MVT::v2i32; break;
1375      }
1376      SDValue Chain = N->getOperand(0);
1377      const SDValue Ops[] = { MemAddr, MemUpdate, MemOpc, Chain };
1378      std::vector<EVT> ResTys(4, RegVT);
1379      ResTys.push_back(MVT::Other);
1380      SDNode *VLd = CurDAG->getMachineNode(Opc, dl, ResTys, Ops, 4);
1381      SDNode *Q0 = PairDRegs(VT, SDValue(VLd, 0), SDValue(VLd, 1));
1382      SDNode *Q1 = PairDRegs(VT, SDValue(VLd, 2), SDValue(VLd, 3));
1383      ReplaceUses(SDValue(N, 0), SDValue(Q0, 0));
1384      ReplaceUses(SDValue(N, 1), SDValue(Q1, 0));
1385      ReplaceUses(SDValue(N, 2), SDValue(VLd, 4));
1386      return NULL;
1387    }
1388
1389    case Intrinsic::arm_neon_vld3: {
1390      SDValue MemAddr, MemUpdate, MemOpc;
1391      if (!SelectAddrMode6(Op, N->getOperand(2), MemAddr, MemUpdate, MemOpc))
1392        return NULL;
1393      if (VT.is64BitVector()) {
1394        switch (VT.getSimpleVT().SimpleTy) {
1395        default: llvm_unreachable("unhandled vld3 type");
1396        case MVT::v8i8:  Opc = ARM::VLD3d8; break;
1397        case MVT::v4i16: Opc = ARM::VLD3d16; break;
1398        case MVT::v2f32:
1399        case MVT::v2i32: Opc = ARM::VLD3d32; break;
1400        case MVT::v1i64: Opc = ARM::VLD3d64; break;
1401        }
1402        SDValue Chain = N->getOperand(0);
1403        const SDValue Ops[] = { MemAddr, MemUpdate, MemOpc, Chain };
1404        return CurDAG->getMachineNode(Opc, dl, VT, VT, VT, MVT::Other, Ops, 4);
1405      }
1406      // Quad registers are loaded with two separate instructions, where one
1407      // loads the even registers and the other loads the odd registers.
1408      EVT RegVT = VT;
1409      unsigned Opc2 = 0;
1410      switch (VT.getSimpleVT().SimpleTy) {
1411      default: llvm_unreachable("unhandled vld3 type");
1412      case MVT::v16i8:
1413        Opc = ARM::VLD3q8a;  Opc2 = ARM::VLD3q8b;  RegVT = MVT::v8i8; break;
1414      case MVT::v8i16:
1415        Opc = ARM::VLD3q16a; Opc2 = ARM::VLD3q16b; RegVT = MVT::v4i16; break;
1416      case MVT::v4f32:
1417        Opc = ARM::VLD3q32a; Opc2 = ARM::VLD3q32b; RegVT = MVT::v2f32; break;
1418      case MVT::v4i32:
1419        Opc = ARM::VLD3q32a; Opc2 = ARM::VLD3q32b; RegVT = MVT::v2i32; break;
1420      }
1421      SDValue Chain = N->getOperand(0);
1422      // Enable writeback to the address register.
1423      MemOpc = CurDAG->getTargetConstant(ARM_AM::getAM6Opc(true), MVT::i32);
1424
1425      std::vector<EVT> ResTys(3, RegVT);
1426      ResTys.push_back(MemAddr.getValueType());
1427      ResTys.push_back(MVT::Other);
1428
1429      const SDValue OpsA[] = { MemAddr, MemUpdate, MemOpc, Chain };
1430      SDNode *VLdA = CurDAG->getMachineNode(Opc, dl, ResTys, OpsA, 4);
1431      Chain = SDValue(VLdA, 4);
1432
1433      const SDValue OpsB[] = { SDValue(VLdA, 3), MemUpdate, MemOpc, Chain };
1434      SDNode *VLdB = CurDAG->getMachineNode(Opc2, dl, ResTys, OpsB, 4);
1435      Chain = SDValue(VLdB, 4);
1436
1437      SDNode *Q0 = PairDRegs(VT, SDValue(VLdA, 0), SDValue(VLdB, 0));
1438      SDNode *Q1 = PairDRegs(VT, SDValue(VLdA, 1), SDValue(VLdB, 1));
1439      SDNode *Q2 = PairDRegs(VT, SDValue(VLdA, 2), SDValue(VLdB, 2));
1440      ReplaceUses(SDValue(N, 0), SDValue(Q0, 0));
1441      ReplaceUses(SDValue(N, 1), SDValue(Q1, 0));
1442      ReplaceUses(SDValue(N, 2), SDValue(Q2, 0));
1443      ReplaceUses(SDValue(N, 3), Chain);
1444      return NULL;
1445    }
1446
1447    case Intrinsic::arm_neon_vld4: {
1448      SDValue MemAddr, MemUpdate, MemOpc;
1449      if (!SelectAddrMode6(Op, N->getOperand(2), MemAddr, MemUpdate, MemOpc))
1450        return NULL;
1451      if (VT.is64BitVector()) {
1452        switch (VT.getSimpleVT().SimpleTy) {
1453        default: llvm_unreachable("unhandled vld4 type");
1454        case MVT::v8i8:  Opc = ARM::VLD4d8; break;
1455        case MVT::v4i16: Opc = ARM::VLD4d16; break;
1456        case MVT::v2f32:
1457        case MVT::v2i32: Opc = ARM::VLD4d32; break;
1458        case MVT::v1i64: Opc = ARM::VLD4d64; break;
1459        }
1460        SDValue Chain = N->getOperand(0);
1461        const SDValue Ops[] = { MemAddr, MemUpdate, MemOpc, Chain };
1462        std::vector<EVT> ResTys(4, VT);
1463        ResTys.push_back(MVT::Other);
1464        return CurDAG->getMachineNode(Opc, dl, ResTys, Ops, 4);
1465      }
1466      // Quad registers are loaded with two separate instructions, where one
1467      // loads the even registers and the other loads the odd registers.
1468      EVT RegVT = VT;
1469      unsigned Opc2 = 0;
1470      switch (VT.getSimpleVT().SimpleTy) {
1471      default: llvm_unreachable("unhandled vld4 type");
1472      case MVT::v16i8:
1473        Opc = ARM::VLD4q8a;  Opc2 = ARM::VLD4q8b;  RegVT = MVT::v8i8; break;
1474      case MVT::v8i16:
1475        Opc = ARM::VLD4q16a; Opc2 = ARM::VLD4q16b; RegVT = MVT::v4i16; break;
1476      case MVT::v4f32:
1477        Opc = ARM::VLD4q32a; Opc2 = ARM::VLD4q32b; RegVT = MVT::v2f32; break;
1478      case MVT::v4i32:
1479        Opc = ARM::VLD4q32a; Opc2 = ARM::VLD4q32b; RegVT = MVT::v2i32; break;
1480      }
1481      SDValue Chain = N->getOperand(0);
1482      // Enable writeback to the address register.
1483      MemOpc = CurDAG->getTargetConstant(ARM_AM::getAM6Opc(true), MVT::i32);
1484
1485      std::vector<EVT> ResTys(4, RegVT);
1486      ResTys.push_back(MemAddr.getValueType());
1487      ResTys.push_back(MVT::Other);
1488
1489      const SDValue OpsA[] = { MemAddr, MemUpdate, MemOpc, Chain };
1490      SDNode *VLdA = CurDAG->getMachineNode(Opc, dl, ResTys, OpsA, 4);
1491      Chain = SDValue(VLdA, 5);
1492
1493      const SDValue OpsB[] = { SDValue(VLdA, 4), MemUpdate, MemOpc, Chain };
1494      SDNode *VLdB = CurDAG->getMachineNode(Opc2, dl, ResTys, OpsB, 4);
1495      Chain = SDValue(VLdB, 5);
1496
1497      SDNode *Q0 = PairDRegs(VT, SDValue(VLdA, 0), SDValue(VLdB, 0));
1498      SDNode *Q1 = PairDRegs(VT, SDValue(VLdA, 1), SDValue(VLdB, 1));
1499      SDNode *Q2 = PairDRegs(VT, SDValue(VLdA, 2), SDValue(VLdB, 2));
1500      SDNode *Q3 = PairDRegs(VT, SDValue(VLdA, 3), SDValue(VLdB, 3));
1501      ReplaceUses(SDValue(N, 0), SDValue(Q0, 0));
1502      ReplaceUses(SDValue(N, 1), SDValue(Q1, 0));
1503      ReplaceUses(SDValue(N, 2), SDValue(Q2, 0));
1504      ReplaceUses(SDValue(N, 3), SDValue(Q3, 0));
1505      ReplaceUses(SDValue(N, 4), Chain);
1506      return NULL;
1507    }
1508
1509    case Intrinsic::arm_neon_vld2lane: {
1510      SDValue MemAddr, MemUpdate, MemOpc;
1511      if (!SelectAddrMode6(Op, N->getOperand(2), MemAddr, MemUpdate, MemOpc))
1512        return NULL;
1513      switch (VT.getSimpleVT().SimpleTy) {
1514      default: llvm_unreachable("unhandled vld2lane type");
1515      case MVT::v8i8:  Opc = ARM::VLD2LNd8; break;
1516      case MVT::v4i16: Opc = ARM::VLD2LNd16; break;
1517      case MVT::v2f32:
1518      case MVT::v2i32: Opc = ARM::VLD2LNd32; break;
1519      }
1520      SDValue Chain = N->getOperand(0);
1521      const SDValue Ops[] = { MemAddr, MemUpdate, MemOpc,
1522                              N->getOperand(3), N->getOperand(4),
1523                              N->getOperand(5), Chain };
1524      return CurDAG->getMachineNode(Opc, dl, VT, VT, MVT::Other, Ops, 7);
1525    }
1526
1527    case Intrinsic::arm_neon_vld3lane: {
1528      SDValue MemAddr, MemUpdate, MemOpc;
1529      if (!SelectAddrMode6(Op, N->getOperand(2), MemAddr, MemUpdate, MemOpc))
1530        return NULL;
1531      switch (VT.getSimpleVT().SimpleTy) {
1532      default: llvm_unreachable("unhandled vld3lane type");
1533      case MVT::v8i8:  Opc = ARM::VLD3LNd8; break;
1534      case MVT::v4i16: Opc = ARM::VLD3LNd16; break;
1535      case MVT::v2f32:
1536      case MVT::v2i32: Opc = ARM::VLD3LNd32; break;
1537      }
1538      SDValue Chain = N->getOperand(0);
1539      const SDValue Ops[] = { MemAddr, MemUpdate, MemOpc,
1540                              N->getOperand(3), N->getOperand(4),
1541                              N->getOperand(5), N->getOperand(6), Chain };
1542      return CurDAG->getMachineNode(Opc, dl, VT, VT, VT, MVT::Other, Ops, 8);
1543    }
1544
1545    case Intrinsic::arm_neon_vld4lane: {
1546      SDValue MemAddr, MemUpdate, MemOpc;
1547      if (!SelectAddrMode6(Op, N->getOperand(2), MemAddr, MemUpdate, MemOpc))
1548        return NULL;
1549      switch (VT.getSimpleVT().SimpleTy) {
1550      default: llvm_unreachable("unhandled vld4lane type");
1551      case MVT::v8i8:  Opc = ARM::VLD4LNd8; break;
1552      case MVT::v4i16: Opc = ARM::VLD4LNd16; break;
1553      case MVT::v2f32:
1554      case MVT::v2i32: Opc = ARM::VLD4LNd32; break;
1555      }
1556      SDValue Chain = N->getOperand(0);
1557      const SDValue Ops[] = { MemAddr, MemUpdate, MemOpc,
1558                              N->getOperand(3), N->getOperand(4),
1559                              N->getOperand(5), N->getOperand(6),
1560                              N->getOperand(7), Chain };
1561      std::vector<EVT> ResTys(4, VT);
1562      ResTys.push_back(MVT::Other);
1563      return CurDAG->getMachineNode(Opc, dl, ResTys, Ops, 9);
1564    }
1565
1566    case Intrinsic::arm_neon_vst2: {
1567      SDValue MemAddr, MemUpdate, MemOpc;
1568      if (!SelectAddrMode6(Op, N->getOperand(2), MemAddr, MemUpdate, MemOpc))
1569        return NULL;
1570      VT = N->getOperand(3).getValueType();
1571      if (VT.is64BitVector()) {
1572        switch (VT.getSimpleVT().SimpleTy) {
1573        default: llvm_unreachable("unhandled vst2 type");
1574        case MVT::v8i8:  Opc = ARM::VST2d8; break;
1575        case MVT::v4i16: Opc = ARM::VST2d16; break;
1576        case MVT::v2f32:
1577        case MVT::v2i32: Opc = ARM::VST2d32; break;
1578        case MVT::v1i64: Opc = ARM::VST2d64; break;
1579        }
1580        SDValue Chain = N->getOperand(0);
1581        const SDValue Ops[] = { MemAddr, MemUpdate, MemOpc,
1582                                N->getOperand(3), N->getOperand(4), Chain };
1583        return CurDAG->getMachineNode(Opc, dl, MVT::Other, Ops, 6);
1584      }
1585      // Quad registers are stored as pairs of double registers.
1586      EVT RegVT;
1587      switch (VT.getSimpleVT().SimpleTy) {
1588      default: llvm_unreachable("unhandled vst2 type");
1589      case MVT::v16i8: Opc = ARM::VST2q8; RegVT = MVT::v8i8; break;
1590      case MVT::v8i16: Opc = ARM::VST2q16; RegVT = MVT::v4i16; break;
1591      case MVT::v4f32: Opc = ARM::VST2q32; RegVT = MVT::v2f32; break;
1592      case MVT::v4i32: Opc = ARM::VST2q32; RegVT = MVT::v2i32; break;
1593      }
1594      SDValue Chain = N->getOperand(0);
1595      SDValue D0 = CurDAG->getTargetExtractSubreg(ARM::DSUBREG_0, dl, RegVT,
1596                                                  N->getOperand(3));
1597      SDValue D1 = CurDAG->getTargetExtractSubreg(ARM::DSUBREG_1, dl, RegVT,
1598                                                  N->getOperand(3));
1599      SDValue D2 = CurDAG->getTargetExtractSubreg(ARM::DSUBREG_0, dl, RegVT,
1600                                                  N->getOperand(4));
1601      SDValue D3 = CurDAG->getTargetExtractSubreg(ARM::DSUBREG_1, dl, RegVT,
1602                                                  N->getOperand(4));
1603      const SDValue Ops[] = { MemAddr, MemUpdate, MemOpc,
1604                              D0, D1, D2, D3, Chain };
1605      return CurDAG->getMachineNode(Opc, dl, MVT::Other, Ops, 8);
1606    }
1607
1608    case Intrinsic::arm_neon_vst3: {
1609      SDValue MemAddr, MemUpdate, MemOpc;
1610      if (!SelectAddrMode6(Op, N->getOperand(2), MemAddr, MemUpdate, MemOpc))
1611        return NULL;
1612      VT = N->getOperand(3).getValueType();
1613      if (VT.is64BitVector()) {
1614        switch (VT.getSimpleVT().SimpleTy) {
1615        default: llvm_unreachable("unhandled vst3 type");
1616        case MVT::v8i8:  Opc = ARM::VST3d8; break;
1617        case MVT::v4i16: Opc = ARM::VST3d16; break;
1618        case MVT::v2f32:
1619        case MVT::v2i32: Opc = ARM::VST3d32; break;
1620        case MVT::v1i64: Opc = ARM::VST3d64; break;
1621        }
1622        SDValue Chain = N->getOperand(0);
1623        const SDValue Ops[] = { MemAddr, MemUpdate, MemOpc,
1624                                N->getOperand(3), N->getOperand(4),
1625                                N->getOperand(5), Chain };
1626        return CurDAG->getMachineNode(Opc, dl, MVT::Other, Ops, 7);
1627      }
1628      // Quad registers are stored with two separate instructions, where one
1629      // stores the even registers and the other stores the odd registers.
1630      EVT RegVT;
1631      unsigned Opc2 = 0;
1632      switch (VT.getSimpleVT().SimpleTy) {
1633      default: llvm_unreachable("unhandled vst3 type");
1634      case MVT::v16i8:
1635        Opc = ARM::VST3q8a;  Opc2 = ARM::VST3q8b;  RegVT = MVT::v8i8; break;
1636      case MVT::v8i16:
1637        Opc = ARM::VST3q16a; Opc2 = ARM::VST3q16b; RegVT = MVT::v4i16; break;
1638      case MVT::v4f32:
1639        Opc = ARM::VST3q32a; Opc2 = ARM::VST3q32b; RegVT = MVT::v2f32; break;
1640      case MVT::v4i32:
1641        Opc = ARM::VST3q32a; Opc2 = ARM::VST3q32b; RegVT = MVT::v2i32; break;
1642      }
1643      SDValue Chain = N->getOperand(0);
1644      // Enable writeback to the address register.
1645      MemOpc = CurDAG->getTargetConstant(ARM_AM::getAM6Opc(true), MVT::i32);
1646
1647      SDValue D0 = CurDAG->getTargetExtractSubreg(ARM::DSUBREG_0, dl, RegVT,
1648                                                  N->getOperand(3));
1649      SDValue D2 = CurDAG->getTargetExtractSubreg(ARM::DSUBREG_0, dl, RegVT,
1650                                                  N->getOperand(4));
1651      SDValue D4 = CurDAG->getTargetExtractSubreg(ARM::DSUBREG_0, dl, RegVT,
1652                                                  N->getOperand(5));
1653      const SDValue OpsA[] = { MemAddr, MemUpdate, MemOpc, D0, D2, D4, Chain };
1654      SDNode *VStA = CurDAG->getMachineNode(Opc, dl, MemAddr.getValueType(),
1655                                            MVT::Other, OpsA, 7);
1656      Chain = SDValue(VStA, 1);
1657
1658      SDValue D1 = CurDAG->getTargetExtractSubreg(ARM::DSUBREG_1, dl, RegVT,
1659                                                  N->getOperand(3));
1660      SDValue D3 = CurDAG->getTargetExtractSubreg(ARM::DSUBREG_1, dl, RegVT,
1661                                                  N->getOperand(4));
1662      SDValue D5 = CurDAG->getTargetExtractSubreg(ARM::DSUBREG_1, dl, RegVT,
1663                                                  N->getOperand(5));
1664      MemAddr = SDValue(VStA, 0);
1665      const SDValue OpsB[] = { MemAddr, MemUpdate, MemOpc, D1, D3, D5, Chain };
1666      SDNode *VStB = CurDAG->getMachineNode(Opc2, dl, MemAddr.getValueType(),
1667                                            MVT::Other, OpsB, 7);
1668      Chain = SDValue(VStB, 1);
1669      ReplaceUses(SDValue(N, 0), Chain);
1670      return NULL;
1671    }
1672
1673    case Intrinsic::arm_neon_vst4: {
1674      SDValue MemAddr, MemUpdate, MemOpc;
1675      if (!SelectAddrMode6(Op, N->getOperand(2), MemAddr, MemUpdate, MemOpc))
1676        return NULL;
1677      VT = N->getOperand(3).getValueType();
1678      if (VT.is64BitVector()) {
1679        switch (VT.getSimpleVT().SimpleTy) {
1680        default: llvm_unreachable("unhandled vst4 type");
1681        case MVT::v8i8:  Opc = ARM::VST4d8; break;
1682        case MVT::v4i16: Opc = ARM::VST4d16; break;
1683        case MVT::v2f32:
1684        case MVT::v2i32: Opc = ARM::VST4d32; break;
1685        case MVT::v1i64: Opc = ARM::VST4d64; break;
1686        }
1687        SDValue Chain = N->getOperand(0);
1688        const SDValue Ops[] = { MemAddr, MemUpdate, MemOpc,
1689                                N->getOperand(3), N->getOperand(4),
1690                                N->getOperand(5), N->getOperand(6), Chain };
1691        return CurDAG->getMachineNode(Opc, dl, MVT::Other, Ops, 8);
1692      }
1693      // Quad registers are stored with two separate instructions, where one
1694      // stores the even registers and the other stores the odd registers.
1695      EVT RegVT;
1696      unsigned Opc2 = 0;
1697      switch (VT.getSimpleVT().SimpleTy) {
1698      default: llvm_unreachable("unhandled vst4 type");
1699      case MVT::v16i8:
1700        Opc = ARM::VST4q8a;  Opc2 = ARM::VST4q8b;  RegVT = MVT::v8i8; break;
1701      case MVT::v8i16:
1702        Opc = ARM::VST4q16a; Opc2 = ARM::VST4q16b; RegVT = MVT::v4i16; break;
1703      case MVT::v4f32:
1704        Opc = ARM::VST4q32a; Opc2 = ARM::VST4q32b; RegVT = MVT::v2f32; break;
1705      case MVT::v4i32:
1706        Opc = ARM::VST4q32a; Opc2 = ARM::VST4q32b; RegVT = MVT::v2i32; break;
1707      }
1708      SDValue Chain = N->getOperand(0);
1709      // Enable writeback to the address register.
1710      MemOpc = CurDAG->getTargetConstant(ARM_AM::getAM6Opc(true), MVT::i32);
1711
1712      SDValue D0 = CurDAG->getTargetExtractSubreg(ARM::DSUBREG_0, dl, RegVT,
1713                                                  N->getOperand(3));
1714      SDValue D2 = CurDAG->getTargetExtractSubreg(ARM::DSUBREG_0, dl, RegVT,
1715                                                  N->getOperand(4));
1716      SDValue D4 = CurDAG->getTargetExtractSubreg(ARM::DSUBREG_0, dl, RegVT,
1717                                                  N->getOperand(5));
1718      SDValue D6 = CurDAG->getTargetExtractSubreg(ARM::DSUBREG_0, dl, RegVT,
1719                                                  N->getOperand(6));
1720      const SDValue OpsA[] = { MemAddr, MemUpdate, MemOpc,
1721                               D0, D2, D4, D6, Chain };
1722      SDNode *VStA = CurDAG->getMachineNode(Opc, dl, MemAddr.getValueType(),
1723                                            MVT::Other, OpsA, 8);
1724      Chain = SDValue(VStA, 1);
1725
1726      SDValue D1 = CurDAG->getTargetExtractSubreg(ARM::DSUBREG_1, dl, RegVT,
1727                                                  N->getOperand(3));
1728      SDValue D3 = CurDAG->getTargetExtractSubreg(ARM::DSUBREG_1, dl, RegVT,
1729                                                  N->getOperand(4));
1730      SDValue D5 = CurDAG->getTargetExtractSubreg(ARM::DSUBREG_1, dl, RegVT,
1731                                                  N->getOperand(5));
1732      SDValue D7 = CurDAG->getTargetExtractSubreg(ARM::DSUBREG_1, dl, RegVT,
1733                                                  N->getOperand(6));
1734      MemAddr = SDValue(VStA, 0);
1735      const SDValue OpsB[] = { MemAddr, MemUpdate, MemOpc,
1736                               D1, D3, D5, D7, Chain };
1737      SDNode *VStB = CurDAG->getMachineNode(Opc2, dl, MemAddr.getValueType(),
1738                                            MVT::Other, OpsB, 8);
1739      Chain = SDValue(VStB, 1);
1740      ReplaceUses(SDValue(N, 0), Chain);
1741      return NULL;
1742    }
1743
1744    case Intrinsic::arm_neon_vst2lane: {
1745      SDValue MemAddr, MemUpdate, MemOpc;
1746      if (!SelectAddrMode6(Op, N->getOperand(2), MemAddr, MemUpdate, MemOpc))
1747        return NULL;
1748      switch (N->getOperand(3).getValueType().getSimpleVT().SimpleTy) {
1749      default: llvm_unreachable("unhandled vst2lane type");
1750      case MVT::v8i8:  Opc = ARM::VST2LNd8; break;
1751      case MVT::v4i16: Opc = ARM::VST2LNd16; break;
1752      case MVT::v2f32:
1753      case MVT::v2i32: Opc = ARM::VST2LNd32; break;
1754      }
1755      SDValue Chain = N->getOperand(0);
1756      const SDValue Ops[] = { MemAddr, MemUpdate, MemOpc,
1757                              N->getOperand(3), N->getOperand(4),
1758                              N->getOperand(5), Chain };
1759      return CurDAG->getMachineNode(Opc, dl, MVT::Other, Ops, 7);
1760    }
1761
1762    case Intrinsic::arm_neon_vst3lane: {
1763      SDValue MemAddr, MemUpdate, MemOpc;
1764      if (!SelectAddrMode6(Op, N->getOperand(2), MemAddr, MemUpdate, MemOpc))
1765        return NULL;
1766      switch (N->getOperand(3).getValueType().getSimpleVT().SimpleTy) {
1767      default: llvm_unreachable("unhandled vst3lane type");
1768      case MVT::v8i8:  Opc = ARM::VST3LNd8; break;
1769      case MVT::v4i16: Opc = ARM::VST3LNd16; break;
1770      case MVT::v2f32:
1771      case MVT::v2i32: Opc = ARM::VST3LNd32; break;
1772      }
1773      SDValue Chain = N->getOperand(0);
1774      const SDValue Ops[] = { MemAddr, MemUpdate, MemOpc,
1775                              N->getOperand(3), N->getOperand(4),
1776                              N->getOperand(5), N->getOperand(6), Chain };
1777      return CurDAG->getMachineNode(Opc, dl, MVT::Other, Ops, 8);
1778    }
1779
1780    case Intrinsic::arm_neon_vst4lane: {
1781      SDValue MemAddr, MemUpdate, MemOpc;
1782      if (!SelectAddrMode6(Op, N->getOperand(2), MemAddr, MemUpdate, MemOpc))
1783        return NULL;
1784      switch (N->getOperand(3).getValueType().getSimpleVT().SimpleTy) {
1785      default: llvm_unreachable("unhandled vst4lane type");
1786      case MVT::v8i8:  Opc = ARM::VST4LNd8; break;
1787      case MVT::v4i16: Opc = ARM::VST4LNd16; break;
1788      case MVT::v2f32:
1789      case MVT::v2i32: Opc = ARM::VST4LNd32; break;
1790      }
1791      SDValue Chain = N->getOperand(0);
1792      const SDValue Ops[] = { MemAddr, MemUpdate, MemOpc,
1793                              N->getOperand(3), N->getOperand(4),
1794                              N->getOperand(5), N->getOperand(6),
1795                              N->getOperand(7), Chain };
1796      return CurDAG->getMachineNode(Opc, dl, MVT::Other, Ops, 9);
1797    }
1798    }
1799  }
1800  }
1801
1802  return SelectCode(Op);
1803}
1804
1805bool ARMDAGToDAGISel::
1806SelectInlineAsmMemoryOperand(const SDValue &Op, char ConstraintCode,
1807                             std::vector<SDValue> &OutOps) {
1808  assert(ConstraintCode == 'm' && "unexpected asm memory constraint");
1809
1810  SDValue Base, Offset, Opc;
1811  if (!SelectAddrMode2(Op, Op, Base, Offset, Opc))
1812    return true;
1813
1814  OutOps.push_back(Base);
1815  OutOps.push_back(Offset);
1816  OutOps.push_back(Opc);
1817  return false;
1818}
1819
1820/// createARMISelDag - This pass converts a legalized DAG into a
1821/// ARM-specific DAG, ready for instruction scheduling.
1822///
1823FunctionPass *llvm::createARMISelDag(ARMBaseTargetMachine &TM,
1824                                     CodeGenOpt::Level OptLevel) {
1825  return new ARMDAGToDAGISel(TM, OptLevel);
1826}
1827