ARMISelDAGToDAG.cpp revision 602b0c8c17f458d2c80f2deb3c8e554d516ee316 
1//===-- ARMISelDAGToDAG.cpp - A dag to dag inst selector for ARM ----------===//
2//
3//                     The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10// This file defines an instruction selector for the ARM target.
11//
12//===----------------------------------------------------------------------===//
13
14#include "ARM.h"
15#include "ARMAddressingModes.h"
16#include "ARMConstantPoolValue.h"
17#include "ARMISelLowering.h"
18#include "ARMTargetMachine.h"
19#include "llvm/CallingConv.h"
20#include "llvm/Constants.h"
21#include "llvm/DerivedTypes.h"
22#include "llvm/Function.h"
23#include "llvm/Intrinsics.h"
24#include "llvm/LLVMContext.h"
25#include "llvm/CodeGen/MachineFrameInfo.h"
26#include "llvm/CodeGen/MachineFunction.h"
27#include "llvm/CodeGen/MachineInstrBuilder.h"
28#include "llvm/CodeGen/SelectionDAG.h"
29#include "llvm/CodeGen/SelectionDAGISel.h"
30#include "llvm/Target/TargetLowering.h"
31#include "llvm/Target/TargetOptions.h"
32#include "llvm/Support/Compiler.h"
33#include "llvm/Support/Debug.h"
34#include "llvm/Support/ErrorHandling.h"
35#include "llvm/Support/raw_ostream.h"
36
37using namespace llvm;
38
39//===--------------------------------------------------------------------===//
40/// ARMDAGToDAGISel - ARM specific code to select ARM machine
41/// instructions for SelectionDAG operations.
42///
43namespace {
44class ARMDAGToDAGISel : public SelectionDAGISel {
45  ARMBaseTargetMachine &TM;
46
47  /// Subtarget - Keep a pointer to the ARMSubtarget around so that we can
48  /// make the right decision when generating code for different targets.
49  const ARMSubtarget *Subtarget;
50
51public:
52  explicit ARMDAGToDAGISel(ARMBaseTargetMachine &tm)
53    : SelectionDAGISel(tm), TM(tm),
54    Subtarget(&TM.getSubtarget<ARMSubtarget>()) {
55  }
56
57  virtual const char *getPassName() const {
58    return "ARM Instruction Selection";
59  }
60
61 /// getI32Imm - Return a target constant with the specified value, of type i32.
62  inline SDValue getI32Imm(unsigned Imm) {
63    return CurDAG->getTargetConstant(Imm, MVT::i32);
64  }
65
66  SDNode *Select(SDValue Op);
67  virtual void InstructionSelect();
68  bool SelectShifterOperandReg(SDValue Op, SDValue N, SDValue &A,
69                               SDValue &B, SDValue &C);
70  bool SelectAddrMode2(SDValue Op, SDValue N, SDValue &Base,
71                       SDValue &Offset, SDValue &Opc);
72  bool SelectAddrMode2Offset(SDValue Op, SDValue N,
73                             SDValue &Offset, SDValue &Opc);
74  bool SelectAddrMode3(SDValue Op, SDValue N, SDValue &Base,
75                       SDValue &Offset, SDValue &Opc);
76  bool SelectAddrMode3Offset(SDValue Op, SDValue N,
77                             SDValue &Offset, SDValue &Opc);
78  bool SelectAddrMode4(SDValue Op, SDValue N, SDValue &Addr,
79                       SDValue &Mode);
80  bool SelectAddrMode5(SDValue Op, SDValue N, SDValue &Base,
81                       SDValue &Offset);
82  bool SelectAddrMode6(SDValue Op, SDValue N, SDValue &Addr, SDValue &Update,
83                       SDValue &Opc);
84
85  bool SelectAddrModePC(SDValue Op, SDValue N, SDValue &Offset,
86                        SDValue &Label);
87
88  bool SelectThumbAddrModeRR(SDValue Op, SDValue N, SDValue &Base,
89                             SDValue &Offset);
90  bool SelectThumbAddrModeRI5(SDValue Op, SDValue N, unsigned Scale,
91                              SDValue &Base, SDValue &OffImm,
92                              SDValue &Offset);
93  bool SelectThumbAddrModeS1(SDValue Op, SDValue N, SDValue &Base,
94                             SDValue &OffImm, SDValue &Offset);
95  bool SelectThumbAddrModeS2(SDValue Op, SDValue N, SDValue &Base,
96                             SDValue &OffImm, SDValue &Offset);
97  bool SelectThumbAddrModeS4(SDValue Op, SDValue N, SDValue &Base,
98                             SDValue &OffImm, SDValue &Offset);
99  bool SelectThumbAddrModeSP(SDValue Op, SDValue N, SDValue &Base,
100                             SDValue &OffImm);
101
102  bool SelectT2ShifterOperandReg(SDValue Op, SDValue N,
103                                 SDValue &BaseReg, SDValue &Opc);
104  bool SelectT2AddrModeImm12(SDValue Op, SDValue N, SDValue &Base,
105                             SDValue &OffImm);
106  bool SelectT2AddrModeImm8(SDValue Op, SDValue N, SDValue &Base,
107                            SDValue &OffImm);
108  bool SelectT2AddrModeImm8Offset(SDValue Op, SDValue N,
109                                 SDValue &OffImm);
110  bool SelectT2AddrModeImm8s4(SDValue Op, SDValue N, SDValue &Base,
111                              SDValue &OffImm);
112  bool SelectT2AddrModeSoReg(SDValue Op, SDValue N, SDValue &Base,
113                             SDValue &OffReg, SDValue &ShImm);
114
115  // Include the pieces autogenerated from the target description.
116#include "ARMGenDAGISel.inc"
117
118private:
119  /// SelectARMIndexedLoad - Indexed (pre/post inc/dec) load matching code for
120  /// ARM.
121  SDNode *SelectARMIndexedLoad(SDValue Op);
122  SDNode *SelectT2IndexedLoad(SDValue Op);
123
124  /// SelectDYN_ALLOC - Select dynamic alloc for Thumb.
125  SDNode *SelectDYN_ALLOC(SDValue Op);
126
127  /// SelectInlineAsmMemoryOperand - Implement addressing mode selection for
128  /// inline asm expressions.
129  virtual bool SelectInlineAsmMemoryOperand(const SDValue &Op,
130                                            char ConstraintCode,
131                                            std::vector<SDValue> &OutOps);
132};
133}
134
135void ARMDAGToDAGISel::InstructionSelect() {
136  DEBUG(BB->dump());
137
138  SelectRoot(*CurDAG);
139  CurDAG->RemoveDeadNodes();
140}
141
142bool ARMDAGToDAGISel::SelectShifterOperandReg(SDValue Op,
143                                              SDValue N,
144                                              SDValue &BaseReg,
145                                              SDValue &ShReg,
146                                              SDValue &Opc) {
147  ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(N);
148
149  // Don't match base register only case. That is matched to a separate
150  // lower complexity pattern with explicit register operand.
151  if (ShOpcVal == ARM_AM::no_shift) return false;
152
153  BaseReg = N.getOperand(0);
154  unsigned ShImmVal = 0;
155  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
156    ShReg = CurDAG->getRegister(0, MVT::i32);
157    ShImmVal = RHS->getZExtValue() & 31;
158  } else {
159    ShReg = N.getOperand(1);
160  }
161  Opc = CurDAG->getTargetConstant(ARM_AM::getSORegOpc(ShOpcVal, ShImmVal),
162                                  MVT::i32);
163  return true;
164}
165
166bool ARMDAGToDAGISel::SelectAddrMode2(SDValue Op, SDValue N,
167                                      SDValue &Base, SDValue &Offset,
168                                      SDValue &Opc) {
169  if (N.getOpcode() == ISD::MUL) {
170    if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
171      // X * [3,5,9] -> X + X * [2,4,8] etc.
172      int RHSC = (int)RHS->getZExtValue();
173      if (RHSC & 1) {
174        RHSC = RHSC & ~1;
175        ARM_AM::AddrOpc AddSub = ARM_AM::add;
176        if (RHSC < 0) {
177          AddSub = ARM_AM::sub;
178          RHSC = - RHSC;
179        }
180        if (isPowerOf2_32(RHSC)) {
181          unsigned ShAmt = Log2_32(RHSC);
182          Base = Offset = N.getOperand(0);
183          Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, ShAmt,
184                                                            ARM_AM::lsl),
185                                          MVT::i32);
186          return true;
187        }
188      }
189    }
190  }
191
192  if (N.getOpcode() != ISD::ADD && N.getOpcode() != ISD::SUB) {
193    Base = N;
194    if (N.getOpcode() == ISD::FrameIndex) {
195      int FI = cast<FrameIndexSDNode>(N)->getIndex();
196      Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
197    } else if (N.getOpcode() == ARMISD::Wrapper) {
198      Base = N.getOperand(0);
199    }
200    Offset = CurDAG->getRegister(0, MVT::i32);
201    Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(ARM_AM::add, 0,
202                                                      ARM_AM::no_shift),
203                                    MVT::i32);
204    return true;
205  }
206
207  // Match simple R +/- imm12 operands.
208  if (N.getOpcode() == ISD::ADD)
209    if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
210      int RHSC = (int)RHS->getZExtValue();
211      if ((RHSC >= 0 && RHSC < 0x1000) ||
212          (RHSC < 0 && RHSC > -0x1000)) { // 12 bits.
213        Base = N.getOperand(0);
214        if (Base.getOpcode() == ISD::FrameIndex) {
215          int FI = cast<FrameIndexSDNode>(Base)->getIndex();
216          Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
217        }
218        Offset = CurDAG->getRegister(0, MVT::i32);
219
220        ARM_AM::AddrOpc AddSub = ARM_AM::add;
221        if (RHSC < 0) {
222          AddSub = ARM_AM::sub;
223          RHSC = - RHSC;
224        }
225        Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, RHSC,
226                                                          ARM_AM::no_shift),
227                                        MVT::i32);
228        return true;
229      }
230    }
231
232  // Otherwise this is R +/- [possibly shifted] R
233  ARM_AM::AddrOpc AddSub = N.getOpcode() == ISD::ADD ? ARM_AM::add:ARM_AM::sub;
234  ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(N.getOperand(1));
235  unsigned ShAmt = 0;
236
237  Base   = N.getOperand(0);
238  Offset = N.getOperand(1);
239
240  if (ShOpcVal != ARM_AM::no_shift) {
241    // Check to see if the RHS of the shift is a constant, if not, we can't fold
242    // it.
243    if (ConstantSDNode *Sh =
244           dyn_cast<ConstantSDNode>(N.getOperand(1).getOperand(1))) {
245      ShAmt = Sh->getZExtValue();
246      Offset = N.getOperand(1).getOperand(0);
247    } else {
248      ShOpcVal = ARM_AM::no_shift;
249    }
250  }
251
252  // Try matching (R shl C) + (R).
253  if (N.getOpcode() == ISD::ADD && ShOpcVal == ARM_AM::no_shift) {
254    ShOpcVal = ARM_AM::getShiftOpcForNode(N.getOperand(0));
255    if (ShOpcVal != ARM_AM::no_shift) {
256      // Check to see if the RHS of the shift is a constant, if not, we can't
257      // fold it.
258      if (ConstantSDNode *Sh =
259          dyn_cast<ConstantSDNode>(N.getOperand(0).getOperand(1))) {
260        ShAmt = Sh->getZExtValue();
261        Offset = N.getOperand(0).getOperand(0);
262        Base = N.getOperand(1);
263      } else {
264        ShOpcVal = ARM_AM::no_shift;
265      }
266    }
267  }
268
269  Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, ShAmt, ShOpcVal),
270                                  MVT::i32);
271  return true;
272}
273
274bool ARMDAGToDAGISel::SelectAddrMode2Offset(SDValue Op, SDValue N,
275                                            SDValue &Offset, SDValue &Opc) {
276  unsigned Opcode = Op.getOpcode();
277  ISD::MemIndexedMode AM = (Opcode == ISD::LOAD)
278    ? cast<LoadSDNode>(Op)->getAddressingMode()
279    : cast<StoreSDNode>(Op)->getAddressingMode();
280  ARM_AM::AddrOpc AddSub = (AM == ISD::PRE_INC || AM == ISD::POST_INC)
281    ? ARM_AM::add : ARM_AM::sub;
282  if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(N)) {
283    int Val = (int)C->getZExtValue();
284    if (Val >= 0 && Val < 0x1000) { // 12 bits.
285      Offset = CurDAG->getRegister(0, MVT::i32);
286      Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, Val,
287                                                        ARM_AM::no_shift),
288                                      MVT::i32);
289      return true;
290    }
291  }
292
293  Offset = N;
294  ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(N);
295  unsigned ShAmt = 0;
296  if (ShOpcVal != ARM_AM::no_shift) {
297    // Check to see if the RHS of the shift is a constant, if not, we can't fold
298    // it.
299    if (ConstantSDNode *Sh = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
300      ShAmt = Sh->getZExtValue();
301      Offset = N.getOperand(0);
302    } else {
303      ShOpcVal = ARM_AM::no_shift;
304    }
305  }
306
307  Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, ShAmt, ShOpcVal),
308                                  MVT::i32);
309  return true;
310}
311
312
313bool ARMDAGToDAGISel::SelectAddrMode3(SDValue Op, SDValue N,
314                                      SDValue &Base, SDValue &Offset,
315                                      SDValue &Opc) {
316  if (N.getOpcode() == ISD::SUB) {
317    // X - C  is canonicalize to X + -C, no need to handle it here.
318    Base = N.getOperand(0);
319    Offset = N.getOperand(1);
320    Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(ARM_AM::sub, 0),MVT::i32);
321    return true;
322  }
323
324  if (N.getOpcode() != ISD::ADD) {
325    Base = N;
326    if (N.getOpcode() == ISD::FrameIndex) {
327      int FI = cast<FrameIndexSDNode>(N)->getIndex();
328      Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
329    }
330    Offset = CurDAG->getRegister(0, MVT::i32);
331    Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(ARM_AM::add, 0),MVT::i32);
332    return true;
333  }
334
335  // If the RHS is +/- imm8, fold into addr mode.
336  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
337    int RHSC = (int)RHS->getZExtValue();
338    if ((RHSC >= 0 && RHSC < 256) ||
339        (RHSC < 0 && RHSC > -256)) { // note -256 itself isn't allowed.
340      Base = N.getOperand(0);
341      if (Base.getOpcode() == ISD::FrameIndex) {
342        int FI = cast<FrameIndexSDNode>(Base)->getIndex();
343        Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
344      }
345      Offset = CurDAG->getRegister(0, MVT::i32);
346
347      ARM_AM::AddrOpc AddSub = ARM_AM::add;
348      if (RHSC < 0) {
349        AddSub = ARM_AM::sub;
350        RHSC = - RHSC;
351      }
352      Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(AddSub, RHSC),MVT::i32);
353      return true;
354    }
355  }
356
357  Base = N.getOperand(0);
358  Offset = N.getOperand(1);
359  Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(ARM_AM::add, 0), MVT::i32);
360  return true;
361}
362
363bool ARMDAGToDAGISel::SelectAddrMode3Offset(SDValue Op, SDValue N,
364                                            SDValue &Offset, SDValue &Opc) {
365  unsigned Opcode = Op.getOpcode();
366  ISD::MemIndexedMode AM = (Opcode == ISD::LOAD)
367    ? cast<LoadSDNode>(Op)->getAddressingMode()
368    : cast<StoreSDNode>(Op)->getAddressingMode();
369  ARM_AM::AddrOpc AddSub = (AM == ISD::PRE_INC || AM == ISD::POST_INC)
370    ? ARM_AM::add : ARM_AM::sub;
371  if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(N)) {
372    int Val = (int)C->getZExtValue();
373    if (Val >= 0 && Val < 256) {
374      Offset = CurDAG->getRegister(0, MVT::i32);
375      Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(AddSub, Val), MVT::i32);
376      return true;
377    }
378  }
379
380  Offset = N;
381  Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(AddSub, 0), MVT::i32);
382  return true;
383}
384
385bool ARMDAGToDAGISel::SelectAddrMode4(SDValue Op, SDValue N,
386                                      SDValue &Addr, SDValue &Mode) {
387  Addr = N;
388  Mode = CurDAG->getTargetConstant(0, MVT::i32);
389  return true;
390}
391
392bool ARMDAGToDAGISel::SelectAddrMode5(SDValue Op, SDValue N,
393                                      SDValue &Base, SDValue &Offset) {
394  if (N.getOpcode() != ISD::ADD) {
395    Base = N;
396    if (N.getOpcode() == ISD::FrameIndex) {
397      int FI = cast<FrameIndexSDNode>(N)->getIndex();
398      Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
399    } else if (N.getOpcode() == ARMISD::Wrapper) {
400      Base = N.getOperand(0);
401    }
402    Offset = CurDAG->getTargetConstant(ARM_AM::getAM5Opc(ARM_AM::add, 0),
403                                       MVT::i32);
404    return true;
405  }
406
407  // If the RHS is +/- imm8, fold into addr mode.
408  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
409    int RHSC = (int)RHS->getZExtValue();
410    if ((RHSC & 3) == 0) {  // The constant is implicitly multiplied by 4.
411      RHSC >>= 2;
412      if ((RHSC >= 0 && RHSC < 256) ||
413          (RHSC < 0 && RHSC > -256)) { // note -256 itself isn't allowed.
414        Base = N.getOperand(0);
415        if (Base.getOpcode() == ISD::FrameIndex) {
416          int FI = cast<FrameIndexSDNode>(Base)->getIndex();
417          Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
418        }
419
420        ARM_AM::AddrOpc AddSub = ARM_AM::add;
421        if (RHSC < 0) {
422          AddSub = ARM_AM::sub;
423          RHSC = - RHSC;
424        }
425        Offset = CurDAG->getTargetConstant(ARM_AM::getAM5Opc(AddSub, RHSC),
426                                           MVT::i32);
427        return true;
428      }
429    }
430  }
431
432  Base = N;
433  Offset = CurDAG->getTargetConstant(ARM_AM::getAM5Opc(ARM_AM::add, 0),
434                                     MVT::i32);
435  return true;
436}
437
438bool ARMDAGToDAGISel::SelectAddrMode6(SDValue Op, SDValue N,
439                                      SDValue &Addr, SDValue &Update,
440                                      SDValue &Opc) {
441  Addr = N;
442  // The optional writeback is handled in ARMLoadStoreOpt.
443  Update = CurDAG->getRegister(0, MVT::i32);
444  Opc = CurDAG->getTargetConstant(ARM_AM::getAM6Opc(false), MVT::i32);
445  return true;
446}
447
448bool ARMDAGToDAGISel::SelectAddrModePC(SDValue Op, SDValue N,
449                                       SDValue &Offset, SDValue &Label) {
450  if (N.getOpcode() == ARMISD::PIC_ADD && N.hasOneUse()) {
451    Offset = N.getOperand(0);
452    SDValue N1 = N.getOperand(1);
453    Label  = CurDAG->getTargetConstant(cast<ConstantSDNode>(N1)->getZExtValue(),
454                                       MVT::i32);
455    return true;
456  }
457  return false;
458}
459
460bool ARMDAGToDAGISel::SelectThumbAddrModeRR(SDValue Op, SDValue N,
461                                            SDValue &Base, SDValue &Offset){
462  // FIXME dl should come from the parent load or store, not the address
463  DebugLoc dl = Op.getDebugLoc();
464  if (N.getOpcode() != ISD::ADD) {
465    ConstantSDNode *NC = dyn_cast<ConstantSDNode>(N);
466    if (!NC || NC->getZExtValue() != 0)
467      return false;
468
469    Base = Offset = N;
470    return true;
471  }
472
473  Base = N.getOperand(0);
474  Offset = N.getOperand(1);
475  return true;
476}
477
478bool
479ARMDAGToDAGISel::SelectThumbAddrModeRI5(SDValue Op, SDValue N,
480                                        unsigned Scale, SDValue &Base,
481                                        SDValue &OffImm, SDValue &Offset) {
482  if (Scale == 4) {
483    SDValue TmpBase, TmpOffImm;
484    if (SelectThumbAddrModeSP(Op, N, TmpBase, TmpOffImm))
485      return false;  // We want to select tLDRspi / tSTRspi instead.
486    if (N.getOpcode() == ARMISD::Wrapper &&
487        N.getOperand(0).getOpcode() == ISD::TargetConstantPool)
488      return false;  // We want to select tLDRpci instead.
489  }
490
491  if (N.getOpcode() != ISD::ADD) {
492    Base = (N.getOpcode() == ARMISD::Wrapper) ? N.getOperand(0) : N;
493    Offset = CurDAG->getRegister(0, MVT::i32);
494    OffImm = CurDAG->getTargetConstant(0, MVT::i32);
495    return true;
496  }
497
498  // Thumb does not have [sp, r] address mode.
499  RegisterSDNode *LHSR = dyn_cast<RegisterSDNode>(N.getOperand(0));
500  RegisterSDNode *RHSR = dyn_cast<RegisterSDNode>(N.getOperand(1));
501  if ((LHSR && LHSR->getReg() == ARM::SP) ||
502      (RHSR && RHSR->getReg() == ARM::SP)) {
503    Base = N;
504    Offset = CurDAG->getRegister(0, MVT::i32);
505    OffImm = CurDAG->getTargetConstant(0, MVT::i32);
506    return true;
507  }
508
509  // If the RHS is + imm5 * scale, fold into addr mode.
510  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
511    int RHSC = (int)RHS->getZExtValue();
512    if ((RHSC & (Scale-1)) == 0) {  // The constant is implicitly multiplied.
513      RHSC /= Scale;
514      if (RHSC >= 0 && RHSC < 32) {
515        Base = N.getOperand(0);
516        Offset = CurDAG->getRegister(0, MVT::i32);
517        OffImm = CurDAG->getTargetConstant(RHSC, MVT::i32);
518        return true;
519      }
520    }
521  }
522
523  Base = N.getOperand(0);
524  Offset = N.getOperand(1);
525  OffImm = CurDAG->getTargetConstant(0, MVT::i32);
526  return true;
527}
528
529bool ARMDAGToDAGISel::SelectThumbAddrModeS1(SDValue Op, SDValue N,
530                                            SDValue &Base, SDValue &OffImm,
531                                            SDValue &Offset) {
532  return SelectThumbAddrModeRI5(Op, N, 1, Base, OffImm, Offset);
533}
534
535bool ARMDAGToDAGISel::SelectThumbAddrModeS2(SDValue Op, SDValue N,
536                                            SDValue &Base, SDValue &OffImm,
537                                            SDValue &Offset) {
538  return SelectThumbAddrModeRI5(Op, N, 2, Base, OffImm, Offset);
539}
540
541bool ARMDAGToDAGISel::SelectThumbAddrModeS4(SDValue Op, SDValue N,
542                                            SDValue &Base, SDValue &OffImm,
543                                            SDValue &Offset) {
544  return SelectThumbAddrModeRI5(Op, N, 4, Base, OffImm, Offset);
545}
546
547bool ARMDAGToDAGISel::SelectThumbAddrModeSP(SDValue Op, SDValue N,
548                                           SDValue &Base, SDValue &OffImm) {
549  if (N.getOpcode() == ISD::FrameIndex) {
550    int FI = cast<FrameIndexSDNode>(N)->getIndex();
551    Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
552    OffImm = CurDAG->getTargetConstant(0, MVT::i32);
553    return true;
554  }
555
556  if (N.getOpcode() != ISD::ADD)
557    return false;
558
559  RegisterSDNode *LHSR = dyn_cast<RegisterSDNode>(N.getOperand(0));
560  if (N.getOperand(0).getOpcode() == ISD::FrameIndex ||
561      (LHSR && LHSR->getReg() == ARM::SP)) {
562    // If the RHS is + imm8 * scale, fold into addr mode.
563    if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
564      int RHSC = (int)RHS->getZExtValue();
565      if ((RHSC & 3) == 0) {  // The constant is implicitly multiplied.
566        RHSC >>= 2;
567        if (RHSC >= 0 && RHSC < 256) {
568          Base = N.getOperand(0);
569          if (Base.getOpcode() == ISD::FrameIndex) {
570            int FI = cast<FrameIndexSDNode>(Base)->getIndex();
571            Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
572          }
573          OffImm = CurDAG->getTargetConstant(RHSC, MVT::i32);
574          return true;
575        }
576      }
577    }
578  }
579
580  return false;
581}
582
583bool ARMDAGToDAGISel::SelectT2ShifterOperandReg(SDValue Op, SDValue N,
584                                                SDValue &BaseReg,
585                                                SDValue &Opc) {
586  ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(N);
587
588  // Don't match base register only case. That is matched to a separate
589  // lower complexity pattern with explicit register operand.
590  if (ShOpcVal == ARM_AM::no_shift) return false;
591
592  BaseReg = N.getOperand(0);
593  unsigned ShImmVal = 0;
594  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
595    ShImmVal = RHS->getZExtValue() & 31;
596    Opc = getI32Imm(ARM_AM::getSORegOpc(ShOpcVal, ShImmVal));
597    return true;
598  }
599
600  return false;
601}
602
603bool ARMDAGToDAGISel::SelectT2AddrModeImm12(SDValue Op, SDValue N,
604                                            SDValue &Base, SDValue &OffImm) {
605  // Match simple R + imm12 operands.
606
607  // Base only.
608  if (N.getOpcode() != ISD::ADD && N.getOpcode() != ISD::SUB) {
609    if (N.getOpcode() == ISD::FrameIndex) {
610      // Match frame index...
611      int FI = cast<FrameIndexSDNode>(N)->getIndex();
612      Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
613      OffImm  = CurDAG->getTargetConstant(0, MVT::i32);
614      return true;
615    } else if (N.getOpcode() == ARMISD::Wrapper) {
616      Base = N.getOperand(0);
617      if (Base.getOpcode() == ISD::TargetConstantPool)
618        return false;  // We want to select t2LDRpci instead.
619    } else
620      Base = N;
621    OffImm  = CurDAG->getTargetConstant(0, MVT::i32);
622    return true;
623  }
624
625  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
626    if (SelectT2AddrModeImm8(Op, N, Base, OffImm))
627      // Let t2LDRi8 handle (R - imm8).
628      return false;
629
630    int RHSC = (int)RHS->getZExtValue();
631    if (N.getOpcode() == ISD::SUB)
632      RHSC = -RHSC;
633
634    if (RHSC >= 0 && RHSC < 0x1000) { // 12 bits (unsigned)
635      Base   = N.getOperand(0);
636      if (Base.getOpcode() == ISD::FrameIndex) {
637        int FI = cast<FrameIndexSDNode>(Base)->getIndex();
638        Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
639      }
640      OffImm = CurDAG->getTargetConstant(RHSC, MVT::i32);
641      return true;
642    }
643  }
644
645  // Base only.
646  Base = N;
647  OffImm  = CurDAG->getTargetConstant(0, MVT::i32);
648  return true;
649}
650
651bool ARMDAGToDAGISel::SelectT2AddrModeImm8(SDValue Op, SDValue N,
652                                           SDValue &Base, SDValue &OffImm) {
653  // Match simple R - imm8 operands.
654  if (N.getOpcode() == ISD::ADD || N.getOpcode() == ISD::SUB) {
655    if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
656      int RHSC = (int)RHS->getSExtValue();
657      if (N.getOpcode() == ISD::SUB)
658        RHSC = -RHSC;
659
660      if ((RHSC >= -255) && (RHSC < 0)) { // 8 bits (always negative)
661        Base = N.getOperand(0);
662        if (Base.getOpcode() == ISD::FrameIndex) {
663          int FI = cast<FrameIndexSDNode>(Base)->getIndex();
664          Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
665        }
666        OffImm = CurDAG->getTargetConstant(RHSC, MVT::i32);
667        return true;
668      }
669    }
670  }
671
672  return false;
673}
674
675bool ARMDAGToDAGISel::SelectT2AddrModeImm8Offset(SDValue Op, SDValue N,
676                                                 SDValue &OffImm){
677  unsigned Opcode = Op.getOpcode();
678  ISD::MemIndexedMode AM = (Opcode == ISD::LOAD)
679    ? cast<LoadSDNode>(Op)->getAddressingMode()
680    : cast<StoreSDNode>(Op)->getAddressingMode();
681  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N)) {
682    int RHSC = (int)RHS->getZExtValue();
683    if (RHSC >= 0 && RHSC < 0x100) { // 8 bits.
684      OffImm = ((AM == ISD::PRE_INC) || (AM == ISD::POST_INC))
685        ? CurDAG->getTargetConstant(RHSC, MVT::i32)
686        : CurDAG->getTargetConstant(-RHSC, MVT::i32);
687      return true;
688    }
689  }
690
691  return false;
692}
693
694bool ARMDAGToDAGISel::SelectT2AddrModeImm8s4(SDValue Op, SDValue N,
695                                             SDValue &Base, SDValue &OffImm) {
696  if (N.getOpcode() == ISD::ADD) {
697    if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
698      int RHSC = (int)RHS->getZExtValue();
699      if (((RHSC & 0x3) == 0) &&
700          ((RHSC >= 0 && RHSC < 0x400) || (RHSC < 0 && RHSC > -0x400))) { // 8 bits.
701        Base   = N.getOperand(0);
702        OffImm = CurDAG->getTargetConstant(RHSC, MVT::i32);
703        return true;
704      }
705    }
706  } else if (N.getOpcode() == ISD::SUB) {
707    if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
708      int RHSC = (int)RHS->getZExtValue();
709      if (((RHSC & 0x3) == 0) && (RHSC >= 0 && RHSC < 0x400)) { // 8 bits.
710        Base   = N.getOperand(0);
711        OffImm = CurDAG->getTargetConstant(-RHSC, MVT::i32);
712        return true;
713      }
714    }
715  }
716
717  return false;
718}
719
720bool ARMDAGToDAGISel::SelectT2AddrModeSoReg(SDValue Op, SDValue N,
721                                            SDValue &Base,
722                                            SDValue &OffReg, SDValue &ShImm) {
723  // (R - imm8) should be handled by t2LDRi8. The rest are handled by t2LDRi12.
724  if (N.getOpcode() != ISD::ADD)
725    return false;
726
727  // Leave (R + imm12) for t2LDRi12, (R - imm8) for t2LDRi8.
728  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
729    int RHSC = (int)RHS->getZExtValue();
730    if (RHSC >= 0 && RHSC < 0x1000) // 12 bits (unsigned)
731      return false;
732    else if (RHSC < 0 && RHSC >= -255) // 8 bits
733      return false;
734  }
735
736  // Look for (R + R) or (R + (R << [1,2,3])).
737  unsigned ShAmt = 0;
738  Base   = N.getOperand(0);
739  OffReg = N.getOperand(1);
740
741  // Swap if it is ((R << c) + R).
742  ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(OffReg);
743  if (ShOpcVal != ARM_AM::lsl) {
744    ShOpcVal = ARM_AM::getShiftOpcForNode(Base);
745    if (ShOpcVal == ARM_AM::lsl)
746      std::swap(Base, OffReg);
747  }
748
749  if (ShOpcVal == ARM_AM::lsl) {
750    // Check to see if the RHS of the shift is a constant, if not, we can't fold
751    // it.
752    if (ConstantSDNode *Sh = dyn_cast<ConstantSDNode>(OffReg.getOperand(1))) {
753      ShAmt = Sh->getZExtValue();
754      if (ShAmt >= 4) {
755        ShAmt = 0;
756        ShOpcVal = ARM_AM::no_shift;
757      } else
758        OffReg = OffReg.getOperand(0);
759    } else {
760      ShOpcVal = ARM_AM::no_shift;
761    }
762  }
763
764  ShImm = CurDAG->getTargetConstant(ShAmt, MVT::i32);
765
766  return true;
767}
768
769//===--------------------------------------------------------------------===//
770
771/// getAL - Returns a ARMCC::AL immediate node.
772static inline SDValue getAL(SelectionDAG *CurDAG) {
773  return CurDAG->getTargetConstant((uint64_t)ARMCC::AL, MVT::i32);
774}
775
776SDNode *ARMDAGToDAGISel::SelectARMIndexedLoad(SDValue Op) {
777  LoadSDNode *LD = cast<LoadSDNode>(Op);
778  ISD::MemIndexedMode AM = LD->getAddressingMode();
779  if (AM == ISD::UNINDEXED)
780    return NULL;
781
782  EVT LoadedVT = LD->getMemoryVT();
783  SDValue Offset, AMOpc;
784  bool isPre = (AM == ISD::PRE_INC) || (AM == ISD::PRE_DEC);
785  unsigned Opcode = 0;
786  bool Match = false;
787  if (LoadedVT == MVT::i32 &&
788      SelectAddrMode2Offset(Op, LD->getOffset(), Offset, AMOpc)) {
789    Opcode = isPre ? ARM::LDR_PRE : ARM::LDR_POST;
790    Match = true;
791  } else if (LoadedVT == MVT::i16 &&
792             SelectAddrMode3Offset(Op, LD->getOffset(), Offset, AMOpc)) {
793    Match = true;
794    Opcode = (LD->getExtensionType() == ISD::SEXTLOAD)
795      ? (isPre ? ARM::LDRSH_PRE : ARM::LDRSH_POST)
796      : (isPre ? ARM::LDRH_PRE : ARM::LDRH_POST);
797  } else if (LoadedVT == MVT::i8 || LoadedVT == MVT::i1) {
798    if (LD->getExtensionType() == ISD::SEXTLOAD) {
799      if (SelectAddrMode3Offset(Op, LD->getOffset(), Offset, AMOpc)) {
800        Match = true;
801        Opcode = isPre ? ARM::LDRSB_PRE : ARM::LDRSB_POST;
802      }
803    } else {
804      if (SelectAddrMode2Offset(Op, LD->getOffset(), Offset, AMOpc)) {
805        Match = true;
806        Opcode = isPre ? ARM::LDRB_PRE : ARM::LDRB_POST;
807      }
808    }
809  }
810
811  if (Match) {
812    SDValue Chain = LD->getChain();
813    SDValue Base = LD->getBasePtr();
814    SDValue Ops[]= { Base, Offset, AMOpc, getAL(CurDAG),
815                     CurDAG->getRegister(0, MVT::i32), Chain };
816    return CurDAG->getMachineNode(Opcode, Op.getDebugLoc(), MVT::i32, MVT::i32,
817                                  MVT::Other, Ops, 6);
818  }
819
820  return NULL;
821}
822
823SDNode *ARMDAGToDAGISel::SelectT2IndexedLoad(SDValue Op) {
824  LoadSDNode *LD = cast<LoadSDNode>(Op);
825  ISD::MemIndexedMode AM = LD->getAddressingMode();
826  if (AM == ISD::UNINDEXED)
827    return NULL;
828
829  EVT LoadedVT = LD->getMemoryVT();
830  bool isSExtLd = LD->getExtensionType() == ISD::SEXTLOAD;
831  SDValue Offset;
832  bool isPre = (AM == ISD::PRE_INC) || (AM == ISD::PRE_DEC);
833  unsigned Opcode = 0;
834  bool Match = false;
835  if (SelectT2AddrModeImm8Offset(Op, LD->getOffset(), Offset)) {
836    switch (LoadedVT.getSimpleVT().SimpleTy) {
837    case MVT::i32:
838      Opcode = isPre ? ARM::t2LDR_PRE : ARM::t2LDR_POST;
839      break;
840    case MVT::i16:
841      if (isSExtLd)
842        Opcode = isPre ? ARM::t2LDRSH_PRE : ARM::t2LDRSH_POST;
843      else
844        Opcode = isPre ? ARM::t2LDRH_PRE : ARM::t2LDRH_POST;
845      break;
846    case MVT::i8:
847    case MVT::i1:
848      if (isSExtLd)
849        Opcode = isPre ? ARM::t2LDRSB_PRE : ARM::t2LDRSB_POST;
850      else
851        Opcode = isPre ? ARM::t2LDRB_PRE : ARM::t2LDRB_POST;
852      break;
853    default:
854      return NULL;
855    }
856    Match = true;
857  }
858
859  if (Match) {
860    SDValue Chain = LD->getChain();
861    SDValue Base = LD->getBasePtr();
862    SDValue Ops[]= { Base, Offset, getAL(CurDAG),
863                     CurDAG->getRegister(0, MVT::i32), Chain };
864    return CurDAG->getMachineNode(Opcode, Op.getDebugLoc(), MVT::i32, MVT::i32,
865                                  MVT::Other, Ops, 5);
866  }
867
868  return NULL;
869}
870
871SDNode *ARMDAGToDAGISel::SelectDYN_ALLOC(SDValue Op) {
872  SDNode *N = Op.getNode();
873  DebugLoc dl = N->getDebugLoc();
874  EVT VT = Op.getValueType();
875  SDValue Chain = Op.getOperand(0);
876  SDValue Size = Op.getOperand(1);
877  SDValue Align = Op.getOperand(2);
878  SDValue SP = CurDAG->getRegister(ARM::SP, MVT::i32);
879  int32_t AlignVal = cast<ConstantSDNode>(Align)->getSExtValue();
880  if (AlignVal < 0)
881    // We need to align the stack. Use Thumb1 tAND which is the only thumb
882    // instruction that can read and write SP. This matches to a pseudo
883    // instruction that has a chain to ensure the result is written back to
884    // the stack pointer.
885    SP = SDValue(CurDAG->getMachineNode(ARM::tANDsp, dl, VT, SP, Align), 0);
886
887  bool isC = isa<ConstantSDNode>(Size);
888  uint32_t C = isC ? cast<ConstantSDNode>(Size)->getZExtValue() : ~0UL;
889  // Handle the most common case for both Thumb1 and Thumb2:
890  // tSUBspi - immediate is between 0 ... 508 inclusive.
891  if (C <= 508 && ((C & 3) == 0))
892    // FIXME: tSUBspi encode scale 4 implicitly.
893    return CurDAG->SelectNodeTo(N, ARM::tSUBspi_, VT, MVT::Other, SP,
894                                CurDAG->getTargetConstant(C/4, MVT::i32),
895                                Chain);
896
897  if (Subtarget->isThumb1Only()) {
898    // Use tADDspr since Thumb1 does not have a sub r, sp, r. ARMISelLowering
899    // should have negated the size operand already. FIXME: We can't insert
900    // new target independent node at this stage so we are forced to negate
901    // it earlier. Is there a better solution?
902    return CurDAG->SelectNodeTo(N, ARM::tADDspr_, VT, MVT::Other, SP, Size,
903                                Chain);
904  } else if (Subtarget->isThumb2()) {
905    if (isC && Predicate_t2_so_imm(Size.getNode())) {
906      // t2SUBrSPi
907      SDValue Ops[] = { SP, CurDAG->getTargetConstant(C, MVT::i32), Chain };
908      return CurDAG->SelectNodeTo(N, ARM::t2SUBrSPi_, VT, MVT::Other, Ops, 3);
909    } else if (isC && Predicate_imm0_4095(Size.getNode())) {
910      // t2SUBrSPi12
911      SDValue Ops[] = { SP, CurDAG->getTargetConstant(C, MVT::i32), Chain };
912      return CurDAG->SelectNodeTo(N, ARM::t2SUBrSPi12_, VT, MVT::Other, Ops, 3);
913    } else {
914      // t2SUBrSPs
915      SDValue Ops[] = { SP, Size,
916                        getI32Imm(ARM_AM::getSORegOpc(ARM_AM::lsl,0)), Chain };
917      return CurDAG->SelectNodeTo(N, ARM::t2SUBrSPs_, VT, MVT::Other, Ops, 4);
918    }
919  }
920
921  // FIXME: Add ADD / SUB sp instructions for ARM.
922  return 0;
923}
924
925SDNode *ARMDAGToDAGISel::Select(SDValue Op) {
926  SDNode *N = Op.getNode();
927  DebugLoc dl = N->getDebugLoc();
928
929  if (N->isMachineOpcode())
930    return NULL;   // Already selected.
931
932  switch (N->getOpcode()) {
933  default: break;
934  case ISD::Constant: {
935    unsigned Val = cast<ConstantSDNode>(N)->getZExtValue();
936    bool UseCP = true;
937    if (Subtarget->isThumb()) {
938      if (Subtarget->hasThumb2())
939        // Thumb2 has the MOVT instruction, so all immediates can
940        // be done with MOV + MOVT, at worst.
941        UseCP = 0;
942      else
943        UseCP = (Val > 255 &&                          // MOV
944                 ~Val > 255 &&                         // MOV + MVN
945                 !ARM_AM::isThumbImmShiftedVal(Val));  // MOV + LSL
946    } else
947      UseCP = (ARM_AM::getSOImmVal(Val) == -1 &&     // MOV
948               ARM_AM::getSOImmVal(~Val) == -1 &&    // MVN
949               !ARM_AM::isSOImmTwoPartVal(Val));     // two instrs.
950    if (UseCP) {
951      SDValue CPIdx =
952        CurDAG->getTargetConstantPool(ConstantInt::get(
953                                  Type::getInt32Ty(*CurDAG->getContext()), Val),
954                                      TLI.getPointerTy());
955
956      SDNode *ResNode;
957      if (Subtarget->isThumb1Only()) {
958        SDValue Pred = CurDAG->getTargetConstant(0xEULL, MVT::i32);
959        SDValue PredReg = CurDAG->getRegister(0, MVT::i32);
960        SDValue Ops[] = { CPIdx, Pred, PredReg, CurDAG->getEntryNode() };
961        ResNode = CurDAG->getMachineNode(ARM::tLDRcp, dl, MVT::i32, MVT::Other,
962                                         Ops, 4);
963      } else {
964        SDValue Ops[] = {
965          CPIdx,
966          CurDAG->getRegister(0, MVT::i32),
967          CurDAG->getTargetConstant(0, MVT::i32),
968          getAL(CurDAG),
969          CurDAG->getRegister(0, MVT::i32),
970          CurDAG->getEntryNode()
971        };
972        ResNode=CurDAG->getMachineNode(ARM::LDRcp, dl, MVT::i32, MVT::Other,
973                                       Ops, 6);
974      }
975      ReplaceUses(Op, SDValue(ResNode, 0));
976      return NULL;
977    }
978
979    // Other cases are autogenerated.
980    break;
981  }
982  case ISD::FrameIndex: {
983    // Selects to ADDri FI, 0 which in turn will become ADDri SP, imm.
984    int FI = cast<FrameIndexSDNode>(N)->getIndex();
985    SDValue TFI = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
986    if (Subtarget->isThumb1Only()) {
987      return CurDAG->SelectNodeTo(N, ARM::tADDrSPi, MVT::i32, TFI,
988                                  CurDAG->getTargetConstant(0, MVT::i32));
989    } else {
990      unsigned Opc = ((Subtarget->isThumb() && Subtarget->hasThumb2()) ?
991                      ARM::t2ADDri : ARM::ADDri);
992      SDValue Ops[] = { TFI, CurDAG->getTargetConstant(0, MVT::i32),
993                        getAL(CurDAG), CurDAG->getRegister(0, MVT::i32),
994                        CurDAG->getRegister(0, MVT::i32) };
995      return CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops, 5);
996    }
997  }
998  case ARMISD::DYN_ALLOC:
999    return SelectDYN_ALLOC(Op);
1000  case ISD::MUL:
1001    if (Subtarget->isThumb1Only())
1002      break;
1003    if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
1004      unsigned RHSV = C->getZExtValue();
1005      if (!RHSV) break;
1006      if (isPowerOf2_32(RHSV-1)) {  // 2^n+1?
1007        unsigned ShImm = Log2_32(RHSV-1);
1008        if (ShImm >= 32)
1009          break;
1010        SDValue V = Op.getOperand(0);
1011        ShImm = ARM_AM::getSORegOpc(ARM_AM::lsl, ShImm);
1012        SDValue ShImmOp = CurDAG->getTargetConstant(ShImm, MVT::i32);
1013        SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
1014        if (Subtarget->isThumb()) {
1015          SDValue Ops[] = { V, V, ShImmOp, getAL(CurDAG), Reg0, Reg0 };
1016          return CurDAG->SelectNodeTo(N, ARM::t2ADDrs, MVT::i32, Ops, 6);
1017        } else {
1018          SDValue Ops[] = { V, V, Reg0, ShImmOp, getAL(CurDAG), Reg0, Reg0 };
1019          return CurDAG->SelectNodeTo(N, ARM::ADDrs, MVT::i32, Ops, 7);
1020        }
1021      }
1022      if (isPowerOf2_32(RHSV+1)) {  // 2^n-1?
1023        unsigned ShImm = Log2_32(RHSV+1);
1024        if (ShImm >= 32)
1025          break;
1026        SDValue V = Op.getOperand(0);
1027        ShImm = ARM_AM::getSORegOpc(ARM_AM::lsl, ShImm);
1028        SDValue ShImmOp = CurDAG->getTargetConstant(ShImm, MVT::i32);
1029        SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
1030        if (Subtarget->isThumb()) {
1031          SDValue Ops[] = { V, V, ShImmOp, getAL(CurDAG), Reg0 };
1032          return CurDAG->SelectNodeTo(N, ARM::t2RSBrs, MVT::i32, Ops, 5);
1033        } else {
1034          SDValue Ops[] = { V, V, Reg0, ShImmOp, getAL(CurDAG), Reg0, Reg0 };
1035          return CurDAG->SelectNodeTo(N, ARM::RSBrs, MVT::i32, Ops, 7);
1036        }
1037      }
1038    }
1039    break;
1040  case ARMISD::FMRRD:
1041    return CurDAG->getMachineNode(ARM::FMRRD, dl, MVT::i32, MVT::i32,
1042                                  Op.getOperand(0), getAL(CurDAG),
1043                                  CurDAG->getRegister(0, MVT::i32));
1044  case ISD::UMUL_LOHI: {
1045    if (Subtarget->isThumb1Only())
1046      break;
1047    if (Subtarget->isThumb()) {
1048      SDValue Ops[] = { Op.getOperand(0), Op.getOperand(1),
1049                        getAL(CurDAG), CurDAG->getRegister(0, MVT::i32),
1050                        CurDAG->getRegister(0, MVT::i32) };
1051      return CurDAG->getMachineNode(ARM::t2UMULL, dl, MVT::i32, MVT::i32, Ops,4);
1052    } else {
1053      SDValue Ops[] = { Op.getOperand(0), Op.getOperand(1),
1054                        getAL(CurDAG), CurDAG->getRegister(0, MVT::i32),
1055                        CurDAG->getRegister(0, MVT::i32) };
1056      return CurDAG->getMachineNode(ARM::UMULL, dl, MVT::i32, MVT::i32, Ops, 5);
1057    }
1058  }
1059  case ISD::SMUL_LOHI: {
1060    if (Subtarget->isThumb1Only())
1061      break;
1062    if (Subtarget->isThumb()) {
1063      SDValue Ops[] = { Op.getOperand(0), Op.getOperand(1),
1064                        getAL(CurDAG), CurDAG->getRegister(0, MVT::i32) };
1065      return CurDAG->getMachineNode(ARM::t2SMULL, dl, MVT::i32, MVT::i32, Ops,4);
1066    } else {
1067      SDValue Ops[] = { Op.getOperand(0), Op.getOperand(1),
1068                        getAL(CurDAG), CurDAG->getRegister(0, MVT::i32),
1069                        CurDAG->getRegister(0, MVT::i32) };
1070      return CurDAG->getMachineNode(ARM::SMULL, dl, MVT::i32, MVT::i32, Ops, 5);
1071    }
1072  }
1073  case ISD::LOAD: {
1074    SDNode *ResNode = 0;
1075    if (Subtarget->isThumb() && Subtarget->hasThumb2())
1076      ResNode = SelectT2IndexedLoad(Op);
1077    else
1078      ResNode = SelectARMIndexedLoad(Op);
1079    if (ResNode)
1080      return ResNode;
1081    // Other cases are autogenerated.
1082    break;
1083  }
1084  case ARMISD::BRCOND: {
1085    // Pattern: (ARMbrcond:void (bb:Other):$dst, (imm:i32):$cc)
1086    // Emits: (Bcc:void (bb:Other):$dst, (imm:i32):$cc)
1087    // Pattern complexity = 6  cost = 1  size = 0
1088
1089    // Pattern: (ARMbrcond:void (bb:Other):$dst, (imm:i32):$cc)
1090    // Emits: (tBcc:void (bb:Other):$dst, (imm:i32):$cc)
1091    // Pattern complexity = 6  cost = 1  size = 0
1092
1093    // Pattern: (ARMbrcond:void (bb:Other):$dst, (imm:i32):$cc)
1094    // Emits: (t2Bcc:void (bb:Other):$dst, (imm:i32):$cc)
1095    // Pattern complexity = 6  cost = 1  size = 0
1096
1097    unsigned Opc = Subtarget->isThumb() ?
1098      ((Subtarget->hasThumb2()) ? ARM::t2Bcc : ARM::tBcc) : ARM::Bcc;
1099    SDValue Chain = Op.getOperand(0);
1100    SDValue N1 = Op.getOperand(1);
1101    SDValue N2 = Op.getOperand(2);
1102    SDValue N3 = Op.getOperand(3);
1103    SDValue InFlag = Op.getOperand(4);
1104    assert(N1.getOpcode() == ISD::BasicBlock);
1105    assert(N2.getOpcode() == ISD::Constant);
1106    assert(N3.getOpcode() == ISD::Register);
1107
1108    SDValue Tmp2 = CurDAG->getTargetConstant(((unsigned)
1109                               cast<ConstantSDNode>(N2)->getZExtValue()),
1110                               MVT::i32);
1111    SDValue Ops[] = { N1, Tmp2, N3, Chain, InFlag };
1112    SDNode *ResNode = CurDAG->getMachineNode(Opc, dl, MVT::Other,
1113                                             MVT::Flag, Ops, 5);
1114    Chain = SDValue(ResNode, 0);
1115    if (Op.getNode()->getNumValues() == 2) {
1116      InFlag = SDValue(ResNode, 1);
1117      ReplaceUses(SDValue(Op.getNode(), 1), InFlag);
1118    }
1119    ReplaceUses(SDValue(Op.getNode(), 0), SDValue(Chain.getNode(), Chain.getResNo()));
1120    return NULL;
1121  }
1122  case ARMISD::CMOV: {
1123    EVT VT = Op.getValueType();
1124    SDValue N0 = Op.getOperand(0);
1125    SDValue N1 = Op.getOperand(1);
1126    SDValue N2 = Op.getOperand(2);
1127    SDValue N3 = Op.getOperand(3);
1128    SDValue InFlag = Op.getOperand(4);
1129    assert(N2.getOpcode() == ISD::Constant);
1130    assert(N3.getOpcode() == ISD::Register);
1131
1132    if (!Subtarget->isThumb1Only() && VT == MVT::i32) {
1133      // Pattern: (ARMcmov:i32 GPR:i32:$false, so_reg:i32:$true, (imm:i32):$cc)
1134      // Emits: (MOVCCs:i32 GPR:i32:$false, so_reg:i32:$true, (imm:i32):$cc)
1135      // Pattern complexity = 18  cost = 1  size = 0
1136      SDValue CPTmp0;
1137      SDValue CPTmp1;
1138      SDValue CPTmp2;
1139      if (Subtarget->isThumb()) {
1140        if (SelectT2ShifterOperandReg(Op, N1, CPTmp0, CPTmp1)) {
1141          unsigned SOVal = cast<ConstantSDNode>(CPTmp1)->getZExtValue();
1142          unsigned SOShOp = ARM_AM::getSORegShOp(SOVal);
1143          unsigned Opc = 0;
1144          switch (SOShOp) {
1145          case ARM_AM::lsl: Opc = ARM::t2MOVCClsl; break;
1146          case ARM_AM::lsr: Opc = ARM::t2MOVCClsr; break;
1147          case ARM_AM::asr: Opc = ARM::t2MOVCCasr; break;
1148          case ARM_AM::ror: Opc = ARM::t2MOVCCror; break;
1149          default:
1150            llvm_unreachable("Unknown so_reg opcode!");
1151            break;
1152          }
1153          SDValue SOShImm =
1154            CurDAG->getTargetConstant(ARM_AM::getSORegOffset(SOVal), MVT::i32);
1155          SDValue Tmp2 = CurDAG->getTargetConstant(((unsigned)
1156                                   cast<ConstantSDNode>(N2)->getZExtValue()),
1157                                   MVT::i32);
1158          SDValue Ops[] = { N0, CPTmp0, SOShImm, Tmp2, N3, InFlag };
1159          return CurDAG->SelectNodeTo(Op.getNode(), Opc, MVT::i32,Ops, 6);
1160        }
1161      } else {
1162        if (SelectShifterOperandReg(Op, N1, CPTmp0, CPTmp1, CPTmp2)) {
1163          SDValue Tmp2 = CurDAG->getTargetConstant(((unsigned)
1164                                   cast<ConstantSDNode>(N2)->getZExtValue()),
1165                                   MVT::i32);
1166          SDValue Ops[] = { N0, CPTmp0, CPTmp1, CPTmp2, Tmp2, N3, InFlag };
1167          return CurDAG->SelectNodeTo(Op.getNode(),
1168                                      ARM::MOVCCs, MVT::i32, Ops, 7);
1169        }
1170      }
1171
1172      // Pattern: (ARMcmov:i32 GPR:i32:$false,
1173      //             (imm:i32)<<P:Predicate_so_imm>>:$true,
1174      //             (imm:i32):$cc)
1175      // Emits: (MOVCCi:i32 GPR:i32:$false,
1176      //           (so_imm:i32 (imm:i32):$true), (imm:i32):$cc)
1177      // Pattern complexity = 10  cost = 1  size = 0
1178      if (N3.getOpcode() == ISD::Constant) {
1179        if (Subtarget->isThumb()) {
1180          if (Predicate_t2_so_imm(N3.getNode())) {
1181            SDValue Tmp1 = CurDAG->getTargetConstant(((unsigned)
1182                                     cast<ConstantSDNode>(N1)->getZExtValue()),
1183                                     MVT::i32);
1184            SDValue Tmp2 = CurDAG->getTargetConstant(((unsigned)
1185                                     cast<ConstantSDNode>(N2)->getZExtValue()),
1186                                     MVT::i32);
1187            SDValue Ops[] = { N0, Tmp1, Tmp2, N3, InFlag };
1188            return CurDAG->SelectNodeTo(Op.getNode(),
1189                                        ARM::t2MOVCCi, MVT::i32, Ops, 5);
1190          }
1191        } else {
1192          if (Predicate_so_imm(N3.getNode())) {
1193            SDValue Tmp1 = CurDAG->getTargetConstant(((unsigned)
1194                                     cast<ConstantSDNode>(N1)->getZExtValue()),
1195                                     MVT::i32);
1196            SDValue Tmp2 = CurDAG->getTargetConstant(((unsigned)
1197                                     cast<ConstantSDNode>(N2)->getZExtValue()),
1198                                     MVT::i32);
1199            SDValue Ops[] = { N0, Tmp1, Tmp2, N3, InFlag };
1200            return CurDAG->SelectNodeTo(Op.getNode(),
1201                                        ARM::MOVCCi, MVT::i32, Ops, 5);
1202          }
1203        }
1204      }
1205    }
1206
1207    // Pattern: (ARMcmov:i32 GPR:i32:$false, GPR:i32:$true, (imm:i32):$cc)
1208    // Emits: (MOVCCr:i32 GPR:i32:$false, GPR:i32:$true, (imm:i32):$cc)
1209    // Pattern complexity = 6  cost = 1  size = 0
1210    //
1211    // Pattern: (ARMcmov:i32 GPR:i32:$false, GPR:i32:$true, (imm:i32):$cc)
1212    // Emits: (tMOVCCr:i32 GPR:i32:$false, GPR:i32:$true, (imm:i32):$cc)
1213    // Pattern complexity = 6  cost = 11  size = 0
1214    //
1215    // Also FCPYScc and FCPYDcc.
1216    SDValue Tmp2 = CurDAG->getTargetConstant(((unsigned)
1217                               cast<ConstantSDNode>(N2)->getZExtValue()),
1218                               MVT::i32);
1219    SDValue Ops[] = { N0, N1, Tmp2, N3, InFlag };
1220    unsigned Opc = 0;
1221    switch (VT.getSimpleVT().SimpleTy) {
1222    default: assert(false && "Illegal conditional move type!");
1223      break;
1224    case MVT::i32:
1225      Opc = Subtarget->isThumb()
1226        ? (Subtarget->hasThumb2() ? ARM::t2MOVCCr : ARM::tMOVCCr_pseudo)
1227        : ARM::MOVCCr;
1228      break;
1229    case MVT::f32:
1230      Opc = ARM::FCPYScc;
1231      break;
1232    case MVT::f64:
1233      Opc = ARM::FCPYDcc;
1234      break;
1235    }
1236    return CurDAG->SelectNodeTo(Op.getNode(), Opc, VT, Ops, 5);
1237  }
1238  case ARMISD::CNEG: {
1239    EVT VT = Op.getValueType();
1240    SDValue N0 = Op.getOperand(0);
1241    SDValue N1 = Op.getOperand(1);
1242    SDValue N2 = Op.getOperand(2);
1243    SDValue N3 = Op.getOperand(3);
1244    SDValue InFlag = Op.getOperand(4);
1245    assert(N2.getOpcode() == ISD::Constant);
1246    assert(N3.getOpcode() == ISD::Register);
1247
1248    SDValue Tmp2 = CurDAG->getTargetConstant(((unsigned)
1249                               cast<ConstantSDNode>(N2)->getZExtValue()),
1250                               MVT::i32);
1251    SDValue Ops[] = { N0, N1, Tmp2, N3, InFlag };
1252    unsigned Opc = 0;
1253    switch (VT.getSimpleVT().SimpleTy) {
1254    default: assert(false && "Illegal conditional move type!");
1255      break;
1256    case MVT::f32:
1257      Opc = ARM::FNEGScc;
1258      break;
1259    case MVT::f64:
1260      Opc = ARM::FNEGDcc;
1261      break;
1262    }
1263    return CurDAG->SelectNodeTo(Op.getNode(), Opc, VT, Ops, 5);
1264  }
1265
1266  case ARMISD::VZIP: {
1267    unsigned Opc = 0;
1268    EVT VT = N->getValueType(0);
1269    switch (VT.getSimpleVT().SimpleTy) {
1270    default: return NULL;
1271    case MVT::v8i8:  Opc = ARM::VZIPd8; break;
1272    case MVT::v4i16: Opc = ARM::VZIPd16; break;
1273    case MVT::v2f32:
1274    case MVT::v2i32: Opc = ARM::VZIPd32; break;
1275    case MVT::v16i8: Opc = ARM::VZIPq8; break;
1276    case MVT::v8i16: Opc = ARM::VZIPq16; break;
1277    case MVT::v4f32:
1278    case MVT::v4i32: Opc = ARM::VZIPq32; break;
1279    }
1280    return CurDAG->getMachineNode(Opc, dl, VT, VT,
1281                                  N->getOperand(0), N->getOperand(1));
1282  }
1283  case ARMISD::VUZP: {
1284    unsigned Opc = 0;
1285    EVT VT = N->getValueType(0);
1286    switch (VT.getSimpleVT().SimpleTy) {
1287    default: return NULL;
1288    case MVT::v8i8:  Opc = ARM::VUZPd8; break;
1289    case MVT::v4i16: Opc = ARM::VUZPd16; break;
1290    case MVT::v2f32:
1291    case MVT::v2i32: Opc = ARM::VUZPd32; break;
1292    case MVT::v16i8: Opc = ARM::VUZPq8; break;
1293    case MVT::v8i16: Opc = ARM::VUZPq16; break;
1294    case MVT::v4f32:
1295    case MVT::v4i32: Opc = ARM::VUZPq32; break;
1296    }
1297    return CurDAG->getMachineNode(Opc, dl, VT, VT,
1298                                  N->getOperand(0), N->getOperand(1));
1299  }
1300  case ARMISD::VTRN: {
1301    unsigned Opc = 0;
1302    EVT VT = N->getValueType(0);
1303    switch (VT.getSimpleVT().SimpleTy) {
1304    default: return NULL;
1305    case MVT::v8i8:  Opc = ARM::VTRNd8; break;
1306    case MVT::v4i16: Opc = ARM::VTRNd16; break;
1307    case MVT::v2f32:
1308    case MVT::v2i32: Opc = ARM::VTRNd32; break;
1309    case MVT::v16i8: Opc = ARM::VTRNq8; break;
1310    case MVT::v8i16: Opc = ARM::VTRNq16; break;
1311    case MVT::v4f32:
1312    case MVT::v4i32: Opc = ARM::VTRNq32; break;
1313    }
1314    return CurDAG->getMachineNode(Opc, dl, VT, VT,
1315                                  N->getOperand(0), N->getOperand(1));
1316  }
1317
1318  case ISD::INTRINSIC_VOID:
1319  case ISD::INTRINSIC_W_CHAIN: {
1320    unsigned IntNo = cast<ConstantSDNode>(N->getOperand(1))->getZExtValue();
1321    EVT VT = N->getValueType(0);
1322    unsigned Opc = 0;
1323
1324    switch (IntNo) {
1325    default:
1326      break;
1327
1328    case Intrinsic::arm_neon_vld2: {
1329      SDValue MemAddr, MemUpdate, MemOpc;
1330      if (!SelectAddrMode6(Op, N->getOperand(2), MemAddr, MemUpdate, MemOpc))
1331        return NULL;
1332      switch (VT.getSimpleVT().SimpleTy) {
1333      default: llvm_unreachable("unhandled vld2 type");
1334      case MVT::v8i8:  Opc = ARM::VLD2d8; break;
1335      case MVT::v4i16: Opc = ARM::VLD2d16; break;
1336      case MVT::v2f32:
1337      case MVT::v2i32: Opc = ARM::VLD2d32; break;
1338      }
1339      SDValue Chain = N->getOperand(0);
1340      const SDValue Ops[] = { MemAddr, MemUpdate, MemOpc, Chain };
1341      return CurDAG->getMachineNode(Opc, dl, VT, VT, MVT::Other, Ops, 4);
1342    }
1343
1344    case Intrinsic::arm_neon_vld3: {
1345      SDValue MemAddr, MemUpdate, MemOpc;
1346      if (!SelectAddrMode6(Op, N->getOperand(2), MemAddr, MemUpdate, MemOpc))
1347        return NULL;
1348      switch (VT.getSimpleVT().SimpleTy) {
1349      default: llvm_unreachable("unhandled vld3 type");
1350      case MVT::v8i8:  Opc = ARM::VLD3d8; break;
1351      case MVT::v4i16: Opc = ARM::VLD3d16; break;
1352      case MVT::v2f32:
1353      case MVT::v2i32: Opc = ARM::VLD3d32; break;
1354      }
1355      SDValue Chain = N->getOperand(0);
1356      const SDValue Ops[] = { MemAddr, MemUpdate, MemOpc, Chain };
1357      return CurDAG->getMachineNode(Opc, dl, VT, VT, VT, MVT::Other, Ops, 4);
1358    }
1359
1360    case Intrinsic::arm_neon_vld4: {
1361      SDValue MemAddr, MemUpdate, MemOpc;
1362      if (!SelectAddrMode6(Op, N->getOperand(2), MemAddr, MemUpdate, MemOpc))
1363        return NULL;
1364      switch (VT.getSimpleVT().SimpleTy) {
1365      default: llvm_unreachable("unhandled vld4 type");
1366      case MVT::v8i8:  Opc = ARM::VLD4d8; break;
1367      case MVT::v4i16: Opc = ARM::VLD4d16; break;
1368      case MVT::v2f32:
1369      case MVT::v2i32: Opc = ARM::VLD4d32; break;
1370      }
1371      SDValue Chain = N->getOperand(0);
1372      const SDValue Ops[] = { MemAddr, MemUpdate, MemOpc, Chain };
1373      std::vector<EVT> ResTys(4, VT);
1374      ResTys.push_back(MVT::Other);
1375      return CurDAG->getMachineNode(Opc, dl, ResTys, Ops, 4);
1376    }
1377
1378    case Intrinsic::arm_neon_vld2lane: {
1379      SDValue MemAddr, MemUpdate, MemOpc;
1380      if (!SelectAddrMode6(Op, N->getOperand(2), MemAddr, MemUpdate, MemOpc))
1381        return NULL;
1382      switch (VT.getSimpleVT().SimpleTy) {
1383      default: llvm_unreachable("unhandled vld2lane type");
1384      case MVT::v8i8:  Opc = ARM::VLD2LNd8; break;
1385      case MVT::v4i16: Opc = ARM::VLD2LNd16; break;
1386      case MVT::v2f32:
1387      case MVT::v2i32: Opc = ARM::VLD2LNd32; break;
1388      }
1389      SDValue Chain = N->getOperand(0);
1390      const SDValue Ops[] = { MemAddr, MemUpdate, MemOpc,
1391                              N->getOperand(3), N->getOperand(4),
1392                              N->getOperand(5), Chain };
1393      return CurDAG->getMachineNode(Opc, dl, VT, VT, MVT::Other, Ops, 7);
1394    }
1395
1396    case Intrinsic::arm_neon_vld3lane: {
1397      SDValue MemAddr, MemUpdate, MemOpc;
1398      if (!SelectAddrMode6(Op, N->getOperand(2), MemAddr, MemUpdate, MemOpc))
1399        return NULL;
1400      switch (VT.getSimpleVT().SimpleTy) {
1401      default: llvm_unreachable("unhandled vld3lane type");
1402      case MVT::v8i8:  Opc = ARM::VLD3LNd8; break;
1403      case MVT::v4i16: Opc = ARM::VLD3LNd16; break;
1404      case MVT::v2f32:
1405      case MVT::v2i32: Opc = ARM::VLD3LNd32; break;
1406      }
1407      SDValue Chain = N->getOperand(0);
1408      const SDValue Ops[] = { MemAddr, MemUpdate, MemOpc,
1409                              N->getOperand(3), N->getOperand(4),
1410                              N->getOperand(5), N->getOperand(6), Chain };
1411      return CurDAG->getMachineNode(Opc, dl, VT, VT, VT, MVT::Other, Ops, 8);
1412    }
1413
1414    case Intrinsic::arm_neon_vld4lane: {
1415      SDValue MemAddr, MemUpdate, MemOpc;
1416      if (!SelectAddrMode6(Op, N->getOperand(2), MemAddr, MemUpdate, MemOpc))
1417        return NULL;
1418      switch (VT.getSimpleVT().SimpleTy) {
1419      default: llvm_unreachable("unhandled vld4lane type");
1420      case MVT::v8i8:  Opc = ARM::VLD4LNd8; break;
1421      case MVT::v4i16: Opc = ARM::VLD4LNd16; break;
1422      case MVT::v2f32:
1423      case MVT::v2i32: Opc = ARM::VLD4LNd32; break;
1424      }
1425      SDValue Chain = N->getOperand(0);
1426      const SDValue Ops[] = { MemAddr, MemUpdate, MemOpc,
1427                              N->getOperand(3), N->getOperand(4),
1428                              N->getOperand(5), N->getOperand(6),
1429                              N->getOperand(7), Chain };
1430      std::vector<EVT> ResTys(4, VT);
1431      ResTys.push_back(MVT::Other);
1432      return CurDAG->getMachineNode(Opc, dl, ResTys, Ops, 9);
1433    }
1434
1435    case Intrinsic::arm_neon_vst2: {
1436      SDValue MemAddr, MemUpdate, MemOpc;
1437      if (!SelectAddrMode6(Op, N->getOperand(2), MemAddr, MemUpdate, MemOpc))
1438        return NULL;
1439      switch (N->getOperand(3).getValueType().getSimpleVT().SimpleTy) {
1440      default: llvm_unreachable("unhandled vst2 type");
1441      case MVT::v8i8:  Opc = ARM::VST2d8; break;
1442      case MVT::v4i16: Opc = ARM::VST2d16; break;
1443      case MVT::v2f32:
1444      case MVT::v2i32: Opc = ARM::VST2d32; break;
1445      }
1446      SDValue Chain = N->getOperand(0);
1447      const SDValue Ops[] = { MemAddr, MemUpdate, MemOpc,
1448                              N->getOperand(3), N->getOperand(4), Chain };
1449      return CurDAG->getMachineNode(Opc, dl, MVT::Other, Ops, 6);
1450    }
1451
1452    case Intrinsic::arm_neon_vst3: {
1453      SDValue MemAddr, MemUpdate, MemOpc;
1454      if (!SelectAddrMode6(Op, N->getOperand(2), MemAddr, MemUpdate, MemOpc))
1455        return NULL;
1456      switch (N->getOperand(3).getValueType().getSimpleVT().SimpleTy) {
1457      default: llvm_unreachable("unhandled vst3 type");
1458      case MVT::v8i8:  Opc = ARM::VST3d8; break;
1459      case MVT::v4i16: Opc = ARM::VST3d16; break;
1460      case MVT::v2f32:
1461      case MVT::v2i32: Opc = ARM::VST3d32; break;
1462      }
1463      SDValue Chain = N->getOperand(0);
1464      const SDValue Ops[] = { MemAddr, MemUpdate, MemOpc,
1465                              N->getOperand(3), N->getOperand(4),
1466                              N->getOperand(5), Chain };
1467      return CurDAG->getMachineNode(Opc, dl, MVT::Other, Ops, 7);
1468    }
1469
1470    case Intrinsic::arm_neon_vst4: {
1471      SDValue MemAddr, MemUpdate, MemOpc;
1472      if (!SelectAddrMode6(Op, N->getOperand(2), MemAddr, MemUpdate, MemOpc))
1473        return NULL;
1474      switch (N->getOperand(3).getValueType().getSimpleVT().SimpleTy) {
1475      default: llvm_unreachable("unhandled vst4 type");
1476      case MVT::v8i8:  Opc = ARM::VST4d8; break;
1477      case MVT::v4i16: Opc = ARM::VST4d16; break;
1478      case MVT::v2f32:
1479      case MVT::v2i32: Opc = ARM::VST4d32; break;
1480      }
1481      SDValue Chain = N->getOperand(0);
1482      const SDValue Ops[] = { MemAddr, MemUpdate, MemOpc,
1483                              N->getOperand(3), N->getOperand(4),
1484                              N->getOperand(5), N->getOperand(6), Chain };
1485      return CurDAG->getMachineNode(Opc, dl, MVT::Other, Ops, 8);
1486    }
1487
1488    case Intrinsic::arm_neon_vst2lane: {
1489      SDValue MemAddr, MemUpdate, MemOpc;
1490      if (!SelectAddrMode6(Op, N->getOperand(2), MemAddr, MemUpdate, MemOpc))
1491        return NULL;
1492      switch (N->getOperand(3).getValueType().getSimpleVT().SimpleTy) {
1493      default: llvm_unreachable("unhandled vst2lane type");
1494      case MVT::v8i8:  Opc = ARM::VST2LNd8; break;
1495      case MVT::v4i16: Opc = ARM::VST2LNd16; break;
1496      case MVT::v2f32:
1497      case MVT::v2i32: Opc = ARM::VST2LNd32; break;
1498      }
1499      SDValue Chain = N->getOperand(0);
1500      const SDValue Ops[] = { MemAddr, MemUpdate, MemOpc,
1501                              N->getOperand(3), N->getOperand(4),
1502                              N->getOperand(5), Chain };
1503      return CurDAG->getMachineNode(Opc, dl, MVT::Other, Ops, 7);
1504    }
1505
1506    case Intrinsic::arm_neon_vst3lane: {
1507      SDValue MemAddr, MemUpdate, MemOpc;
1508      if (!SelectAddrMode6(Op, N->getOperand(2), MemAddr, MemUpdate, MemOpc))
1509        return NULL;
1510      switch (N->getOperand(3).getValueType().getSimpleVT().SimpleTy) {
1511      default: llvm_unreachable("unhandled vst3lane type");
1512      case MVT::v8i8:  Opc = ARM::VST3LNd8; break;
1513      case MVT::v4i16: Opc = ARM::VST3LNd16; break;
1514      case MVT::v2f32:
1515      case MVT::v2i32: Opc = ARM::VST3LNd32; break;
1516      }
1517      SDValue Chain = N->getOperand(0);
1518      const SDValue Ops[] = { MemAddr, MemUpdate, MemOpc,
1519                              N->getOperand(3), N->getOperand(4),
1520                              N->getOperand(5), N->getOperand(6), Chain };
1521      return CurDAG->getMachineNode(Opc, dl, MVT::Other, Ops, 8);
1522    }
1523
1524    case Intrinsic::arm_neon_vst4lane: {
1525      SDValue MemAddr, MemUpdate, MemOpc;
1526      if (!SelectAddrMode6(Op, N->getOperand(2), MemAddr, MemUpdate, MemOpc))
1527        return NULL;
1528      switch (N->getOperand(3).getValueType().getSimpleVT().SimpleTy) {
1529      default: llvm_unreachable("unhandled vst4lane type");
1530      case MVT::v8i8:  Opc = ARM::VST4LNd8; break;
1531      case MVT::v4i16: Opc = ARM::VST4LNd16; break;
1532      case MVT::v2f32:
1533      case MVT::v2i32: Opc = ARM::VST4LNd32; break;
1534      }
1535      SDValue Chain = N->getOperand(0);
1536      const SDValue Ops[] = { MemAddr, MemUpdate, MemOpc,
1537                              N->getOperand(3), N->getOperand(4),
1538                              N->getOperand(5), N->getOperand(6),
1539                              N->getOperand(7), Chain };
1540      return CurDAG->getMachineNode(Opc, dl, MVT::Other, Ops, 9);
1541    }
1542    }
1543  }
1544  }
1545
1546  return SelectCode(Op);
1547}
1548
1549bool ARMDAGToDAGISel::
1550SelectInlineAsmMemoryOperand(const SDValue &Op, char ConstraintCode,
1551                             std::vector<SDValue> &OutOps) {
1552  assert(ConstraintCode == 'm' && "unexpected asm memory constraint");
1553
1554  SDValue Base, Offset, Opc;
1555  if (!SelectAddrMode2(Op, Op, Base, Offset, Opc))
1556    return true;
1557
1558  OutOps.push_back(Base);
1559  OutOps.push_back(Offset);
1560  OutOps.push_back(Opc);
1561  return false;
1562}
1563
1564/// createARMISelDag - This pass converts a legalized DAG into a
1565/// ARM-specific DAG, ready for instruction scheduling.
1566///
1567FunctionPass *llvm::createARMISelDag(ARMBaseTargetMachine &TM) {
1568  return new ARMDAGToDAGISel(TM);
1569}
1570