ARMISelDAGToDAG.cpp revision 7474bb0fbb9640f6f799eea35d7fbaebd2312a09 
1//===-- ARMISelDAGToDAG.cpp - A dag to dag inst selector for ARM ----------===//
2//
3//                     The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10// This file defines an instruction selector for the ARM target.
11//
12//===----------------------------------------------------------------------===//
13
14#include "ARM.h"
15#include "ARMAddressingModes.h"
16#include "ARMISelLowering.h"
17#include "ARMTargetMachine.h"
18#include "llvm/CallingConv.h"
19#include "llvm/Constants.h"
20#include "llvm/DerivedTypes.h"
21#include "llvm/Function.h"
22#include "llvm/Intrinsics.h"
23#include "llvm/LLVMContext.h"
24#include "llvm/CodeGen/MachineFrameInfo.h"
25#include "llvm/CodeGen/MachineFunction.h"
26#include "llvm/CodeGen/MachineInstrBuilder.h"
27#include "llvm/CodeGen/SelectionDAG.h"
28#include "llvm/CodeGen/SelectionDAGISel.h"
29#include "llvm/Target/TargetLowering.h"
30#include "llvm/Target/TargetOptions.h"
31#include "llvm/Support/Compiler.h"
32#include "llvm/Support/Debug.h"
33#include "llvm/Support/ErrorHandling.h"
34#include "llvm/Support/raw_ostream.h"
35
36using namespace llvm;
37
38//===--------------------------------------------------------------------===//
39/// ARMDAGToDAGISel - ARM specific code to select ARM machine
40/// instructions for SelectionDAG operations.
41///
42namespace {
43class ARMDAGToDAGISel : public SelectionDAGISel {
44  ARMBaseTargetMachine &TM;
45
46  /// Subtarget - Keep a pointer to the ARMSubtarget around so that we can
47  /// make the right decision when generating code for different targets.
48  const ARMSubtarget *Subtarget;
49
50public:
51  explicit ARMDAGToDAGISel(ARMBaseTargetMachine &tm,
52                           CodeGenOpt::Level OptLevel)
53    : SelectionDAGISel(tm, OptLevel), TM(tm),
54    Subtarget(&TM.getSubtarget<ARMSubtarget>()) {
55  }
56
57  virtual const char *getPassName() const {
58    return "ARM Instruction Selection";
59  }
60
61  /// getI32Imm - Return a target constant of type i32 with the specified
62  /// value.
63  inline SDValue getI32Imm(unsigned Imm) {
64    return CurDAG->getTargetConstant(Imm, MVT::i32);
65  }
66
67  SDNode *Select(SDNode *N);
68
69  bool SelectShifterOperandReg(SDNode *Op, SDValue N, SDValue &A,
70                               SDValue &B, SDValue &C);
71  bool SelectAddrMode2(SDNode *Op, SDValue N, SDValue &Base,
72                       SDValue &Offset, SDValue &Opc);
73  bool SelectAddrMode2Offset(SDNode *Op, SDValue N,
74                             SDValue &Offset, SDValue &Opc);
75  bool SelectAddrMode3(SDNode *Op, SDValue N, SDValue &Base,
76                       SDValue &Offset, SDValue &Opc);
77  bool SelectAddrMode3Offset(SDNode *Op, SDValue N,
78                             SDValue &Offset, SDValue &Opc);
79  bool SelectAddrMode4(SDNode *Op, SDValue N, SDValue &Addr,
80                       SDValue &Mode);
81  bool SelectAddrMode5(SDNode *Op, SDValue N, SDValue &Base,
82                       SDValue &Offset);
83  bool SelectAddrMode6(SDNode *Op, SDValue N, SDValue &Addr, SDValue &Align);
84
85  bool SelectAddrModePC(SDNode *Op, SDValue N, SDValue &Offset,
86                        SDValue &Label);
87
88  bool SelectThumbAddrModeRR(SDNode *Op, SDValue N, SDValue &Base,
89                             SDValue &Offset);
90  bool SelectThumbAddrModeRI5(SDNode *Op, SDValue N, unsigned Scale,
91                              SDValue &Base, SDValue &OffImm,
92                              SDValue &Offset);
93  bool SelectThumbAddrModeS1(SDNode *Op, SDValue N, SDValue &Base,
94                             SDValue &OffImm, SDValue &Offset);
95  bool SelectThumbAddrModeS2(SDNode *Op, SDValue N, SDValue &Base,
96                             SDValue &OffImm, SDValue &Offset);
97  bool SelectThumbAddrModeS4(SDNode *Op, SDValue N, SDValue &Base,
98                             SDValue &OffImm, SDValue &Offset);
99  bool SelectThumbAddrModeSP(SDNode *Op, SDValue N, SDValue &Base,
100                             SDValue &OffImm);
101
102  bool SelectT2ShifterOperandReg(SDNode *Op, SDValue N,
103                                 SDValue &BaseReg, SDValue &Opc);
104  bool SelectT2AddrModeImm12(SDNode *Op, SDValue N, SDValue &Base,
105                             SDValue &OffImm);
106  bool SelectT2AddrModeImm8(SDNode *Op, SDValue N, SDValue &Base,
107                            SDValue &OffImm);
108  bool SelectT2AddrModeImm8Offset(SDNode *Op, SDValue N,
109                                 SDValue &OffImm);
110  bool SelectT2AddrModeImm8s4(SDNode *Op, SDValue N, SDValue &Base,
111                              SDValue &OffImm);
112  bool SelectT2AddrModeSoReg(SDNode *Op, SDValue N, SDValue &Base,
113                             SDValue &OffReg, SDValue &ShImm);
114
115  // Include the pieces autogenerated from the target description.
116#include "ARMGenDAGISel.inc"
117
118private:
119  /// SelectARMIndexedLoad - Indexed (pre/post inc/dec) load matching code for
120  /// ARM.
121  SDNode *SelectARMIndexedLoad(SDNode *N);
122  SDNode *SelectT2IndexedLoad(SDNode *N);
123
124  /// SelectDYN_ALLOC - Select dynamic alloc for Thumb.
125  SDNode *SelectDYN_ALLOC(SDNode *N);
126
127  /// SelectVLD - Select NEON load intrinsics.  NumVecs should
128  /// be 2, 3 or 4.  The opcode arrays specify the instructions used for
129  /// loads of D registers and even subregs and odd subregs of Q registers.
130  /// For NumVecs == 2, QOpcodes1 is not used.
131  SDNode *SelectVLD(SDNode *N, unsigned NumVecs, unsigned *DOpcodes,
132                    unsigned *QOpcodes0, unsigned *QOpcodes1);
133
134  /// SelectVST - Select NEON store intrinsics.  NumVecs should
135  /// be 2, 3 or 4.  The opcode arrays specify the instructions used for
136  /// stores of D registers and even subregs and odd subregs of Q registers.
137  /// For NumVecs == 2, QOpcodes1 is not used.
138  SDNode *SelectVST(SDNode *N, unsigned NumVecs, unsigned *DOpcodes,
139                    unsigned *QOpcodes0, unsigned *QOpcodes1);
140
141  /// SelectVLDSTLane - Select NEON load/store lane intrinsics.  NumVecs should
142  /// be 2, 3 or 4.  The opcode arrays specify the instructions used for
143  /// load/store of D registers and even subregs and odd subregs of Q registers.
144  SDNode *SelectVLDSTLane(SDNode *N, bool IsLoad, unsigned NumVecs,
145                          unsigned *DOpcodes, unsigned *QOpcodes0,
146                          unsigned *QOpcodes1);
147
148  /// SelectV6T2BitfieldExtractOp - Select SBFX/UBFX instructions for ARM.
149  SDNode *SelectV6T2BitfieldExtractOp(SDNode *N, unsigned Opc);
150
151  /// SelectCMOVOp - Select CMOV instructions for ARM.
152  SDNode *SelectCMOVOp(SDNode *N);
153  SDNode *SelectT2CMOVShiftOp(SDNode *N, SDValue FalseVal, SDValue TrueVal,
154                              ARMCC::CondCodes CCVal, SDValue CCR,
155                              SDValue InFlag);
156  SDNode *SelectARMCMOVShiftOp(SDNode *N, SDValue FalseVal, SDValue TrueVal,
157                               ARMCC::CondCodes CCVal, SDValue CCR,
158                               SDValue InFlag);
159  SDNode *SelectT2CMOVSoImmOp(SDNode *N, SDValue FalseVal, SDValue TrueVal,
160                              ARMCC::CondCodes CCVal, SDValue CCR,
161                              SDValue InFlag);
162  SDNode *SelectARMCMOVSoImmOp(SDNode *N, SDValue FalseVal, SDValue TrueVal,
163                               ARMCC::CondCodes CCVal, SDValue CCR,
164                               SDValue InFlag);
165
166  /// SelectInlineAsmMemoryOperand - Implement addressing mode selection for
167  /// inline asm expressions.
168  virtual bool SelectInlineAsmMemoryOperand(const SDValue &Op,
169                                            char ConstraintCode,
170                                            std::vector<SDValue> &OutOps);
171
172  /// PairDRegs - Insert a pair of double registers into an implicit def to
173  /// form a quad register.
174  SDNode *PairDRegs(EVT VT, SDValue V0, SDValue V1);
175};
176}
177
178/// isInt32Immediate - This method tests to see if the node is a 32-bit constant
179/// operand. If so Imm will receive the 32-bit value.
180static bool isInt32Immediate(SDNode *N, unsigned &Imm) {
181  if (N->getOpcode() == ISD::Constant && N->getValueType(0) == MVT::i32) {
182    Imm = cast<ConstantSDNode>(N)->getZExtValue();
183    return true;
184  }
185  return false;
186}
187
188// isInt32Immediate - This method tests to see if a constant operand.
189// If so Imm will receive the 32 bit value.
190static bool isInt32Immediate(SDValue N, unsigned &Imm) {
191  return isInt32Immediate(N.getNode(), Imm);
192}
193
194// isOpcWithIntImmediate - This method tests to see if the node is a specific
195// opcode and that it has a immediate integer right operand.
196// If so Imm will receive the 32 bit value.
197static bool isOpcWithIntImmediate(SDNode *N, unsigned Opc, unsigned& Imm) {
198  return N->getOpcode() == Opc &&
199         isInt32Immediate(N->getOperand(1).getNode(), Imm);
200}
201
202
203bool ARMDAGToDAGISel::SelectShifterOperandReg(SDNode *Op,
204                                              SDValue N,
205                                              SDValue &BaseReg,
206                                              SDValue &ShReg,
207                                              SDValue &Opc) {
208  ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(N);
209
210  // Don't match base register only case. That is matched to a separate
211  // lower complexity pattern with explicit register operand.
212  if (ShOpcVal == ARM_AM::no_shift) return false;
213
214  BaseReg = N.getOperand(0);
215  unsigned ShImmVal = 0;
216  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
217    ShReg = CurDAG->getRegister(0, MVT::i32);
218    ShImmVal = RHS->getZExtValue() & 31;
219  } else {
220    ShReg = N.getOperand(1);
221  }
222  Opc = CurDAG->getTargetConstant(ARM_AM::getSORegOpc(ShOpcVal, ShImmVal),
223                                  MVT::i32);
224  return true;
225}
226
227bool ARMDAGToDAGISel::SelectAddrMode2(SDNode *Op, SDValue N,
228                                      SDValue &Base, SDValue &Offset,
229                                      SDValue &Opc) {
230  if (N.getOpcode() == ISD::MUL) {
231    if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
232      // X * [3,5,9] -> X + X * [2,4,8] etc.
233      int RHSC = (int)RHS->getZExtValue();
234      if (RHSC & 1) {
235        RHSC = RHSC & ~1;
236        ARM_AM::AddrOpc AddSub = ARM_AM::add;
237        if (RHSC < 0) {
238          AddSub = ARM_AM::sub;
239          RHSC = - RHSC;
240        }
241        if (isPowerOf2_32(RHSC)) {
242          unsigned ShAmt = Log2_32(RHSC);
243          Base = Offset = N.getOperand(0);
244          Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, ShAmt,
245                                                            ARM_AM::lsl),
246                                          MVT::i32);
247          return true;
248        }
249      }
250    }
251  }
252
253  if (N.getOpcode() != ISD::ADD && N.getOpcode() != ISD::SUB) {
254    Base = N;
255    if (N.getOpcode() == ISD::FrameIndex) {
256      int FI = cast<FrameIndexSDNode>(N)->getIndex();
257      Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
258    } else if (N.getOpcode() == ARMISD::Wrapper &&
259               !(Subtarget->useMovt() &&
260                 N.getOperand(0).getOpcode() == ISD::TargetGlobalAddress)) {
261      Base = N.getOperand(0);
262    }
263    Offset = CurDAG->getRegister(0, MVT::i32);
264    Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(ARM_AM::add, 0,
265                                                      ARM_AM::no_shift),
266                                    MVT::i32);
267    return true;
268  }
269
270  // Match simple R +/- imm12 operands.
271  if (N.getOpcode() == ISD::ADD)
272    if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
273      int RHSC = (int)RHS->getZExtValue();
274      if ((RHSC >= 0 && RHSC < 0x1000) ||
275          (RHSC < 0 && RHSC > -0x1000)) { // 12 bits.
276        Base = N.getOperand(0);
277        if (Base.getOpcode() == ISD::FrameIndex) {
278          int FI = cast<FrameIndexSDNode>(Base)->getIndex();
279          Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
280        }
281        Offset = CurDAG->getRegister(0, MVT::i32);
282
283        ARM_AM::AddrOpc AddSub = ARM_AM::add;
284        if (RHSC < 0) {
285          AddSub = ARM_AM::sub;
286          RHSC = - RHSC;
287        }
288        Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, RHSC,
289                                                          ARM_AM::no_shift),
290                                        MVT::i32);
291        return true;
292      }
293    }
294
295  // Otherwise this is R +/- [possibly shifted] R.
296  ARM_AM::AddrOpc AddSub = N.getOpcode() == ISD::ADD ? ARM_AM::add:ARM_AM::sub;
297  ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(N.getOperand(1));
298  unsigned ShAmt = 0;
299
300  Base   = N.getOperand(0);
301  Offset = N.getOperand(1);
302
303  if (ShOpcVal != ARM_AM::no_shift) {
304    // Check to see if the RHS of the shift is a constant, if not, we can't fold
305    // it.
306    if (ConstantSDNode *Sh =
307           dyn_cast<ConstantSDNode>(N.getOperand(1).getOperand(1))) {
308      ShAmt = Sh->getZExtValue();
309      Offset = N.getOperand(1).getOperand(0);
310    } else {
311      ShOpcVal = ARM_AM::no_shift;
312    }
313  }
314
315  // Try matching (R shl C) + (R).
316  if (N.getOpcode() == ISD::ADD && ShOpcVal == ARM_AM::no_shift) {
317    ShOpcVal = ARM_AM::getShiftOpcForNode(N.getOperand(0));
318    if (ShOpcVal != ARM_AM::no_shift) {
319      // Check to see if the RHS of the shift is a constant, if not, we can't
320      // fold it.
321      if (ConstantSDNode *Sh =
322          dyn_cast<ConstantSDNode>(N.getOperand(0).getOperand(1))) {
323        ShAmt = Sh->getZExtValue();
324        Offset = N.getOperand(0).getOperand(0);
325        Base = N.getOperand(1);
326      } else {
327        ShOpcVal = ARM_AM::no_shift;
328      }
329    }
330  }
331
332  Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, ShAmt, ShOpcVal),
333                                  MVT::i32);
334  return true;
335}
336
337bool ARMDAGToDAGISel::SelectAddrMode2Offset(SDNode *Op, SDValue N,
338                                            SDValue &Offset, SDValue &Opc) {
339  unsigned Opcode = Op->getOpcode();
340  ISD::MemIndexedMode AM = (Opcode == ISD::LOAD)
341    ? cast<LoadSDNode>(Op)->getAddressingMode()
342    : cast<StoreSDNode>(Op)->getAddressingMode();
343  ARM_AM::AddrOpc AddSub = (AM == ISD::PRE_INC || AM == ISD::POST_INC)
344    ? ARM_AM::add : ARM_AM::sub;
345  if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(N)) {
346    int Val = (int)C->getZExtValue();
347    if (Val >= 0 && Val < 0x1000) { // 12 bits.
348      Offset = CurDAG->getRegister(0, MVT::i32);
349      Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, Val,
350                                                        ARM_AM::no_shift),
351                                      MVT::i32);
352      return true;
353    }
354  }
355
356  Offset = N;
357  ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(N);
358  unsigned ShAmt = 0;
359  if (ShOpcVal != ARM_AM::no_shift) {
360    // Check to see if the RHS of the shift is a constant, if not, we can't fold
361    // it.
362    if (ConstantSDNode *Sh = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
363      ShAmt = Sh->getZExtValue();
364      Offset = N.getOperand(0);
365    } else {
366      ShOpcVal = ARM_AM::no_shift;
367    }
368  }
369
370  Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, ShAmt, ShOpcVal),
371                                  MVT::i32);
372  return true;
373}
374
375
376bool ARMDAGToDAGISel::SelectAddrMode3(SDNode *Op, SDValue N,
377                                      SDValue &Base, SDValue &Offset,
378                                      SDValue &Opc) {
379  if (N.getOpcode() == ISD::SUB) {
380    // X - C  is canonicalize to X + -C, no need to handle it here.
381    Base = N.getOperand(0);
382    Offset = N.getOperand(1);
383    Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(ARM_AM::sub, 0),MVT::i32);
384    return true;
385  }
386
387  if (N.getOpcode() != ISD::ADD) {
388    Base = N;
389    if (N.getOpcode() == ISD::FrameIndex) {
390      int FI = cast<FrameIndexSDNode>(N)->getIndex();
391      Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
392    }
393    Offset = CurDAG->getRegister(0, MVT::i32);
394    Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(ARM_AM::add, 0),MVT::i32);
395    return true;
396  }
397
398  // If the RHS is +/- imm8, fold into addr mode.
399  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
400    int RHSC = (int)RHS->getZExtValue();
401    if ((RHSC >= 0 && RHSC < 256) ||
402        (RHSC < 0 && RHSC > -256)) { // note -256 itself isn't allowed.
403      Base = N.getOperand(0);
404      if (Base.getOpcode() == ISD::FrameIndex) {
405        int FI = cast<FrameIndexSDNode>(Base)->getIndex();
406        Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
407      }
408      Offset = CurDAG->getRegister(0, MVT::i32);
409
410      ARM_AM::AddrOpc AddSub = ARM_AM::add;
411      if (RHSC < 0) {
412        AddSub = ARM_AM::sub;
413        RHSC = - RHSC;
414      }
415      Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(AddSub, RHSC),MVT::i32);
416      return true;
417    }
418  }
419
420  Base = N.getOperand(0);
421  Offset = N.getOperand(1);
422  Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(ARM_AM::add, 0), MVT::i32);
423  return true;
424}
425
426bool ARMDAGToDAGISel::SelectAddrMode3Offset(SDNode *Op, SDValue N,
427                                            SDValue &Offset, SDValue &Opc) {
428  unsigned Opcode = Op->getOpcode();
429  ISD::MemIndexedMode AM = (Opcode == ISD::LOAD)
430    ? cast<LoadSDNode>(Op)->getAddressingMode()
431    : cast<StoreSDNode>(Op)->getAddressingMode();
432  ARM_AM::AddrOpc AddSub = (AM == ISD::PRE_INC || AM == ISD::POST_INC)
433    ? ARM_AM::add : ARM_AM::sub;
434  if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(N)) {
435    int Val = (int)C->getZExtValue();
436    if (Val >= 0 && Val < 256) {
437      Offset = CurDAG->getRegister(0, MVT::i32);
438      Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(AddSub, Val), MVT::i32);
439      return true;
440    }
441  }
442
443  Offset = N;
444  Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(AddSub, 0), MVT::i32);
445  return true;
446}
447
448bool ARMDAGToDAGISel::SelectAddrMode4(SDNode *Op, SDValue N,
449                                      SDValue &Addr, SDValue &Mode) {
450  Addr = N;
451  Mode = CurDAG->getTargetConstant(0, MVT::i32);
452  return true;
453}
454
455bool ARMDAGToDAGISel::SelectAddrMode5(SDNode *Op, SDValue N,
456                                      SDValue &Base, SDValue &Offset) {
457  if (N.getOpcode() != ISD::ADD) {
458    Base = N;
459    if (N.getOpcode() == ISD::FrameIndex) {
460      int FI = cast<FrameIndexSDNode>(N)->getIndex();
461      Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
462    } else if (N.getOpcode() == ARMISD::Wrapper &&
463               !(Subtarget->useMovt() &&
464                 N.getOperand(0).getOpcode() == ISD::TargetGlobalAddress)) {
465      Base = N.getOperand(0);
466    }
467    Offset = CurDAG->getTargetConstant(ARM_AM::getAM5Opc(ARM_AM::add, 0),
468                                       MVT::i32);
469    return true;
470  }
471
472  // If the RHS is +/- imm8, fold into addr mode.
473  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
474    int RHSC = (int)RHS->getZExtValue();
475    if ((RHSC & 3) == 0) {  // The constant is implicitly multiplied by 4.
476      RHSC >>= 2;
477      if ((RHSC >= 0 && RHSC < 256) ||
478          (RHSC < 0 && RHSC > -256)) { // note -256 itself isn't allowed.
479        Base = N.getOperand(0);
480        if (Base.getOpcode() == ISD::FrameIndex) {
481          int FI = cast<FrameIndexSDNode>(Base)->getIndex();
482          Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
483        }
484
485        ARM_AM::AddrOpc AddSub = ARM_AM::add;
486        if (RHSC < 0) {
487          AddSub = ARM_AM::sub;
488          RHSC = - RHSC;
489        }
490        Offset = CurDAG->getTargetConstant(ARM_AM::getAM5Opc(AddSub, RHSC),
491                                           MVT::i32);
492        return true;
493      }
494    }
495  }
496
497  Base = N;
498  Offset = CurDAG->getTargetConstant(ARM_AM::getAM5Opc(ARM_AM::add, 0),
499                                     MVT::i32);
500  return true;
501}
502
503bool ARMDAGToDAGISel::SelectAddrMode6(SDNode *Op, SDValue N,
504                                      SDValue &Addr, SDValue &Align) {
505  Addr = N;
506  // Default to no alignment.
507  Align = CurDAG->getTargetConstant(0, MVT::i32);
508  return true;
509}
510
511bool ARMDAGToDAGISel::SelectAddrModePC(SDNode *Op, SDValue N,
512                                       SDValue &Offset, SDValue &Label) {
513  if (N.getOpcode() == ARMISD::PIC_ADD && N.hasOneUse()) {
514    Offset = N.getOperand(0);
515    SDValue N1 = N.getOperand(1);
516    Label  = CurDAG->getTargetConstant(cast<ConstantSDNode>(N1)->getZExtValue(),
517                                       MVT::i32);
518    return true;
519  }
520  return false;
521}
522
523bool ARMDAGToDAGISel::SelectThumbAddrModeRR(SDNode *Op, SDValue N,
524                                            SDValue &Base, SDValue &Offset){
525  // FIXME dl should come from the parent load or store, not the address
526  DebugLoc dl = Op->getDebugLoc();
527  if (N.getOpcode() != ISD::ADD) {
528    ConstantSDNode *NC = dyn_cast<ConstantSDNode>(N);
529    if (!NC || NC->getZExtValue() != 0)
530      return false;
531
532    Base = Offset = N;
533    return true;
534  }
535
536  Base = N.getOperand(0);
537  Offset = N.getOperand(1);
538  return true;
539}
540
541bool
542ARMDAGToDAGISel::SelectThumbAddrModeRI5(SDNode *Op, SDValue N,
543                                        unsigned Scale, SDValue &Base,
544                                        SDValue &OffImm, SDValue &Offset) {
545  if (Scale == 4) {
546    SDValue TmpBase, TmpOffImm;
547    if (SelectThumbAddrModeSP(Op, N, TmpBase, TmpOffImm))
548      return false;  // We want to select tLDRspi / tSTRspi instead.
549    if (N.getOpcode() == ARMISD::Wrapper &&
550        N.getOperand(0).getOpcode() == ISD::TargetConstantPool)
551      return false;  // We want to select tLDRpci instead.
552  }
553
554  if (N.getOpcode() != ISD::ADD) {
555    if (N.getOpcode() == ARMISD::Wrapper &&
556        !(Subtarget->useMovt() &&
557          N.getOperand(0).getOpcode() == ISD::TargetGlobalAddress)) {
558      Base = N.getOperand(0);
559    } else
560      Base = N;
561
562    Offset = CurDAG->getRegister(0, MVT::i32);
563    OffImm = CurDAG->getTargetConstant(0, MVT::i32);
564    return true;
565  }
566
567  // Thumb does not have [sp, r] address mode.
568  RegisterSDNode *LHSR = dyn_cast<RegisterSDNode>(N.getOperand(0));
569  RegisterSDNode *RHSR = dyn_cast<RegisterSDNode>(N.getOperand(1));
570  if ((LHSR && LHSR->getReg() == ARM::SP) ||
571      (RHSR && RHSR->getReg() == ARM::SP)) {
572    Base = N;
573    Offset = CurDAG->getRegister(0, MVT::i32);
574    OffImm = CurDAG->getTargetConstant(0, MVT::i32);
575    return true;
576  }
577
578  // If the RHS is + imm5 * scale, fold into addr mode.
579  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
580    int RHSC = (int)RHS->getZExtValue();
581    if ((RHSC & (Scale-1)) == 0) {  // The constant is implicitly multiplied.
582      RHSC /= Scale;
583      if (RHSC >= 0 && RHSC < 32) {
584        Base = N.getOperand(0);
585        Offset = CurDAG->getRegister(0, MVT::i32);
586        OffImm = CurDAG->getTargetConstant(RHSC, MVT::i32);
587        return true;
588      }
589    }
590  }
591
592  Base = N.getOperand(0);
593  Offset = N.getOperand(1);
594  OffImm = CurDAG->getTargetConstant(0, MVT::i32);
595  return true;
596}
597
598bool ARMDAGToDAGISel::SelectThumbAddrModeS1(SDNode *Op, SDValue N,
599                                            SDValue &Base, SDValue &OffImm,
600                                            SDValue &Offset) {
601  return SelectThumbAddrModeRI5(Op, N, 1, Base, OffImm, Offset);
602}
603
604bool ARMDAGToDAGISel::SelectThumbAddrModeS2(SDNode *Op, SDValue N,
605                                            SDValue &Base, SDValue &OffImm,
606                                            SDValue &Offset) {
607  return SelectThumbAddrModeRI5(Op, N, 2, Base, OffImm, Offset);
608}
609
610bool ARMDAGToDAGISel::SelectThumbAddrModeS4(SDNode *Op, SDValue N,
611                                            SDValue &Base, SDValue &OffImm,
612                                            SDValue &Offset) {
613  return SelectThumbAddrModeRI5(Op, N, 4, Base, OffImm, Offset);
614}
615
616bool ARMDAGToDAGISel::SelectThumbAddrModeSP(SDNode *Op, SDValue N,
617                                           SDValue &Base, SDValue &OffImm) {
618  if (N.getOpcode() == ISD::FrameIndex) {
619    int FI = cast<FrameIndexSDNode>(N)->getIndex();
620    Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
621    OffImm = CurDAG->getTargetConstant(0, MVT::i32);
622    return true;
623  }
624
625  if (N.getOpcode() != ISD::ADD)
626    return false;
627
628  RegisterSDNode *LHSR = dyn_cast<RegisterSDNode>(N.getOperand(0));
629  if (N.getOperand(0).getOpcode() == ISD::FrameIndex ||
630      (LHSR && LHSR->getReg() == ARM::SP)) {
631    // If the RHS is + imm8 * scale, fold into addr mode.
632    if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
633      int RHSC = (int)RHS->getZExtValue();
634      if ((RHSC & 3) == 0) {  // The constant is implicitly multiplied.
635        RHSC >>= 2;
636        if (RHSC >= 0 && RHSC < 256) {
637          Base = N.getOperand(0);
638          if (Base.getOpcode() == ISD::FrameIndex) {
639            int FI = cast<FrameIndexSDNode>(Base)->getIndex();
640            Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
641          }
642          OffImm = CurDAG->getTargetConstant(RHSC, MVT::i32);
643          return true;
644        }
645      }
646    }
647  }
648
649  return false;
650}
651
652bool ARMDAGToDAGISel::SelectT2ShifterOperandReg(SDNode *Op, SDValue N,
653                                                SDValue &BaseReg,
654                                                SDValue &Opc) {
655  ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(N);
656
657  // Don't match base register only case. That is matched to a separate
658  // lower complexity pattern with explicit register operand.
659  if (ShOpcVal == ARM_AM::no_shift) return false;
660
661  BaseReg = N.getOperand(0);
662  unsigned ShImmVal = 0;
663  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
664    ShImmVal = RHS->getZExtValue() & 31;
665    Opc = getI32Imm(ARM_AM::getSORegOpc(ShOpcVal, ShImmVal));
666    return true;
667  }
668
669  return false;
670}
671
672bool ARMDAGToDAGISel::SelectT2AddrModeImm12(SDNode *Op, SDValue N,
673                                            SDValue &Base, SDValue &OffImm) {
674  // Match simple R + imm12 operands.
675
676  // Base only.
677  if (N.getOpcode() != ISD::ADD && N.getOpcode() != ISD::SUB) {
678    if (N.getOpcode() == ISD::FrameIndex) {
679      // Match frame index...
680      int FI = cast<FrameIndexSDNode>(N)->getIndex();
681      Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
682      OffImm  = CurDAG->getTargetConstant(0, MVT::i32);
683      return true;
684    } else if (N.getOpcode() == ARMISD::Wrapper &&
685               !(Subtarget->useMovt() &&
686                 N.getOperand(0).getOpcode() == ISD::TargetGlobalAddress)) {
687      Base = N.getOperand(0);
688      if (Base.getOpcode() == ISD::TargetConstantPool)
689        return false;  // We want to select t2LDRpci instead.
690    } else
691      Base = N;
692    OffImm  = CurDAG->getTargetConstant(0, MVT::i32);
693    return true;
694  }
695
696  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
697    if (SelectT2AddrModeImm8(Op, N, Base, OffImm))
698      // Let t2LDRi8 handle (R - imm8).
699      return false;
700
701    int RHSC = (int)RHS->getZExtValue();
702    if (N.getOpcode() == ISD::SUB)
703      RHSC = -RHSC;
704
705    if (RHSC >= 0 && RHSC < 0x1000) { // 12 bits (unsigned)
706      Base   = N.getOperand(0);
707      if (Base.getOpcode() == ISD::FrameIndex) {
708        int FI = cast<FrameIndexSDNode>(Base)->getIndex();
709        Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
710      }
711      OffImm = CurDAG->getTargetConstant(RHSC, MVT::i32);
712      return true;
713    }
714  }
715
716  // Base only.
717  Base = N;
718  OffImm  = CurDAG->getTargetConstant(0, MVT::i32);
719  return true;
720}
721
722bool ARMDAGToDAGISel::SelectT2AddrModeImm8(SDNode *Op, SDValue N,
723                                           SDValue &Base, SDValue &OffImm) {
724  // Match simple R - imm8 operands.
725  if (N.getOpcode() == ISD::ADD || N.getOpcode() == ISD::SUB) {
726    if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
727      int RHSC = (int)RHS->getSExtValue();
728      if (N.getOpcode() == ISD::SUB)
729        RHSC = -RHSC;
730
731      if ((RHSC >= -255) && (RHSC < 0)) { // 8 bits (always negative)
732        Base = N.getOperand(0);
733        if (Base.getOpcode() == ISD::FrameIndex) {
734          int FI = cast<FrameIndexSDNode>(Base)->getIndex();
735          Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
736        }
737        OffImm = CurDAG->getTargetConstant(RHSC, MVT::i32);
738        return true;
739      }
740    }
741  }
742
743  return false;
744}
745
746bool ARMDAGToDAGISel::SelectT2AddrModeImm8Offset(SDNode *Op, SDValue N,
747                                                 SDValue &OffImm){
748  unsigned Opcode = Op->getOpcode();
749  ISD::MemIndexedMode AM = (Opcode == ISD::LOAD)
750    ? cast<LoadSDNode>(Op)->getAddressingMode()
751    : cast<StoreSDNode>(Op)->getAddressingMode();
752  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N)) {
753    int RHSC = (int)RHS->getZExtValue();
754    if (RHSC >= 0 && RHSC < 0x100) { // 8 bits.
755      OffImm = ((AM == ISD::PRE_INC) || (AM == ISD::POST_INC))
756        ? CurDAG->getTargetConstant(RHSC, MVT::i32)
757        : CurDAG->getTargetConstant(-RHSC, MVT::i32);
758      return true;
759    }
760  }
761
762  return false;
763}
764
765bool ARMDAGToDAGISel::SelectT2AddrModeImm8s4(SDNode *Op, SDValue N,
766                                             SDValue &Base, SDValue &OffImm) {
767  if (N.getOpcode() == ISD::ADD) {
768    if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
769      int RHSC = (int)RHS->getZExtValue();
770      if (((RHSC & 0x3) == 0) &&
771          ((RHSC >= 0 && RHSC < 0x400) || (RHSC < 0 && RHSC > -0x400))) { // 8 bits.
772        Base   = N.getOperand(0);
773        OffImm = CurDAG->getTargetConstant(RHSC, MVT::i32);
774        return true;
775      }
776    }
777  } else if (N.getOpcode() == ISD::SUB) {
778    if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
779      int RHSC = (int)RHS->getZExtValue();
780      if (((RHSC & 0x3) == 0) && (RHSC >= 0 && RHSC < 0x400)) { // 8 bits.
781        Base   = N.getOperand(0);
782        OffImm = CurDAG->getTargetConstant(-RHSC, MVT::i32);
783        return true;
784      }
785    }
786  }
787
788  return false;
789}
790
791bool ARMDAGToDAGISel::SelectT2AddrModeSoReg(SDNode *Op, SDValue N,
792                                            SDValue &Base,
793                                            SDValue &OffReg, SDValue &ShImm) {
794  // (R - imm8) should be handled by t2LDRi8. The rest are handled by t2LDRi12.
795  if (N.getOpcode() != ISD::ADD)
796    return false;
797
798  // Leave (R + imm12) for t2LDRi12, (R - imm8) for t2LDRi8.
799  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
800    int RHSC = (int)RHS->getZExtValue();
801    if (RHSC >= 0 && RHSC < 0x1000) // 12 bits (unsigned)
802      return false;
803    else if (RHSC < 0 && RHSC >= -255) // 8 bits
804      return false;
805  }
806
807  // Look for (R + R) or (R + (R << [1,2,3])).
808  unsigned ShAmt = 0;
809  Base   = N.getOperand(0);
810  OffReg = N.getOperand(1);
811
812  // Swap if it is ((R << c) + R).
813  ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(OffReg);
814  if (ShOpcVal != ARM_AM::lsl) {
815    ShOpcVal = ARM_AM::getShiftOpcForNode(Base);
816    if (ShOpcVal == ARM_AM::lsl)
817      std::swap(Base, OffReg);
818  }
819
820  if (ShOpcVal == ARM_AM::lsl) {
821    // Check to see if the RHS of the shift is a constant, if not, we can't fold
822    // it.
823    if (ConstantSDNode *Sh = dyn_cast<ConstantSDNode>(OffReg.getOperand(1))) {
824      ShAmt = Sh->getZExtValue();
825      if (ShAmt >= 4) {
826        ShAmt = 0;
827        ShOpcVal = ARM_AM::no_shift;
828      } else
829        OffReg = OffReg.getOperand(0);
830    } else {
831      ShOpcVal = ARM_AM::no_shift;
832    }
833  }
834
835  ShImm = CurDAG->getTargetConstant(ShAmt, MVT::i32);
836
837  return true;
838}
839
840//===--------------------------------------------------------------------===//
841
842/// getAL - Returns a ARMCC::AL immediate node.
843static inline SDValue getAL(SelectionDAG *CurDAG) {
844  return CurDAG->getTargetConstant((uint64_t)ARMCC::AL, MVT::i32);
845}
846
847SDNode *ARMDAGToDAGISel::SelectARMIndexedLoad(SDNode *N) {
848  LoadSDNode *LD = cast<LoadSDNode>(N);
849  ISD::MemIndexedMode AM = LD->getAddressingMode();
850  if (AM == ISD::UNINDEXED)
851    return NULL;
852
853  EVT LoadedVT = LD->getMemoryVT();
854  SDValue Offset, AMOpc;
855  bool isPre = (AM == ISD::PRE_INC) || (AM == ISD::PRE_DEC);
856  unsigned Opcode = 0;
857  bool Match = false;
858  if (LoadedVT == MVT::i32 &&
859      SelectAddrMode2Offset(N, LD->getOffset(), Offset, AMOpc)) {
860    Opcode = isPre ? ARM::LDR_PRE : ARM::LDR_POST;
861    Match = true;
862  } else if (LoadedVT == MVT::i16 &&
863             SelectAddrMode3Offset(N, LD->getOffset(), Offset, AMOpc)) {
864    Match = true;
865    Opcode = (LD->getExtensionType() == ISD::SEXTLOAD)
866      ? (isPre ? ARM::LDRSH_PRE : ARM::LDRSH_POST)
867      : (isPre ? ARM::LDRH_PRE : ARM::LDRH_POST);
868  } else if (LoadedVT == MVT::i8 || LoadedVT == MVT::i1) {
869    if (LD->getExtensionType() == ISD::SEXTLOAD) {
870      if (SelectAddrMode3Offset(N, LD->getOffset(), Offset, AMOpc)) {
871        Match = true;
872        Opcode = isPre ? ARM::LDRSB_PRE : ARM::LDRSB_POST;
873      }
874    } else {
875      if (SelectAddrMode2Offset(N, LD->getOffset(), Offset, AMOpc)) {
876        Match = true;
877        Opcode = isPre ? ARM::LDRB_PRE : ARM::LDRB_POST;
878      }
879    }
880  }
881
882  if (Match) {
883    SDValue Chain = LD->getChain();
884    SDValue Base = LD->getBasePtr();
885    SDValue Ops[]= { Base, Offset, AMOpc, getAL(CurDAG),
886                     CurDAG->getRegister(0, MVT::i32), Chain };
887    return CurDAG->getMachineNode(Opcode, N->getDebugLoc(), MVT::i32, MVT::i32,
888                                  MVT::Other, Ops, 6);
889  }
890
891  return NULL;
892}
893
894SDNode *ARMDAGToDAGISel::SelectT2IndexedLoad(SDNode *N) {
895  LoadSDNode *LD = cast<LoadSDNode>(N);
896  ISD::MemIndexedMode AM = LD->getAddressingMode();
897  if (AM == ISD::UNINDEXED)
898    return NULL;
899
900  EVT LoadedVT = LD->getMemoryVT();
901  bool isSExtLd = LD->getExtensionType() == ISD::SEXTLOAD;
902  SDValue Offset;
903  bool isPre = (AM == ISD::PRE_INC) || (AM == ISD::PRE_DEC);
904  unsigned Opcode = 0;
905  bool Match = false;
906  if (SelectT2AddrModeImm8Offset(N, LD->getOffset(), Offset)) {
907    switch (LoadedVT.getSimpleVT().SimpleTy) {
908    case MVT::i32:
909      Opcode = isPre ? ARM::t2LDR_PRE : ARM::t2LDR_POST;
910      break;
911    case MVT::i16:
912      if (isSExtLd)
913        Opcode = isPre ? ARM::t2LDRSH_PRE : ARM::t2LDRSH_POST;
914      else
915        Opcode = isPre ? ARM::t2LDRH_PRE : ARM::t2LDRH_POST;
916      break;
917    case MVT::i8:
918    case MVT::i1:
919      if (isSExtLd)
920        Opcode = isPre ? ARM::t2LDRSB_PRE : ARM::t2LDRSB_POST;
921      else
922        Opcode = isPre ? ARM::t2LDRB_PRE : ARM::t2LDRB_POST;
923      break;
924    default:
925      return NULL;
926    }
927    Match = true;
928  }
929
930  if (Match) {
931    SDValue Chain = LD->getChain();
932    SDValue Base = LD->getBasePtr();
933    SDValue Ops[]= { Base, Offset, getAL(CurDAG),
934                     CurDAG->getRegister(0, MVT::i32), Chain };
935    return CurDAG->getMachineNode(Opcode, N->getDebugLoc(), MVT::i32, MVT::i32,
936                                  MVT::Other, Ops, 5);
937  }
938
939  return NULL;
940}
941
942SDNode *ARMDAGToDAGISel::SelectDYN_ALLOC(SDNode *N) {
943  DebugLoc dl = N->getDebugLoc();
944  EVT VT = N->getValueType(0);
945  SDValue Chain = N->getOperand(0);
946  SDValue Size = N->getOperand(1);
947  SDValue Align = N->getOperand(2);
948  SDValue SP = CurDAG->getRegister(ARM::SP, MVT::i32);
949  int32_t AlignVal = cast<ConstantSDNode>(Align)->getSExtValue();
950  if (AlignVal < 0)
951    // We need to align the stack. Use Thumb1 tAND which is the only thumb
952    // instruction that can read and write SP. This matches to a pseudo
953    // instruction that has a chain to ensure the result is written back to
954    // the stack pointer.
955    SP = SDValue(CurDAG->getMachineNode(ARM::tANDsp, dl, VT, SP, Align), 0);
956
957  bool isC = isa<ConstantSDNode>(Size);
958  uint32_t C = isC ? cast<ConstantSDNode>(Size)->getZExtValue() : ~0UL;
959  // Handle the most common case for both Thumb1 and Thumb2:
960  // tSUBspi - immediate is between 0 ... 508 inclusive.
961  if (C <= 508 && ((C & 3) == 0))
962    // FIXME: tSUBspi encode scale 4 implicitly.
963    return CurDAG->SelectNodeTo(N, ARM::tSUBspi_, VT, MVT::Other, SP,
964                                CurDAG->getTargetConstant(C/4, MVT::i32),
965                                Chain);
966
967  if (Subtarget->isThumb1Only()) {
968    // Use tADDspr since Thumb1 does not have a sub r, sp, r. ARMISelLowering
969    // should have negated the size operand already. FIXME: We can't insert
970    // new target independent node at this stage so we are forced to negate
971    // it earlier. Is there a better solution?
972    return CurDAG->SelectNodeTo(N, ARM::tADDspr_, VT, MVT::Other, SP, Size,
973                                Chain);
974  } else if (Subtarget->isThumb2()) {
975    if (isC && Predicate_t2_so_imm(Size.getNode())) {
976      // t2SUBrSPi
977      SDValue Ops[] = { SP, CurDAG->getTargetConstant(C, MVT::i32), Chain };
978      return CurDAG->SelectNodeTo(N, ARM::t2SUBrSPi_, VT, MVT::Other, Ops, 3);
979    } else if (isC && Predicate_imm0_4095(Size.getNode())) {
980      // t2SUBrSPi12
981      SDValue Ops[] = { SP, CurDAG->getTargetConstant(C, MVT::i32), Chain };
982      return CurDAG->SelectNodeTo(N, ARM::t2SUBrSPi12_, VT, MVT::Other, Ops, 3);
983    } else {
984      // t2SUBrSPs
985      SDValue Ops[] = { SP, Size,
986                        getI32Imm(ARM_AM::getSORegOpc(ARM_AM::lsl,0)), Chain };
987      return CurDAG->SelectNodeTo(N, ARM::t2SUBrSPs_, VT, MVT::Other, Ops, 4);
988    }
989  }
990
991  // FIXME: Add ADD / SUB sp instructions for ARM.
992  return 0;
993}
994
995/// PairDRegs - Insert a pair of double registers into an implicit def to
996/// form a quad register.
997SDNode *ARMDAGToDAGISel::PairDRegs(EVT VT, SDValue V0, SDValue V1) {
998  DebugLoc dl = V0.getNode()->getDebugLoc();
999  SDValue Undef =
1000    SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, dl, VT), 0);
1001  SDValue SubReg0 = CurDAG->getTargetConstant(ARM::DSUBREG_0, MVT::i32);
1002  SDValue SubReg1 = CurDAG->getTargetConstant(ARM::DSUBREG_1, MVT::i32);
1003  SDNode *Pair = CurDAG->getMachineNode(TargetOpcode::INSERT_SUBREG, dl,
1004                                        VT, Undef, V0, SubReg0);
1005  return CurDAG->getMachineNode(TargetOpcode::INSERT_SUBREG, dl,
1006                                VT, SDValue(Pair, 0), V1, SubReg1);
1007}
1008
1009/// GetNEONSubregVT - Given a type for a 128-bit NEON vector, return the type
1010/// for a 64-bit subregister of the vector.
1011static EVT GetNEONSubregVT(EVT VT) {
1012  switch (VT.getSimpleVT().SimpleTy) {
1013  default: llvm_unreachable("unhandled NEON type");
1014  case MVT::v16i8: return MVT::v8i8;
1015  case MVT::v8i16: return MVT::v4i16;
1016  case MVT::v4f32: return MVT::v2f32;
1017  case MVT::v4i32: return MVT::v2i32;
1018  case MVT::v2i64: return MVT::v1i64;
1019  }
1020}
1021
1022SDNode *ARMDAGToDAGISel::SelectVLD(SDNode *N, unsigned NumVecs,
1023                                   unsigned *DOpcodes, unsigned *QOpcodes0,
1024                                   unsigned *QOpcodes1) {
1025  assert(NumVecs >=2 && NumVecs <= 4 && "VLD NumVecs out-of-range");
1026  DebugLoc dl = N->getDebugLoc();
1027
1028  SDValue MemAddr, Align;
1029  if (!SelectAddrMode6(N, N->getOperand(2), MemAddr, Align))
1030    return NULL;
1031
1032  SDValue Chain = N->getOperand(0);
1033  EVT VT = N->getValueType(0);
1034  bool is64BitVector = VT.is64BitVector();
1035
1036  unsigned OpcodeIndex;
1037  switch (VT.getSimpleVT().SimpleTy) {
1038  default: llvm_unreachable("unhandled vld type");
1039    // Double-register operations:
1040  case MVT::v8i8:  OpcodeIndex = 0; break;
1041  case MVT::v4i16: OpcodeIndex = 1; break;
1042  case MVT::v2f32:
1043  case MVT::v2i32: OpcodeIndex = 2; break;
1044  case MVT::v1i64: OpcodeIndex = 3; break;
1045    // Quad-register operations:
1046  case MVT::v16i8: OpcodeIndex = 0; break;
1047  case MVT::v8i16: OpcodeIndex = 1; break;
1048  case MVT::v4f32:
1049  case MVT::v4i32: OpcodeIndex = 2; break;
1050  }
1051
1052  SDValue Pred = CurDAG->getTargetConstant(14, MVT::i32);
1053  SDValue PredReg = CurDAG->getRegister(0, MVT::i32);
1054  if (is64BitVector) {
1055    unsigned Opc = DOpcodes[OpcodeIndex];
1056    const SDValue Ops[] = { MemAddr, Align, Pred, PredReg, Chain };
1057    std::vector<EVT> ResTys(NumVecs, VT);
1058    ResTys.push_back(MVT::Other);
1059    return CurDAG->getMachineNode(Opc, dl, ResTys, Ops, 5);
1060  }
1061
1062  EVT RegVT = GetNEONSubregVT(VT);
1063  if (NumVecs == 2) {
1064    // Quad registers are directly supported for VLD2,
1065    // loading 2 pairs of D regs.
1066    unsigned Opc = QOpcodes0[OpcodeIndex];
1067    const SDValue Ops[] = { MemAddr, Align, Pred, PredReg, Chain };
1068    std::vector<EVT> ResTys(4, VT);
1069    ResTys.push_back(MVT::Other);
1070    SDNode *VLd = CurDAG->getMachineNode(Opc, dl, ResTys, Ops, 5);
1071    Chain = SDValue(VLd, 4);
1072
1073    // Combine the even and odd subregs to produce the result.
1074    for (unsigned Vec = 0; Vec < NumVecs; ++Vec) {
1075      SDNode *Q = PairDRegs(VT, SDValue(VLd, 2*Vec), SDValue(VLd, 2*Vec+1));
1076      ReplaceUses(SDValue(N, Vec), SDValue(Q, 0));
1077    }
1078  } else {
1079    // Otherwise, quad registers are loaded with two separate instructions,
1080    // where one loads the even registers and the other loads the odd registers.
1081
1082    std::vector<EVT> ResTys(NumVecs, RegVT);
1083    ResTys.push_back(MemAddr.getValueType());
1084    ResTys.push_back(MVT::Other);
1085
1086    // Load the even subregs.
1087    unsigned Opc = QOpcodes0[OpcodeIndex];
1088    const SDValue OpsA[] = { MemAddr, Align, Pred, PredReg, Chain };
1089    SDNode *VLdA = CurDAG->getMachineNode(Opc, dl, ResTys, OpsA, 5);
1090    Chain = SDValue(VLdA, NumVecs+1);
1091
1092    // Load the odd subregs.
1093    Opc = QOpcodes1[OpcodeIndex];
1094    const SDValue OpsB[] = { SDValue(VLdA, NumVecs),
1095                             Align, Pred, PredReg, Chain };
1096    SDNode *VLdB = CurDAG->getMachineNode(Opc, dl, ResTys, OpsB, 5);
1097    Chain = SDValue(VLdB, NumVecs+1);
1098
1099    // Combine the even and odd subregs to produce the result.
1100    for (unsigned Vec = 0; Vec < NumVecs; ++Vec) {
1101      SDNode *Q = PairDRegs(VT, SDValue(VLdA, Vec), SDValue(VLdB, Vec));
1102      ReplaceUses(SDValue(N, Vec), SDValue(Q, 0));
1103    }
1104  }
1105  ReplaceUses(SDValue(N, NumVecs), Chain);
1106  return NULL;
1107}
1108
1109SDNode *ARMDAGToDAGISel::SelectVST(SDNode *N, unsigned NumVecs,
1110                                   unsigned *DOpcodes, unsigned *QOpcodes0,
1111                                   unsigned *QOpcodes1) {
1112  assert(NumVecs >=2 && NumVecs <= 4 && "VST NumVecs out-of-range");
1113  DebugLoc dl = N->getDebugLoc();
1114
1115  SDValue MemAddr, Align;
1116  if (!SelectAddrMode6(N, N->getOperand(2), MemAddr, Align))
1117    return NULL;
1118
1119  SDValue Chain = N->getOperand(0);
1120  EVT VT = N->getOperand(3).getValueType();
1121  bool is64BitVector = VT.is64BitVector();
1122
1123  unsigned OpcodeIndex;
1124  switch (VT.getSimpleVT().SimpleTy) {
1125  default: llvm_unreachable("unhandled vst type");
1126    // Double-register operations:
1127  case MVT::v8i8:  OpcodeIndex = 0; break;
1128  case MVT::v4i16: OpcodeIndex = 1; break;
1129  case MVT::v2f32:
1130  case MVT::v2i32: OpcodeIndex = 2; break;
1131  case MVT::v1i64: OpcodeIndex = 3; break;
1132    // Quad-register operations:
1133  case MVT::v16i8: OpcodeIndex = 0; break;
1134  case MVT::v8i16: OpcodeIndex = 1; break;
1135  case MVT::v4f32:
1136  case MVT::v4i32: OpcodeIndex = 2; break;
1137  }
1138
1139  SDValue Pred = CurDAG->getTargetConstant(14, MVT::i32);
1140  SDValue PredReg = CurDAG->getRegister(0, MVT::i32);
1141
1142  SmallVector<SDValue, 9> Ops;
1143  Ops.push_back(MemAddr);
1144  Ops.push_back(Align);
1145
1146  if (is64BitVector) {
1147    unsigned Opc = DOpcodes[OpcodeIndex];
1148    for (unsigned Vec = 0; Vec < NumVecs; ++Vec)
1149      Ops.push_back(N->getOperand(Vec+3));
1150    Ops.push_back(Pred);
1151    Ops.push_back(PredReg);
1152    Ops.push_back(Chain);
1153    return CurDAG->getMachineNode(Opc, dl, MVT::Other, Ops.data(), NumVecs+5);
1154  }
1155
1156  EVT RegVT = GetNEONSubregVT(VT);
1157  if (NumVecs == 2) {
1158    // Quad registers are directly supported for VST2,
1159    // storing 2 pairs of D regs.
1160    unsigned Opc = QOpcodes0[OpcodeIndex];
1161    for (unsigned Vec = 0; Vec < NumVecs; ++Vec) {
1162      Ops.push_back(CurDAG->getTargetExtractSubreg(ARM::DSUBREG_0, dl, RegVT,
1163                                                   N->getOperand(Vec+3)));
1164      Ops.push_back(CurDAG->getTargetExtractSubreg(ARM::DSUBREG_1, dl, RegVT,
1165                                                   N->getOperand(Vec+3)));
1166    }
1167    Ops.push_back(Pred);
1168    Ops.push_back(PredReg);
1169    Ops.push_back(Chain);
1170    return CurDAG->getMachineNode(Opc, dl, MVT::Other, Ops.data(), 9);
1171  }
1172
1173  // Otherwise, quad registers are stored with two separate instructions,
1174  // where one stores the even registers and the other stores the odd registers.
1175
1176  // Store the even subregs.
1177  for (unsigned Vec = 0; Vec < NumVecs; ++Vec)
1178    Ops.push_back(CurDAG->getTargetExtractSubreg(ARM::DSUBREG_0, dl, RegVT,
1179                                                 N->getOperand(Vec+3)));
1180  Ops.push_back(Pred);
1181  Ops.push_back(PredReg);
1182  Ops.push_back(Chain);
1183  unsigned Opc = QOpcodes0[OpcodeIndex];
1184  SDNode *VStA = CurDAG->getMachineNode(Opc, dl, MemAddr.getValueType(),
1185                                        MVT::Other, Ops.data(), NumVecs+5);
1186  Chain = SDValue(VStA, 1);
1187
1188  // Store the odd subregs.
1189  Ops[0] = SDValue(VStA, 0); // MemAddr
1190  for (unsigned Vec = 0; Vec < NumVecs; ++Vec)
1191    Ops[Vec+2] = CurDAG->getTargetExtractSubreg(ARM::DSUBREG_1, dl, RegVT,
1192                                                N->getOperand(Vec+3));
1193  Ops[NumVecs+2] = Pred;
1194  Ops[NumVecs+3] = PredReg;
1195  Ops[NumVecs+4] = Chain;
1196  Opc = QOpcodes1[OpcodeIndex];
1197  SDNode *VStB = CurDAG->getMachineNode(Opc, dl, MemAddr.getValueType(),
1198                                        MVT::Other, Ops.data(), NumVecs+5);
1199  Chain = SDValue(VStB, 1);
1200  ReplaceUses(SDValue(N, 0), Chain);
1201  return NULL;
1202}
1203
1204SDNode *ARMDAGToDAGISel::SelectVLDSTLane(SDNode *N, bool IsLoad,
1205                                         unsigned NumVecs, unsigned *DOpcodes,
1206                                         unsigned *QOpcodes0,
1207                                         unsigned *QOpcodes1) {
1208  assert(NumVecs >=2 && NumVecs <= 4 && "VLDSTLane NumVecs out-of-range");
1209  DebugLoc dl = N->getDebugLoc();
1210
1211  SDValue MemAddr, Align;
1212  if (!SelectAddrMode6(N, N->getOperand(2), MemAddr, Align))
1213    return NULL;
1214
1215  SDValue Chain = N->getOperand(0);
1216  unsigned Lane =
1217    cast<ConstantSDNode>(N->getOperand(NumVecs+3))->getZExtValue();
1218  EVT VT = IsLoad ? N->getValueType(0) : N->getOperand(3).getValueType();
1219  bool is64BitVector = VT.is64BitVector();
1220
1221  // Quad registers are handled by load/store of subregs. Find the subreg info.
1222  unsigned NumElts = 0;
1223  int SubregIdx = 0;
1224  EVT RegVT = VT;
1225  if (!is64BitVector) {
1226    RegVT = GetNEONSubregVT(VT);
1227    NumElts = RegVT.getVectorNumElements();
1228    SubregIdx = (Lane < NumElts) ? ARM::DSUBREG_0 : ARM::DSUBREG_1;
1229  }
1230
1231  unsigned OpcodeIndex;
1232  switch (VT.getSimpleVT().SimpleTy) {
1233  default: llvm_unreachable("unhandled vld/vst lane type");
1234    // Double-register operations:
1235  case MVT::v8i8:  OpcodeIndex = 0; break;
1236  case MVT::v4i16: OpcodeIndex = 1; break;
1237  case MVT::v2f32:
1238  case MVT::v2i32: OpcodeIndex = 2; break;
1239    // Quad-register operations:
1240  case MVT::v8i16: OpcodeIndex = 0; break;
1241  case MVT::v4f32:
1242  case MVT::v4i32: OpcodeIndex = 1; break;
1243  }
1244
1245  SDValue Pred = CurDAG->getTargetConstant(14, MVT::i32);
1246  SDValue PredReg = CurDAG->getRegister(0, MVT::i32);
1247
1248  SmallVector<SDValue, 10> Ops;
1249  Ops.push_back(MemAddr);
1250  Ops.push_back(Align);
1251
1252  unsigned Opc = 0;
1253  if (is64BitVector) {
1254    Opc = DOpcodes[OpcodeIndex];
1255    for (unsigned Vec = 0; Vec < NumVecs; ++Vec)
1256      Ops.push_back(N->getOperand(Vec+3));
1257  } else {
1258    // Check if this is loading the even or odd subreg of a Q register.
1259    if (Lane < NumElts) {
1260      Opc = QOpcodes0[OpcodeIndex];
1261    } else {
1262      Lane -= NumElts;
1263      Opc = QOpcodes1[OpcodeIndex];
1264    }
1265    // Extract the subregs of the input vector.
1266    for (unsigned Vec = 0; Vec < NumVecs; ++Vec)
1267      Ops.push_back(CurDAG->getTargetExtractSubreg(SubregIdx, dl, RegVT,
1268                                                   N->getOperand(Vec+3)));
1269  }
1270  Ops.push_back(getI32Imm(Lane));
1271  Ops.push_back(Pred);
1272  Ops.push_back(PredReg);
1273  Ops.push_back(Chain);
1274
1275  if (!IsLoad)
1276    return CurDAG->getMachineNode(Opc, dl, MVT::Other, Ops.data(), NumVecs+6);
1277
1278  std::vector<EVT> ResTys(NumVecs, RegVT);
1279  ResTys.push_back(MVT::Other);
1280  SDNode *VLdLn =
1281    CurDAG->getMachineNode(Opc, dl, ResTys, Ops.data(), NumVecs+6);
1282  // For a 64-bit vector load to D registers, nothing more needs to be done.
1283  if (is64BitVector)
1284    return VLdLn;
1285
1286  // For 128-bit vectors, take the 64-bit results of the load and insert them
1287  // as subregs into the result.
1288  for (unsigned Vec = 0; Vec < NumVecs; ++Vec) {
1289    SDValue QuadVec = CurDAG->getTargetInsertSubreg(SubregIdx, dl, VT,
1290                                                    N->getOperand(Vec+3),
1291                                                    SDValue(VLdLn, Vec));
1292    ReplaceUses(SDValue(N, Vec), QuadVec);
1293  }
1294
1295  Chain = SDValue(VLdLn, NumVecs);
1296  ReplaceUses(SDValue(N, NumVecs), Chain);
1297  return NULL;
1298}
1299
1300SDNode *ARMDAGToDAGISel::SelectV6T2BitfieldExtractOp(SDNode *N,
1301                                                     unsigned Opc) {
1302  if (!Subtarget->hasV6T2Ops())
1303    return NULL;
1304
1305  unsigned Shl_imm = 0;
1306  if (isOpcWithIntImmediate(N->getOperand(0).getNode(), ISD::SHL, Shl_imm)) {
1307    assert(Shl_imm > 0 && Shl_imm < 32 && "bad amount in shift node!");
1308    unsigned Srl_imm = 0;
1309    if (isInt32Immediate(N->getOperand(1), Srl_imm)) {
1310      assert(Srl_imm > 0 && Srl_imm < 32 && "bad amount in shift node!");
1311      unsigned Width = 32 - Srl_imm;
1312      int LSB = Srl_imm - Shl_imm;
1313      if (LSB < 0)
1314        return NULL;
1315      SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
1316      SDValue Ops[] = { N->getOperand(0).getOperand(0),
1317                        CurDAG->getTargetConstant(LSB, MVT::i32),
1318                        CurDAG->getTargetConstant(Width, MVT::i32),
1319                        getAL(CurDAG), Reg0 };
1320      return CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops, 5);
1321    }
1322  }
1323  return NULL;
1324}
1325
1326SDNode *ARMDAGToDAGISel::
1327SelectT2CMOVShiftOp(SDNode *N, SDValue FalseVal, SDValue TrueVal,
1328                    ARMCC::CondCodes CCVal, SDValue CCR, SDValue InFlag) {
1329  SDValue CPTmp0;
1330  SDValue CPTmp1;
1331  if (SelectT2ShifterOperandReg(N, TrueVal, CPTmp0, CPTmp1)) {
1332    unsigned SOVal = cast<ConstantSDNode>(CPTmp1)->getZExtValue();
1333    unsigned SOShOp = ARM_AM::getSORegShOp(SOVal);
1334    unsigned Opc = 0;
1335    switch (SOShOp) {
1336    case ARM_AM::lsl: Opc = ARM::t2MOVCClsl; break;
1337    case ARM_AM::lsr: Opc = ARM::t2MOVCClsr; break;
1338    case ARM_AM::asr: Opc = ARM::t2MOVCCasr; break;
1339    case ARM_AM::ror: Opc = ARM::t2MOVCCror; break;
1340    default:
1341      llvm_unreachable("Unknown so_reg opcode!");
1342      break;
1343    }
1344    SDValue SOShImm =
1345      CurDAG->getTargetConstant(ARM_AM::getSORegOffset(SOVal), MVT::i32);
1346    SDValue CC = CurDAG->getTargetConstant(CCVal, MVT::i32);
1347    SDValue Ops[] = { FalseVal, CPTmp0, SOShImm, CC, CCR, InFlag };
1348    return CurDAG->SelectNodeTo(N, Opc, MVT::i32,Ops, 6);
1349  }
1350  return 0;
1351}
1352
1353SDNode *ARMDAGToDAGISel::
1354SelectARMCMOVShiftOp(SDNode *N, SDValue FalseVal, SDValue TrueVal,
1355                     ARMCC::CondCodes CCVal, SDValue CCR, SDValue InFlag) {
1356  SDValue CPTmp0;
1357  SDValue CPTmp1;
1358  SDValue CPTmp2;
1359  if (SelectShifterOperandReg(N, TrueVal, CPTmp0, CPTmp1, CPTmp2)) {
1360    SDValue CC = CurDAG->getTargetConstant(CCVal, MVT::i32);
1361    SDValue Ops[] = { FalseVal, CPTmp0, CPTmp1, CPTmp2, CC, CCR, InFlag };
1362    return CurDAG->SelectNodeTo(N, ARM::MOVCCs, MVT::i32, Ops, 7);
1363  }
1364  return 0;
1365}
1366
1367SDNode *ARMDAGToDAGISel::
1368SelectT2CMOVSoImmOp(SDNode *N, SDValue FalseVal, SDValue TrueVal,
1369                    ARMCC::CondCodes CCVal, SDValue CCR, SDValue InFlag) {
1370  ConstantSDNode *T = dyn_cast<ConstantSDNode>(TrueVal);
1371  if (!T)
1372    return 0;
1373
1374  if (Predicate_t2_so_imm(TrueVal.getNode())) {
1375    SDValue True = CurDAG->getTargetConstant(T->getZExtValue(), MVT::i32);
1376    SDValue CC = CurDAG->getTargetConstant(CCVal, MVT::i32);
1377    SDValue Ops[] = { FalseVal, True, CC, CCR, InFlag };
1378    return CurDAG->SelectNodeTo(N,
1379                                ARM::t2MOVCCi, MVT::i32, Ops, 5);
1380  }
1381  return 0;
1382}
1383
1384SDNode *ARMDAGToDAGISel::
1385SelectARMCMOVSoImmOp(SDNode *N, SDValue FalseVal, SDValue TrueVal,
1386                     ARMCC::CondCodes CCVal, SDValue CCR, SDValue InFlag) {
1387  ConstantSDNode *T = dyn_cast<ConstantSDNode>(TrueVal);
1388  if (!T)
1389    return 0;
1390
1391  if (Predicate_so_imm(TrueVal.getNode())) {
1392    SDValue True = CurDAG->getTargetConstant(T->getZExtValue(), MVT::i32);
1393    SDValue CC = CurDAG->getTargetConstant(CCVal, MVT::i32);
1394    SDValue Ops[] = { FalseVal, True, CC, CCR, InFlag };
1395    return CurDAG->SelectNodeTo(N,
1396                                ARM::MOVCCi, MVT::i32, Ops, 5);
1397  }
1398  return 0;
1399}
1400
1401SDNode *ARMDAGToDAGISel::SelectCMOVOp(SDNode *N) {
1402  EVT VT = N->getValueType(0);
1403  SDValue FalseVal = N->getOperand(0);
1404  SDValue TrueVal  = N->getOperand(1);
1405  SDValue CC = N->getOperand(2);
1406  SDValue CCR = N->getOperand(3);
1407  SDValue InFlag = N->getOperand(4);
1408  assert(CC.getOpcode() == ISD::Constant);
1409  assert(CCR.getOpcode() == ISD::Register);
1410  ARMCC::CondCodes CCVal =
1411    (ARMCC::CondCodes)cast<ConstantSDNode>(CC)->getZExtValue();
1412
1413  if (!Subtarget->isThumb1Only() && VT == MVT::i32) {
1414    // Pattern: (ARMcmov:i32 GPR:i32:$false, so_reg:i32:$true, (imm:i32):$cc)
1415    // Emits: (MOVCCs:i32 GPR:i32:$false, so_reg:i32:$true, (imm:i32):$cc)
1416    // Pattern complexity = 18  cost = 1  size = 0
1417    SDValue CPTmp0;
1418    SDValue CPTmp1;
1419    SDValue CPTmp2;
1420    if (Subtarget->isThumb()) {
1421      SDNode *Res = SelectT2CMOVShiftOp(N, FalseVal, TrueVal,
1422                                        CCVal, CCR, InFlag);
1423      if (!Res)
1424        Res = SelectT2CMOVShiftOp(N, TrueVal, FalseVal,
1425                               ARMCC::getOppositeCondition(CCVal), CCR, InFlag);
1426      if (Res)
1427        return Res;
1428    } else {
1429      SDNode *Res = SelectARMCMOVShiftOp(N, FalseVal, TrueVal,
1430                                         CCVal, CCR, InFlag);
1431      if (!Res)
1432        Res = SelectARMCMOVShiftOp(N, TrueVal, FalseVal,
1433                               ARMCC::getOppositeCondition(CCVal), CCR, InFlag);
1434      if (Res)
1435        return Res;
1436    }
1437
1438    // Pattern: (ARMcmov:i32 GPR:i32:$false,
1439    //             (imm:i32)<<P:Predicate_so_imm>>:$true,
1440    //             (imm:i32):$cc)
1441    // Emits: (MOVCCi:i32 GPR:i32:$false,
1442    //           (so_imm:i32 (imm:i32):$true), (imm:i32):$cc)
1443    // Pattern complexity = 10  cost = 1  size = 0
1444    if (Subtarget->isThumb()) {
1445      SDNode *Res = SelectT2CMOVSoImmOp(N, FalseVal, TrueVal,
1446                                        CCVal, CCR, InFlag);
1447      if (!Res)
1448        Res = SelectT2CMOVSoImmOp(N, TrueVal, FalseVal,
1449                               ARMCC::getOppositeCondition(CCVal), CCR, InFlag);
1450      if (Res)
1451        return Res;
1452    } else {
1453      SDNode *Res = SelectARMCMOVSoImmOp(N, FalseVal, TrueVal,
1454                                         CCVal, CCR, InFlag);
1455      if (!Res)
1456        Res = SelectARMCMOVSoImmOp(N, TrueVal, FalseVal,
1457                               ARMCC::getOppositeCondition(CCVal), CCR, InFlag);
1458      if (Res)
1459        return Res;
1460    }
1461  }
1462
1463  // Pattern: (ARMcmov:i32 GPR:i32:$false, GPR:i32:$true, (imm:i32):$cc)
1464  // Emits: (MOVCCr:i32 GPR:i32:$false, GPR:i32:$true, (imm:i32):$cc)
1465  // Pattern complexity = 6  cost = 1  size = 0
1466  //
1467  // Pattern: (ARMcmov:i32 GPR:i32:$false, GPR:i32:$true, (imm:i32):$cc)
1468  // Emits: (tMOVCCr:i32 GPR:i32:$false, GPR:i32:$true, (imm:i32):$cc)
1469  // Pattern complexity = 6  cost = 11  size = 0
1470  //
1471  // Also FCPYScc and FCPYDcc.
1472  SDValue Tmp2 = CurDAG->getTargetConstant(CCVal, MVT::i32);
1473  SDValue Ops[] = { FalseVal, TrueVal, Tmp2, CCR, InFlag };
1474  unsigned Opc = 0;
1475  switch (VT.getSimpleVT().SimpleTy) {
1476  default: assert(false && "Illegal conditional move type!");
1477    break;
1478  case MVT::i32:
1479    Opc = Subtarget->isThumb()
1480      ? (Subtarget->hasThumb2() ? ARM::t2MOVCCr : ARM::tMOVCCr_pseudo)
1481      : ARM::MOVCCr;
1482    break;
1483  case MVT::f32:
1484    Opc = ARM::VMOVScc;
1485    break;
1486  case MVT::f64:
1487    Opc = ARM::VMOVDcc;
1488    break;
1489  }
1490  return CurDAG->SelectNodeTo(N, Opc, VT, Ops, 5);
1491}
1492
1493SDNode *ARMDAGToDAGISel::Select(SDNode *N) {
1494  DebugLoc dl = N->getDebugLoc();
1495
1496  if (N->isMachineOpcode())
1497    return NULL;   // Already selected.
1498
1499  switch (N->getOpcode()) {
1500  default: break;
1501  case ISD::Constant: {
1502    unsigned Val = cast<ConstantSDNode>(N)->getZExtValue();
1503    bool UseCP = true;
1504    if (Subtarget->hasThumb2())
1505      // Thumb2-aware targets have the MOVT instruction, so all immediates can
1506      // be done with MOV + MOVT, at worst.
1507      UseCP = 0;
1508    else {
1509      if (Subtarget->isThumb()) {
1510        UseCP = (Val > 255 &&                          // MOV
1511                 ~Val > 255 &&                         // MOV + MVN
1512                 !ARM_AM::isThumbImmShiftedVal(Val));  // MOV + LSL
1513      } else
1514        UseCP = (ARM_AM::getSOImmVal(Val) == -1 &&     // MOV
1515                 ARM_AM::getSOImmVal(~Val) == -1 &&    // MVN
1516                 !ARM_AM::isSOImmTwoPartVal(Val));     // two instrs.
1517    }
1518
1519    if (UseCP) {
1520      SDValue CPIdx =
1521        CurDAG->getTargetConstantPool(ConstantInt::get(
1522                                  Type::getInt32Ty(*CurDAG->getContext()), Val),
1523                                      TLI.getPointerTy());
1524
1525      SDNode *ResNode;
1526      if (Subtarget->isThumb1Only()) {
1527        SDValue Pred = CurDAG->getTargetConstant(14, MVT::i32);
1528        SDValue PredReg = CurDAG->getRegister(0, MVT::i32);
1529        SDValue Ops[] = { CPIdx, Pred, PredReg, CurDAG->getEntryNode() };
1530        ResNode = CurDAG->getMachineNode(ARM::tLDRcp, dl, MVT::i32, MVT::Other,
1531                                         Ops, 4);
1532      } else {
1533        SDValue Ops[] = {
1534          CPIdx,
1535          CurDAG->getRegister(0, MVT::i32),
1536          CurDAG->getTargetConstant(0, MVT::i32),
1537          getAL(CurDAG),
1538          CurDAG->getRegister(0, MVT::i32),
1539          CurDAG->getEntryNode()
1540        };
1541        ResNode=CurDAG->getMachineNode(ARM::LDRcp, dl, MVT::i32, MVT::Other,
1542                                       Ops, 6);
1543      }
1544      ReplaceUses(SDValue(N, 0), SDValue(ResNode, 0));
1545      return NULL;
1546    }
1547
1548    // Other cases are autogenerated.
1549    break;
1550  }
1551  case ISD::FrameIndex: {
1552    // Selects to ADDri FI, 0 which in turn will become ADDri SP, imm.
1553    int FI = cast<FrameIndexSDNode>(N)->getIndex();
1554    SDValue TFI = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
1555    if (Subtarget->isThumb1Only()) {
1556      return CurDAG->SelectNodeTo(N, ARM::tADDrSPi, MVT::i32, TFI,
1557                                  CurDAG->getTargetConstant(0, MVT::i32));
1558    } else {
1559      unsigned Opc = ((Subtarget->isThumb() && Subtarget->hasThumb2()) ?
1560                      ARM::t2ADDri : ARM::ADDri);
1561      SDValue Ops[] = { TFI, CurDAG->getTargetConstant(0, MVT::i32),
1562                        getAL(CurDAG), CurDAG->getRegister(0, MVT::i32),
1563                        CurDAG->getRegister(0, MVT::i32) };
1564      return CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops, 5);
1565    }
1566  }
1567  case ARMISD::DYN_ALLOC:
1568    return SelectDYN_ALLOC(N);
1569  case ISD::SRL:
1570    if (SDNode *I = SelectV6T2BitfieldExtractOp(N,
1571                      Subtarget->isThumb() ? ARM::t2UBFX : ARM::UBFX))
1572      return I;
1573    break;
1574  case ISD::SRA:
1575    if (SDNode *I = SelectV6T2BitfieldExtractOp(N,
1576                      Subtarget->isThumb() ? ARM::t2SBFX : ARM::SBFX))
1577      return I;
1578    break;
1579  case ISD::MUL:
1580    if (Subtarget->isThumb1Only())
1581      break;
1582    if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(N->getOperand(1))) {
1583      unsigned RHSV = C->getZExtValue();
1584      if (!RHSV) break;
1585      if (isPowerOf2_32(RHSV-1)) {  // 2^n+1?
1586        unsigned ShImm = Log2_32(RHSV-1);
1587        if (ShImm >= 32)
1588          break;
1589        SDValue V = N->getOperand(0);
1590        ShImm = ARM_AM::getSORegOpc(ARM_AM::lsl, ShImm);
1591        SDValue ShImmOp = CurDAG->getTargetConstant(ShImm, MVT::i32);
1592        SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
1593        if (Subtarget->isThumb()) {
1594          SDValue Ops[] = { V, V, ShImmOp, getAL(CurDAG), Reg0, Reg0 };
1595          return CurDAG->SelectNodeTo(N, ARM::t2ADDrs, MVT::i32, Ops, 6);
1596        } else {
1597          SDValue Ops[] = { V, V, Reg0, ShImmOp, getAL(CurDAG), Reg0, Reg0 };
1598          return CurDAG->SelectNodeTo(N, ARM::ADDrs, MVT::i32, Ops, 7);
1599        }
1600      }
1601      if (isPowerOf2_32(RHSV+1)) {  // 2^n-1?
1602        unsigned ShImm = Log2_32(RHSV+1);
1603        if (ShImm >= 32)
1604          break;
1605        SDValue V = N->getOperand(0);
1606        ShImm = ARM_AM::getSORegOpc(ARM_AM::lsl, ShImm);
1607        SDValue ShImmOp = CurDAG->getTargetConstant(ShImm, MVT::i32);
1608        SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
1609        if (Subtarget->isThumb()) {
1610          SDValue Ops[] = { V, V, ShImmOp, getAL(CurDAG), Reg0 };
1611          return CurDAG->SelectNodeTo(N, ARM::t2RSBrs, MVT::i32, Ops, 5);
1612        } else {
1613          SDValue Ops[] = { V, V, Reg0, ShImmOp, getAL(CurDAG), Reg0, Reg0 };
1614          return CurDAG->SelectNodeTo(N, ARM::RSBrs, MVT::i32, Ops, 7);
1615        }
1616      }
1617    }
1618    break;
1619  case ISD::AND: {
1620    // (and (or x, c2), c1) and top 16-bits of c1 and c2 match, lower 16-bits
1621    // of c1 are 0xffff, and lower 16-bit of c2 are 0. That is, the top 16-bits
1622    // are entirely contributed by c2 and lower 16-bits are entirely contributed
1623    // by x. That's equal to (or (and x, 0xffff), (and c1, 0xffff0000)).
1624    // Select it to: "movt x, ((c1 & 0xffff) >> 16)
1625    EVT VT = N->getValueType(0);
1626    if (VT != MVT::i32)
1627      break;
1628    unsigned Opc = (Subtarget->isThumb() && Subtarget->hasThumb2())
1629      ? ARM::t2MOVTi16
1630      : (Subtarget->hasV6T2Ops() ? ARM::MOVTi16 : 0);
1631    if (!Opc)
1632      break;
1633    SDValue N0 = N->getOperand(0), N1 = N->getOperand(1);
1634    ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
1635    if (!N1C)
1636      break;
1637    if (N0.getOpcode() == ISD::OR && N0.getNode()->hasOneUse()) {
1638      SDValue N2 = N0.getOperand(1);
1639      ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(N2);
1640      if (!N2C)
1641        break;
1642      unsigned N1CVal = N1C->getZExtValue();
1643      unsigned N2CVal = N2C->getZExtValue();
1644      if ((N1CVal & 0xffff0000U) == (N2CVal & 0xffff0000U) &&
1645          (N1CVal & 0xffffU) == 0xffffU &&
1646          (N2CVal & 0xffffU) == 0x0U) {
1647        SDValue Imm16 = CurDAG->getTargetConstant((N2CVal & 0xFFFF0000U) >> 16,
1648                                                  MVT::i32);
1649        SDValue Ops[] = { N0.getOperand(0), Imm16,
1650                          getAL(CurDAG), CurDAG->getRegister(0, MVT::i32) };
1651        return CurDAG->getMachineNode(Opc, dl, VT, Ops, 4);
1652      }
1653    }
1654    break;
1655  }
1656  case ARMISD::VMOVRRD:
1657    return CurDAG->getMachineNode(ARM::VMOVRRD, dl, MVT::i32, MVT::i32,
1658                                  N->getOperand(0), getAL(CurDAG),
1659                                  CurDAG->getRegister(0, MVT::i32));
1660  case ISD::UMUL_LOHI: {
1661    if (Subtarget->isThumb1Only())
1662      break;
1663    if (Subtarget->isThumb()) {
1664      SDValue Ops[] = { N->getOperand(0), N->getOperand(1),
1665                        getAL(CurDAG), CurDAG->getRegister(0, MVT::i32),
1666                        CurDAG->getRegister(0, MVT::i32) };
1667      return CurDAG->getMachineNode(ARM::t2UMULL, dl, MVT::i32, MVT::i32, Ops,4);
1668    } else {
1669      SDValue Ops[] = { N->getOperand(0), N->getOperand(1),
1670                        getAL(CurDAG), CurDAG->getRegister(0, MVT::i32),
1671                        CurDAG->getRegister(0, MVT::i32) };
1672      return CurDAG->getMachineNode(ARM::UMULL, dl, MVT::i32, MVT::i32, Ops, 5);
1673    }
1674  }
1675  case ISD::SMUL_LOHI: {
1676    if (Subtarget->isThumb1Only())
1677      break;
1678    if (Subtarget->isThumb()) {
1679      SDValue Ops[] = { N->getOperand(0), N->getOperand(1),
1680                        getAL(CurDAG), CurDAG->getRegister(0, MVT::i32) };
1681      return CurDAG->getMachineNode(ARM::t2SMULL, dl, MVT::i32, MVT::i32, Ops,4);
1682    } else {
1683      SDValue Ops[] = { N->getOperand(0), N->getOperand(1),
1684                        getAL(CurDAG), CurDAG->getRegister(0, MVT::i32),
1685                        CurDAG->getRegister(0, MVT::i32) };
1686      return CurDAG->getMachineNode(ARM::SMULL, dl, MVT::i32, MVT::i32, Ops, 5);
1687    }
1688  }
1689  case ISD::LOAD: {
1690    SDNode *ResNode = 0;
1691    if (Subtarget->isThumb() && Subtarget->hasThumb2())
1692      ResNode = SelectT2IndexedLoad(N);
1693    else
1694      ResNode = SelectARMIndexedLoad(N);
1695    if (ResNode)
1696      return ResNode;
1697    // Other cases are autogenerated.
1698    break;
1699  }
1700  case ARMISD::BRCOND: {
1701    // Pattern: (ARMbrcond:void (bb:Other):$dst, (imm:i32):$cc)
1702    // Emits: (Bcc:void (bb:Other):$dst, (imm:i32):$cc)
1703    // Pattern complexity = 6  cost = 1  size = 0
1704
1705    // Pattern: (ARMbrcond:void (bb:Other):$dst, (imm:i32):$cc)
1706    // Emits: (tBcc:void (bb:Other):$dst, (imm:i32):$cc)
1707    // Pattern complexity = 6  cost = 1  size = 0
1708
1709    // Pattern: (ARMbrcond:void (bb:Other):$dst, (imm:i32):$cc)
1710    // Emits: (t2Bcc:void (bb:Other):$dst, (imm:i32):$cc)
1711    // Pattern complexity = 6  cost = 1  size = 0
1712
1713    unsigned Opc = Subtarget->isThumb() ?
1714      ((Subtarget->hasThumb2()) ? ARM::t2Bcc : ARM::tBcc) : ARM::Bcc;
1715    SDValue Chain = N->getOperand(0);
1716    SDValue N1 = N->getOperand(1);
1717    SDValue N2 = N->getOperand(2);
1718    SDValue N3 = N->getOperand(3);
1719    SDValue InFlag = N->getOperand(4);
1720    assert(N1.getOpcode() == ISD::BasicBlock);
1721    assert(N2.getOpcode() == ISD::Constant);
1722    assert(N3.getOpcode() == ISD::Register);
1723
1724    SDValue Tmp2 = CurDAG->getTargetConstant(((unsigned)
1725                               cast<ConstantSDNode>(N2)->getZExtValue()),
1726                               MVT::i32);
1727    SDValue Ops[] = { N1, Tmp2, N3, Chain, InFlag };
1728    SDNode *ResNode = CurDAG->getMachineNode(Opc, dl, MVT::Other,
1729                                             MVT::Flag, Ops, 5);
1730    Chain = SDValue(ResNode, 0);
1731    if (N->getNumValues() == 2) {
1732      InFlag = SDValue(ResNode, 1);
1733      ReplaceUses(SDValue(N, 1), InFlag);
1734    }
1735    ReplaceUses(SDValue(N, 0),
1736                SDValue(Chain.getNode(), Chain.getResNo()));
1737    return NULL;
1738  }
1739  case ARMISD::CMOV:
1740    return SelectCMOVOp(N);
1741  case ARMISD::CNEG: {
1742    EVT VT = N->getValueType(0);
1743    SDValue N0 = N->getOperand(0);
1744    SDValue N1 = N->getOperand(1);
1745    SDValue N2 = N->getOperand(2);
1746    SDValue N3 = N->getOperand(3);
1747    SDValue InFlag = N->getOperand(4);
1748    assert(N2.getOpcode() == ISD::Constant);
1749    assert(N3.getOpcode() == ISD::Register);
1750
1751    SDValue Tmp2 = CurDAG->getTargetConstant(((unsigned)
1752                               cast<ConstantSDNode>(N2)->getZExtValue()),
1753                               MVT::i32);
1754    SDValue Ops[] = { N0, N1, Tmp2, N3, InFlag };
1755    unsigned Opc = 0;
1756    switch (VT.getSimpleVT().SimpleTy) {
1757    default: assert(false && "Illegal conditional move type!");
1758      break;
1759    case MVT::f32:
1760      Opc = ARM::VNEGScc;
1761      break;
1762    case MVT::f64:
1763      Opc = ARM::VNEGDcc;
1764      break;
1765    }
1766    return CurDAG->SelectNodeTo(N, Opc, VT, Ops, 5);
1767  }
1768
1769  case ARMISD::VZIP: {
1770    unsigned Opc = 0;
1771    EVT VT = N->getValueType(0);
1772    switch (VT.getSimpleVT().SimpleTy) {
1773    default: return NULL;
1774    case MVT::v8i8:  Opc = ARM::VZIPd8; break;
1775    case MVT::v4i16: Opc = ARM::VZIPd16; break;
1776    case MVT::v2f32:
1777    case MVT::v2i32: Opc = ARM::VZIPd32; break;
1778    case MVT::v16i8: Opc = ARM::VZIPq8; break;
1779    case MVT::v8i16: Opc = ARM::VZIPq16; break;
1780    case MVT::v4f32:
1781    case MVT::v4i32: Opc = ARM::VZIPq32; break;
1782    }
1783    SDValue Pred = CurDAG->getTargetConstant(14, MVT::i32);
1784    SDValue PredReg = CurDAG->getRegister(0, MVT::i32);
1785    SDValue Ops[] = { N->getOperand(0), N->getOperand(1), Pred, PredReg };
1786    return CurDAG->getMachineNode(Opc, dl, VT, VT, Ops, 4);
1787  }
1788  case ARMISD::VUZP: {
1789    unsigned Opc = 0;
1790    EVT VT = N->getValueType(0);
1791    switch (VT.getSimpleVT().SimpleTy) {
1792    default: return NULL;
1793    case MVT::v8i8:  Opc = ARM::VUZPd8; break;
1794    case MVT::v4i16: Opc = ARM::VUZPd16; break;
1795    case MVT::v2f32:
1796    case MVT::v2i32: Opc = ARM::VUZPd32; break;
1797    case MVT::v16i8: Opc = ARM::VUZPq8; break;
1798    case MVT::v8i16: Opc = ARM::VUZPq16; break;
1799    case MVT::v4f32:
1800    case MVT::v4i32: Opc = ARM::VUZPq32; break;
1801    }
1802    SDValue Pred = CurDAG->getTargetConstant(14, MVT::i32);
1803    SDValue PredReg = CurDAG->getRegister(0, MVT::i32);
1804    SDValue Ops[] = { N->getOperand(0), N->getOperand(1), Pred, PredReg };
1805    return CurDAG->getMachineNode(Opc, dl, VT, VT, Ops, 4);
1806  }
1807  case ARMISD::VTRN: {
1808    unsigned Opc = 0;
1809    EVT VT = N->getValueType(0);
1810    switch (VT.getSimpleVT().SimpleTy) {
1811    default: return NULL;
1812    case MVT::v8i8:  Opc = ARM::VTRNd8; break;
1813    case MVT::v4i16: Opc = ARM::VTRNd16; break;
1814    case MVT::v2f32:
1815    case MVT::v2i32: Opc = ARM::VTRNd32; break;
1816    case MVT::v16i8: Opc = ARM::VTRNq8; break;
1817    case MVT::v8i16: Opc = ARM::VTRNq16; break;
1818    case MVT::v4f32:
1819    case MVT::v4i32: Opc = ARM::VTRNq32; break;
1820    }
1821    SDValue Pred = CurDAG->getTargetConstant(14, MVT::i32);
1822    SDValue PredReg = CurDAG->getRegister(0, MVT::i32);
1823    SDValue Ops[] = { N->getOperand(0), N->getOperand(1), Pred, PredReg };
1824    return CurDAG->getMachineNode(Opc, dl, VT, VT, Ops, 4);
1825  }
1826
1827  case ISD::INTRINSIC_VOID:
1828  case ISD::INTRINSIC_W_CHAIN: {
1829    unsigned IntNo = cast<ConstantSDNode>(N->getOperand(1))->getZExtValue();
1830    switch (IntNo) {
1831    default:
1832      break;
1833
1834    case Intrinsic::arm_neon_vld2: {
1835      unsigned DOpcodes[] = { ARM::VLD2d8, ARM::VLD2d16,
1836                              ARM::VLD2d32, ARM::VLD2d64 };
1837      unsigned QOpcodes[] = { ARM::VLD2q8, ARM::VLD2q16, ARM::VLD2q32 };
1838      return SelectVLD(N, 2, DOpcodes, QOpcodes, 0);
1839    }
1840
1841    case Intrinsic::arm_neon_vld3: {
1842      unsigned DOpcodes[] = { ARM::VLD3d8, ARM::VLD3d16,
1843                              ARM::VLD3d32, ARM::VLD3d64 };
1844      unsigned QOpcodes0[] = { ARM::VLD3q8a, ARM::VLD3q16a, ARM::VLD3q32a };
1845      unsigned QOpcodes1[] = { ARM::VLD3q8b, ARM::VLD3q16b, ARM::VLD3q32b };
1846      return SelectVLD(N, 3, DOpcodes, QOpcodes0, QOpcodes1);
1847    }
1848
1849    case Intrinsic::arm_neon_vld4: {
1850      unsigned DOpcodes[] = { ARM::VLD4d8, ARM::VLD4d16,
1851                              ARM::VLD4d32, ARM::VLD4d64 };
1852      unsigned QOpcodes0[] = { ARM::VLD4q8a, ARM::VLD4q16a, ARM::VLD4q32a };
1853      unsigned QOpcodes1[] = { ARM::VLD4q8b, ARM::VLD4q16b, ARM::VLD4q32b };
1854      return SelectVLD(N, 4, DOpcodes, QOpcodes0, QOpcodes1);
1855    }
1856
1857    case Intrinsic::arm_neon_vld2lane: {
1858      unsigned DOpcodes[] = { ARM::VLD2LNd8, ARM::VLD2LNd16, ARM::VLD2LNd32 };
1859      unsigned QOpcodes0[] = { ARM::VLD2LNq16a, ARM::VLD2LNq32a };
1860      unsigned QOpcodes1[] = { ARM::VLD2LNq16b, ARM::VLD2LNq32b };
1861      return SelectVLDSTLane(N, true, 2, DOpcodes, QOpcodes0, QOpcodes1);
1862    }
1863
1864    case Intrinsic::arm_neon_vld3lane: {
1865      unsigned DOpcodes[] = { ARM::VLD3LNd8, ARM::VLD3LNd16, ARM::VLD3LNd32 };
1866      unsigned QOpcodes0[] = { ARM::VLD3LNq16a, ARM::VLD3LNq32a };
1867      unsigned QOpcodes1[] = { ARM::VLD3LNq16b, ARM::VLD3LNq32b };
1868      return SelectVLDSTLane(N, true, 3, DOpcodes, QOpcodes0, QOpcodes1);
1869    }
1870
1871    case Intrinsic::arm_neon_vld4lane: {
1872      unsigned DOpcodes[] = { ARM::VLD4LNd8, ARM::VLD4LNd16, ARM::VLD4LNd32 };
1873      unsigned QOpcodes0[] = { ARM::VLD4LNq16a, ARM::VLD4LNq32a };
1874      unsigned QOpcodes1[] = { ARM::VLD4LNq16b, ARM::VLD4LNq32b };
1875      return SelectVLDSTLane(N, true, 4, DOpcodes, QOpcodes0, QOpcodes1);
1876    }
1877
1878    case Intrinsic::arm_neon_vst2: {
1879      unsigned DOpcodes[] = { ARM::VST2d8, ARM::VST2d16,
1880                              ARM::VST2d32, ARM::VST2d64 };
1881      unsigned QOpcodes[] = { ARM::VST2q8, ARM::VST2q16, ARM::VST2q32 };
1882      return SelectVST(N, 2, DOpcodes, QOpcodes, 0);
1883    }
1884
1885    case Intrinsic::arm_neon_vst3: {
1886      unsigned DOpcodes[] = { ARM::VST3d8, ARM::VST3d16,
1887                              ARM::VST3d32, ARM::VST3d64 };
1888      unsigned QOpcodes0[] = { ARM::VST3q8a, ARM::VST3q16a, ARM::VST3q32a };
1889      unsigned QOpcodes1[] = { ARM::VST3q8b, ARM::VST3q16b, ARM::VST3q32b };
1890      return SelectVST(N, 3, DOpcodes, QOpcodes0, QOpcodes1);
1891    }
1892
1893    case Intrinsic::arm_neon_vst4: {
1894      unsigned DOpcodes[] = { ARM::VST4d8, ARM::VST4d16,
1895                              ARM::VST4d32, ARM::VST4d64 };
1896      unsigned QOpcodes0[] = { ARM::VST4q8a, ARM::VST4q16a, ARM::VST4q32a };
1897      unsigned QOpcodes1[] = { ARM::VST4q8b, ARM::VST4q16b, ARM::VST4q32b };
1898      return SelectVST(N, 4, DOpcodes, QOpcodes0, QOpcodes1);
1899    }
1900
1901    case Intrinsic::arm_neon_vst2lane: {
1902      unsigned DOpcodes[] = { ARM::VST2LNd8, ARM::VST2LNd16, ARM::VST2LNd32 };
1903      unsigned QOpcodes0[] = { ARM::VST2LNq16a, ARM::VST2LNq32a };
1904      unsigned QOpcodes1[] = { ARM::VST2LNq16b, ARM::VST2LNq32b };
1905      return SelectVLDSTLane(N, false, 2, DOpcodes, QOpcodes0, QOpcodes1);
1906    }
1907
1908    case Intrinsic::arm_neon_vst3lane: {
1909      unsigned DOpcodes[] = { ARM::VST3LNd8, ARM::VST3LNd16, ARM::VST3LNd32 };
1910      unsigned QOpcodes0[] = { ARM::VST3LNq16a, ARM::VST3LNq32a };
1911      unsigned QOpcodes1[] = { ARM::VST3LNq16b, ARM::VST3LNq32b };
1912      return SelectVLDSTLane(N, false, 3, DOpcodes, QOpcodes0, QOpcodes1);
1913    }
1914
1915    case Intrinsic::arm_neon_vst4lane: {
1916      unsigned DOpcodes[] = { ARM::VST4LNd8, ARM::VST4LNd16, ARM::VST4LNd32 };
1917      unsigned QOpcodes0[] = { ARM::VST4LNq16a, ARM::VST4LNq32a };
1918      unsigned QOpcodes1[] = { ARM::VST4LNq16b, ARM::VST4LNq32b };
1919      return SelectVLDSTLane(N, false, 4, DOpcodes, QOpcodes0, QOpcodes1);
1920    }
1921    }
1922  }
1923  }
1924
1925  return SelectCode(N);
1926}
1927
1928bool ARMDAGToDAGISel::
1929SelectInlineAsmMemoryOperand(const SDValue &Op, char ConstraintCode,
1930                             std::vector<SDValue> &OutOps) {
1931  assert(ConstraintCode == 'm' && "unexpected asm memory constraint");
1932  // Require the address to be in a register.  That is safe for all ARM
1933  // variants and it is hard to do anything much smarter without knowing
1934  // how the operand is used.
1935  OutOps.push_back(Op);
1936  return false;
1937}
1938
1939/// createARMISelDag - This pass converts a legalized DAG into a
1940/// ARM-specific DAG, ready for instruction scheduling.
1941///
1942FunctionPass *llvm::createARMISelDag(ARMBaseTargetMachine &TM,
1943                                     CodeGenOpt::Level OptLevel) {
1944  return new ARMDAGToDAGISel(TM, OptLevel);
1945}
1946