ARMISelDAGToDAG.cpp revision df9a4f0591e3f3e7ae3b3c119a0c8cf678594f6b 
1//===-- ARMISelDAGToDAG.cpp - A dag to dag inst selector for ARM ----------===//
2//
3//                     The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10// This file defines an instruction selector for the ARM target.
11//
12//===----------------------------------------------------------------------===//
13
14#include "ARM.h"
15#include "ARMAddressingModes.h"
16#include "ARMISelLowering.h"
17#include "ARMTargetMachine.h"
18#include "llvm/CallingConv.h"
19#include "llvm/Constants.h"
20#include "llvm/DerivedTypes.h"
21#include "llvm/Function.h"
22#include "llvm/Intrinsics.h"
23#include "llvm/LLVMContext.h"
24#include "llvm/CodeGen/MachineFrameInfo.h"
25#include "llvm/CodeGen/MachineFunction.h"
26#include "llvm/CodeGen/MachineInstrBuilder.h"
27#include "llvm/CodeGen/SelectionDAG.h"
28#include "llvm/CodeGen/SelectionDAGISel.h"
29#include "llvm/Target/TargetLowering.h"
30#include "llvm/Target/TargetOptions.h"
31#include "llvm/Support/Compiler.h"
32#include "llvm/Support/Debug.h"
33#include "llvm/Support/ErrorHandling.h"
34#include "llvm/Support/raw_ostream.h"
35
36using namespace llvm;
37
38//===--------------------------------------------------------------------===//
39/// ARMDAGToDAGISel - ARM specific code to select ARM machine
40/// instructions for SelectionDAG operations.
41///
42namespace {
43class ARMDAGToDAGISel : public SelectionDAGISel {
44  ARMBaseTargetMachine &TM;
45
46  /// Subtarget - Keep a pointer to the ARMSubtarget around so that we can
47  /// make the right decision when generating code for different targets.
48  const ARMSubtarget *Subtarget;
49
50public:
51  explicit ARMDAGToDAGISel(ARMBaseTargetMachine &tm,
52                           CodeGenOpt::Level OptLevel)
53    : SelectionDAGISel(tm, OptLevel), TM(tm),
54    Subtarget(&TM.getSubtarget<ARMSubtarget>()) {
55  }
56
57  virtual const char *getPassName() const {
58    return "ARM Instruction Selection";
59  }
60
61  /// getI32Imm - Return a target constant of type i32 with the specified
62  /// value.
63  inline SDValue getI32Imm(unsigned Imm) {
64    return CurDAG->getTargetConstant(Imm, MVT::i32);
65  }
66
67  SDNode *Select(SDNode *N);
68
69  bool SelectShifterOperandReg(SDNode *Op, SDValue N, SDValue &A,
70                               SDValue &B, SDValue &C);
71  bool SelectAddrMode2(SDNode *Op, SDValue N, SDValue &Base,
72                       SDValue &Offset, SDValue &Opc);
73  bool SelectAddrMode2Offset(SDNode *Op, SDValue N,
74                             SDValue &Offset, SDValue &Opc);
75  bool SelectAddrMode3(SDNode *Op, SDValue N, SDValue &Base,
76                       SDValue &Offset, SDValue &Opc);
77  bool SelectAddrMode3Offset(SDNode *Op, SDValue N,
78                             SDValue &Offset, SDValue &Opc);
79  bool SelectAddrMode4(SDNode *Op, SDValue N, SDValue &Addr,
80                       SDValue &Mode);
81  bool SelectAddrMode5(SDNode *Op, SDValue N, SDValue &Base,
82                       SDValue &Offset);
83  bool SelectAddrMode6(SDNode *Op, SDValue N, SDValue &Addr, SDValue &Align);
84
85  bool SelectAddrModePC(SDNode *Op, SDValue N, SDValue &Offset,
86                        SDValue &Label);
87
88  bool SelectThumbAddrModeRR(SDNode *Op, SDValue N, SDValue &Base,
89                             SDValue &Offset);
90  bool SelectThumbAddrModeRI5(SDNode *Op, SDValue N, unsigned Scale,
91                              SDValue &Base, SDValue &OffImm,
92                              SDValue &Offset);
93  bool SelectThumbAddrModeS1(SDNode *Op, SDValue N, SDValue &Base,
94                             SDValue &OffImm, SDValue &Offset);
95  bool SelectThumbAddrModeS2(SDNode *Op, SDValue N, SDValue &Base,
96                             SDValue &OffImm, SDValue &Offset);
97  bool SelectThumbAddrModeS4(SDNode *Op, SDValue N, SDValue &Base,
98                             SDValue &OffImm, SDValue &Offset);
99  bool SelectThumbAddrModeSP(SDNode *Op, SDValue N, SDValue &Base,
100                             SDValue &OffImm);
101
102  bool SelectT2ShifterOperandReg(SDNode *Op, SDValue N,
103                                 SDValue &BaseReg, SDValue &Opc);
104  bool SelectT2AddrModeImm12(SDNode *Op, SDValue N, SDValue &Base,
105                             SDValue &OffImm);
106  bool SelectT2AddrModeImm8(SDNode *Op, SDValue N, SDValue &Base,
107                            SDValue &OffImm);
108  bool SelectT2AddrModeImm8Offset(SDNode *Op, SDValue N,
109                                 SDValue &OffImm);
110  bool SelectT2AddrModeImm8s4(SDNode *Op, SDValue N, SDValue &Base,
111                              SDValue &OffImm);
112  bool SelectT2AddrModeSoReg(SDNode *Op, SDValue N, SDValue &Base,
113                             SDValue &OffReg, SDValue &ShImm);
114
115  // Include the pieces autogenerated from the target description.
116#include "ARMGenDAGISel.inc"
117
118private:
119  /// SelectARMIndexedLoad - Indexed (pre/post inc/dec) load matching code for
120  /// ARM.
121  SDNode *SelectARMIndexedLoad(SDNode *N);
122  SDNode *SelectT2IndexedLoad(SDNode *N);
123
124  /// SelectDYN_ALLOC - Select dynamic alloc for Thumb.
125  SDNode *SelectDYN_ALLOC(SDNode *N);
126
127  /// SelectVLD - Select NEON load intrinsics.  NumVecs should be
128  /// 1, 2, 3 or 4.  The opcode arrays specify the instructions used for
129  /// loads of D registers and even subregs and odd subregs of Q registers.
130  /// For NumVecs <= 2, QOpcodes1 is not used.
131  SDNode *SelectVLD(SDNode *N, unsigned NumVecs, unsigned *DOpcodes,
132                    unsigned *QOpcodes0, unsigned *QOpcodes1);
133
134  /// SelectVST - Select NEON store intrinsics.  NumVecs should
135  /// be 1, 2, 3 or 4.  The opcode arrays specify the instructions used for
136  /// stores of D registers and even subregs and odd subregs of Q registers.
137  /// For NumVecs <= 2, QOpcodes1 is not used.
138  SDNode *SelectVST(SDNode *N, unsigned NumVecs, unsigned *DOpcodes,
139                    unsigned *QOpcodes0, unsigned *QOpcodes1);
140
141  /// SelectVLDSTLane - Select NEON load/store lane intrinsics.  NumVecs should
142  /// be 2, 3 or 4.  The opcode arrays specify the instructions used for
143  /// load/store of D registers and even subregs and odd subregs of Q registers.
144  SDNode *SelectVLDSTLane(SDNode *N, bool IsLoad, unsigned NumVecs,
145                          unsigned *DOpcodes, unsigned *QOpcodes0,
146                          unsigned *QOpcodes1);
147
148  /// SelectV6T2BitfieldExtractOp - Select SBFX/UBFX instructions for ARM.
149  SDNode *SelectV6T2BitfieldExtractOp(SDNode *N, unsigned Opc);
150
151  /// SelectCMOVOp - Select CMOV instructions for ARM.
152  SDNode *SelectCMOVOp(SDNode *N);
153  SDNode *SelectT2CMOVShiftOp(SDNode *N, SDValue FalseVal, SDValue TrueVal,
154                              ARMCC::CondCodes CCVal, SDValue CCR,
155                              SDValue InFlag);
156  SDNode *SelectARMCMOVShiftOp(SDNode *N, SDValue FalseVal, SDValue TrueVal,
157                               ARMCC::CondCodes CCVal, SDValue CCR,
158                               SDValue InFlag);
159  SDNode *SelectT2CMOVSoImmOp(SDNode *N, SDValue FalseVal, SDValue TrueVal,
160                              ARMCC::CondCodes CCVal, SDValue CCR,
161                              SDValue InFlag);
162  SDNode *SelectARMCMOVSoImmOp(SDNode *N, SDValue FalseVal, SDValue TrueVal,
163                               ARMCC::CondCodes CCVal, SDValue CCR,
164                               SDValue InFlag);
165
166  /// SelectInlineAsmMemoryOperand - Implement addressing mode selection for
167  /// inline asm expressions.
168  virtual bool SelectInlineAsmMemoryOperand(const SDValue &Op,
169                                            char ConstraintCode,
170                                            std::vector<SDValue> &OutOps);
171
172  /// PairDRegs - Insert a pair of double registers into an implicit def to
173  /// form a quad register.
174  SDNode *PairDRegs(EVT VT, SDValue V0, SDValue V1);
175};
176}
177
178/// isInt32Immediate - This method tests to see if the node is a 32-bit constant
179/// operand. If so Imm will receive the 32-bit value.
180static bool isInt32Immediate(SDNode *N, unsigned &Imm) {
181  if (N->getOpcode() == ISD::Constant && N->getValueType(0) == MVT::i32) {
182    Imm = cast<ConstantSDNode>(N)->getZExtValue();
183    return true;
184  }
185  return false;
186}
187
188// isInt32Immediate - This method tests to see if a constant operand.
189// If so Imm will receive the 32 bit value.
190static bool isInt32Immediate(SDValue N, unsigned &Imm) {
191  return isInt32Immediate(N.getNode(), Imm);
192}
193
194// isOpcWithIntImmediate - This method tests to see if the node is a specific
195// opcode and that it has a immediate integer right operand.
196// If so Imm will receive the 32 bit value.
197static bool isOpcWithIntImmediate(SDNode *N, unsigned Opc, unsigned& Imm) {
198  return N->getOpcode() == Opc &&
199         isInt32Immediate(N->getOperand(1).getNode(), Imm);
200}
201
202
203bool ARMDAGToDAGISel::SelectShifterOperandReg(SDNode *Op,
204                                              SDValue N,
205                                              SDValue &BaseReg,
206                                              SDValue &ShReg,
207                                              SDValue &Opc) {
208  ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(N);
209
210  // Don't match base register only case. That is matched to a separate
211  // lower complexity pattern with explicit register operand.
212  if (ShOpcVal == ARM_AM::no_shift) return false;
213
214  BaseReg = N.getOperand(0);
215  unsigned ShImmVal = 0;
216  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
217    ShReg = CurDAG->getRegister(0, MVT::i32);
218    ShImmVal = RHS->getZExtValue() & 31;
219  } else {
220    ShReg = N.getOperand(1);
221  }
222  Opc = CurDAG->getTargetConstant(ARM_AM::getSORegOpc(ShOpcVal, ShImmVal),
223                                  MVT::i32);
224  return true;
225}
226
227bool ARMDAGToDAGISel::SelectAddrMode2(SDNode *Op, SDValue N,
228                                      SDValue &Base, SDValue &Offset,
229                                      SDValue &Opc) {
230  if (N.getOpcode() == ISD::MUL) {
231    if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
232      // X * [3,5,9] -> X + X * [2,4,8] etc.
233      int RHSC = (int)RHS->getZExtValue();
234      if (RHSC & 1) {
235        RHSC = RHSC & ~1;
236        ARM_AM::AddrOpc AddSub = ARM_AM::add;
237        if (RHSC < 0) {
238          AddSub = ARM_AM::sub;
239          RHSC = - RHSC;
240        }
241        if (isPowerOf2_32(RHSC)) {
242          unsigned ShAmt = Log2_32(RHSC);
243          Base = Offset = N.getOperand(0);
244          Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, ShAmt,
245                                                            ARM_AM::lsl),
246                                          MVT::i32);
247          return true;
248        }
249      }
250    }
251  }
252
253  if (N.getOpcode() != ISD::ADD && N.getOpcode() != ISD::SUB) {
254    Base = N;
255    if (N.getOpcode() == ISD::FrameIndex) {
256      int FI = cast<FrameIndexSDNode>(N)->getIndex();
257      Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
258    } else if (N.getOpcode() == ARMISD::Wrapper &&
259               !(Subtarget->useMovt() &&
260                 N.getOperand(0).getOpcode() == ISD::TargetGlobalAddress)) {
261      Base = N.getOperand(0);
262    }
263    Offset = CurDAG->getRegister(0, MVT::i32);
264    Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(ARM_AM::add, 0,
265                                                      ARM_AM::no_shift),
266                                    MVT::i32);
267    return true;
268  }
269
270  // Match simple R +/- imm12 operands.
271  if (N.getOpcode() == ISD::ADD)
272    if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
273      int RHSC = (int)RHS->getZExtValue();
274      if ((RHSC >= 0 && RHSC < 0x1000) ||
275          (RHSC < 0 && RHSC > -0x1000)) { // 12 bits.
276        Base = N.getOperand(0);
277        if (Base.getOpcode() == ISD::FrameIndex) {
278          int FI = cast<FrameIndexSDNode>(Base)->getIndex();
279          Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
280        }
281        Offset = CurDAG->getRegister(0, MVT::i32);
282
283        ARM_AM::AddrOpc AddSub = ARM_AM::add;
284        if (RHSC < 0) {
285          AddSub = ARM_AM::sub;
286          RHSC = - RHSC;
287        }
288        Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, RHSC,
289                                                          ARM_AM::no_shift),
290                                        MVT::i32);
291        return true;
292      }
293    }
294
295  // Otherwise this is R +/- [possibly shifted] R.
296  ARM_AM::AddrOpc AddSub = N.getOpcode() == ISD::ADD ? ARM_AM::add:ARM_AM::sub;
297  ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(N.getOperand(1));
298  unsigned ShAmt = 0;
299
300  Base   = N.getOperand(0);
301  Offset = N.getOperand(1);
302
303  if (ShOpcVal != ARM_AM::no_shift) {
304    // Check to see if the RHS of the shift is a constant, if not, we can't fold
305    // it.
306    if (ConstantSDNode *Sh =
307           dyn_cast<ConstantSDNode>(N.getOperand(1).getOperand(1))) {
308      ShAmt = Sh->getZExtValue();
309      Offset = N.getOperand(1).getOperand(0);
310    } else {
311      ShOpcVal = ARM_AM::no_shift;
312    }
313  }
314
315  // Try matching (R shl C) + (R).
316  if (N.getOpcode() == ISD::ADD && ShOpcVal == ARM_AM::no_shift) {
317    ShOpcVal = ARM_AM::getShiftOpcForNode(N.getOperand(0));
318    if (ShOpcVal != ARM_AM::no_shift) {
319      // Check to see if the RHS of the shift is a constant, if not, we can't
320      // fold it.
321      if (ConstantSDNode *Sh =
322          dyn_cast<ConstantSDNode>(N.getOperand(0).getOperand(1))) {
323        ShAmt = Sh->getZExtValue();
324        Offset = N.getOperand(0).getOperand(0);
325        Base = N.getOperand(1);
326      } else {
327        ShOpcVal = ARM_AM::no_shift;
328      }
329    }
330  }
331
332  Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, ShAmt, ShOpcVal),
333                                  MVT::i32);
334  return true;
335}
336
337bool ARMDAGToDAGISel::SelectAddrMode2Offset(SDNode *Op, SDValue N,
338                                            SDValue &Offset, SDValue &Opc) {
339  unsigned Opcode = Op->getOpcode();
340  ISD::MemIndexedMode AM = (Opcode == ISD::LOAD)
341    ? cast<LoadSDNode>(Op)->getAddressingMode()
342    : cast<StoreSDNode>(Op)->getAddressingMode();
343  ARM_AM::AddrOpc AddSub = (AM == ISD::PRE_INC || AM == ISD::POST_INC)
344    ? ARM_AM::add : ARM_AM::sub;
345  if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(N)) {
346    int Val = (int)C->getZExtValue();
347    if (Val >= 0 && Val < 0x1000) { // 12 bits.
348      Offset = CurDAG->getRegister(0, MVT::i32);
349      Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, Val,
350                                                        ARM_AM::no_shift),
351                                      MVT::i32);
352      return true;
353    }
354  }
355
356  Offset = N;
357  ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(N);
358  unsigned ShAmt = 0;
359  if (ShOpcVal != ARM_AM::no_shift) {
360    // Check to see if the RHS of the shift is a constant, if not, we can't fold
361    // it.
362    if (ConstantSDNode *Sh = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
363      ShAmt = Sh->getZExtValue();
364      Offset = N.getOperand(0);
365    } else {
366      ShOpcVal = ARM_AM::no_shift;
367    }
368  }
369
370  Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, ShAmt, ShOpcVal),
371                                  MVT::i32);
372  return true;
373}
374
375
376bool ARMDAGToDAGISel::SelectAddrMode3(SDNode *Op, SDValue N,
377                                      SDValue &Base, SDValue &Offset,
378                                      SDValue &Opc) {
379  if (N.getOpcode() == ISD::SUB) {
380    // X - C  is canonicalize to X + -C, no need to handle it here.
381    Base = N.getOperand(0);
382    Offset = N.getOperand(1);
383    Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(ARM_AM::sub, 0),MVT::i32);
384    return true;
385  }
386
387  if (N.getOpcode() != ISD::ADD) {
388    Base = N;
389    if (N.getOpcode() == ISD::FrameIndex) {
390      int FI = cast<FrameIndexSDNode>(N)->getIndex();
391      Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
392    }
393    Offset = CurDAG->getRegister(0, MVT::i32);
394    Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(ARM_AM::add, 0),MVT::i32);
395    return true;
396  }
397
398  // If the RHS is +/- imm8, fold into addr mode.
399  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
400    int RHSC = (int)RHS->getZExtValue();
401    if ((RHSC >= 0 && RHSC < 256) ||
402        (RHSC < 0 && RHSC > -256)) { // note -256 itself isn't allowed.
403      Base = N.getOperand(0);
404      if (Base.getOpcode() == ISD::FrameIndex) {
405        int FI = cast<FrameIndexSDNode>(Base)->getIndex();
406        Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
407      }
408      Offset = CurDAG->getRegister(0, MVT::i32);
409
410      ARM_AM::AddrOpc AddSub = ARM_AM::add;
411      if (RHSC < 0) {
412        AddSub = ARM_AM::sub;
413        RHSC = - RHSC;
414      }
415      Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(AddSub, RHSC),MVT::i32);
416      return true;
417    }
418  }
419
420  Base = N.getOperand(0);
421  Offset = N.getOperand(1);
422  Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(ARM_AM::add, 0), MVT::i32);
423  return true;
424}
425
426bool ARMDAGToDAGISel::SelectAddrMode3Offset(SDNode *Op, SDValue N,
427                                            SDValue &Offset, SDValue &Opc) {
428  unsigned Opcode = Op->getOpcode();
429  ISD::MemIndexedMode AM = (Opcode == ISD::LOAD)
430    ? cast<LoadSDNode>(Op)->getAddressingMode()
431    : cast<StoreSDNode>(Op)->getAddressingMode();
432  ARM_AM::AddrOpc AddSub = (AM == ISD::PRE_INC || AM == ISD::POST_INC)
433    ? ARM_AM::add : ARM_AM::sub;
434  if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(N)) {
435    int Val = (int)C->getZExtValue();
436    if (Val >= 0 && Val < 256) {
437      Offset = CurDAG->getRegister(0, MVT::i32);
438      Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(AddSub, Val), MVT::i32);
439      return true;
440    }
441  }
442
443  Offset = N;
444  Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(AddSub, 0), MVT::i32);
445  return true;
446}
447
448bool ARMDAGToDAGISel::SelectAddrMode4(SDNode *Op, SDValue N,
449                                      SDValue &Addr, SDValue &Mode) {
450  Addr = N;
451  Mode = CurDAG->getTargetConstant(0, MVT::i32);
452  return true;
453}
454
455bool ARMDAGToDAGISel::SelectAddrMode5(SDNode *Op, SDValue N,
456                                      SDValue &Base, SDValue &Offset) {
457  if (N.getOpcode() != ISD::ADD) {
458    Base = N;
459    if (N.getOpcode() == ISD::FrameIndex) {
460      int FI = cast<FrameIndexSDNode>(N)->getIndex();
461      Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
462    } else if (N.getOpcode() == ARMISD::Wrapper &&
463               !(Subtarget->useMovt() &&
464                 N.getOperand(0).getOpcode() == ISD::TargetGlobalAddress)) {
465      Base = N.getOperand(0);
466    }
467    Offset = CurDAG->getTargetConstant(ARM_AM::getAM5Opc(ARM_AM::add, 0),
468                                       MVT::i32);
469    return true;
470  }
471
472  // If the RHS is +/- imm8, fold into addr mode.
473  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
474    int RHSC = (int)RHS->getZExtValue();
475    if ((RHSC & 3) == 0) {  // The constant is implicitly multiplied by 4.
476      RHSC >>= 2;
477      if ((RHSC >= 0 && RHSC < 256) ||
478          (RHSC < 0 && RHSC > -256)) { // note -256 itself isn't allowed.
479        Base = N.getOperand(0);
480        if (Base.getOpcode() == ISD::FrameIndex) {
481          int FI = cast<FrameIndexSDNode>(Base)->getIndex();
482          Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
483        }
484
485        ARM_AM::AddrOpc AddSub = ARM_AM::add;
486        if (RHSC < 0) {
487          AddSub = ARM_AM::sub;
488          RHSC = - RHSC;
489        }
490        Offset = CurDAG->getTargetConstant(ARM_AM::getAM5Opc(AddSub, RHSC),
491                                           MVT::i32);
492        return true;
493      }
494    }
495  }
496
497  Base = N;
498  Offset = CurDAG->getTargetConstant(ARM_AM::getAM5Opc(ARM_AM::add, 0),
499                                     MVT::i32);
500  return true;
501}
502
503bool ARMDAGToDAGISel::SelectAddrMode6(SDNode *Op, SDValue N,
504                                      SDValue &Addr, SDValue &Align) {
505  Addr = N;
506  // Default to no alignment.
507  Align = CurDAG->getTargetConstant(0, MVT::i32);
508  return true;
509}
510
511bool ARMDAGToDAGISel::SelectAddrModePC(SDNode *Op, SDValue N,
512                                       SDValue &Offset, SDValue &Label) {
513  if (N.getOpcode() == ARMISD::PIC_ADD && N.hasOneUse()) {
514    Offset = N.getOperand(0);
515    SDValue N1 = N.getOperand(1);
516    Label  = CurDAG->getTargetConstant(cast<ConstantSDNode>(N1)->getZExtValue(),
517                                       MVT::i32);
518    return true;
519  }
520  return false;
521}
522
523bool ARMDAGToDAGISel::SelectThumbAddrModeRR(SDNode *Op, SDValue N,
524                                            SDValue &Base, SDValue &Offset){
525  // FIXME dl should come from the parent load or store, not the address
526  DebugLoc dl = Op->getDebugLoc();
527  if (N.getOpcode() != ISD::ADD) {
528    ConstantSDNode *NC = dyn_cast<ConstantSDNode>(N);
529    if (!NC || NC->getZExtValue() != 0)
530      return false;
531
532    Base = Offset = N;
533    return true;
534  }
535
536  Base = N.getOperand(0);
537  Offset = N.getOperand(1);
538  return true;
539}
540
541bool
542ARMDAGToDAGISel::SelectThumbAddrModeRI5(SDNode *Op, SDValue N,
543                                        unsigned Scale, SDValue &Base,
544                                        SDValue &OffImm, SDValue &Offset) {
545  if (Scale == 4) {
546    SDValue TmpBase, TmpOffImm;
547    if (SelectThumbAddrModeSP(Op, N, TmpBase, TmpOffImm))
548      return false;  // We want to select tLDRspi / tSTRspi instead.
549    if (N.getOpcode() == ARMISD::Wrapper &&
550        N.getOperand(0).getOpcode() == ISD::TargetConstantPool)
551      return false;  // We want to select tLDRpci instead.
552  }
553
554  if (N.getOpcode() != ISD::ADD) {
555    if (N.getOpcode() == ARMISD::Wrapper &&
556        !(Subtarget->useMovt() &&
557          N.getOperand(0).getOpcode() == ISD::TargetGlobalAddress)) {
558      Base = N.getOperand(0);
559    } else
560      Base = N;
561
562    Offset = CurDAG->getRegister(0, MVT::i32);
563    OffImm = CurDAG->getTargetConstant(0, MVT::i32);
564    return true;
565  }
566
567  // Thumb does not have [sp, r] address mode.
568  RegisterSDNode *LHSR = dyn_cast<RegisterSDNode>(N.getOperand(0));
569  RegisterSDNode *RHSR = dyn_cast<RegisterSDNode>(N.getOperand(1));
570  if ((LHSR && LHSR->getReg() == ARM::SP) ||
571      (RHSR && RHSR->getReg() == ARM::SP)) {
572    Base = N;
573    Offset = CurDAG->getRegister(0, MVT::i32);
574    OffImm = CurDAG->getTargetConstant(0, MVT::i32);
575    return true;
576  }
577
578  // If the RHS is + imm5 * scale, fold into addr mode.
579  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
580    int RHSC = (int)RHS->getZExtValue();
581    if ((RHSC & (Scale-1)) == 0) {  // The constant is implicitly multiplied.
582      RHSC /= Scale;
583      if (RHSC >= 0 && RHSC < 32) {
584        Base = N.getOperand(0);
585        Offset = CurDAG->getRegister(0, MVT::i32);
586        OffImm = CurDAG->getTargetConstant(RHSC, MVT::i32);
587        return true;
588      }
589    }
590  }
591
592  Base = N.getOperand(0);
593  Offset = N.getOperand(1);
594  OffImm = CurDAG->getTargetConstant(0, MVT::i32);
595  return true;
596}
597
598bool ARMDAGToDAGISel::SelectThumbAddrModeS1(SDNode *Op, SDValue N,
599                                            SDValue &Base, SDValue &OffImm,
600                                            SDValue &Offset) {
601  return SelectThumbAddrModeRI5(Op, N, 1, Base, OffImm, Offset);
602}
603
604bool ARMDAGToDAGISel::SelectThumbAddrModeS2(SDNode *Op, SDValue N,
605                                            SDValue &Base, SDValue &OffImm,
606                                            SDValue &Offset) {
607  return SelectThumbAddrModeRI5(Op, N, 2, Base, OffImm, Offset);
608}
609
610bool ARMDAGToDAGISel::SelectThumbAddrModeS4(SDNode *Op, SDValue N,
611                                            SDValue &Base, SDValue &OffImm,
612                                            SDValue &Offset) {
613  return SelectThumbAddrModeRI5(Op, N, 4, Base, OffImm, Offset);
614}
615
616bool ARMDAGToDAGISel::SelectThumbAddrModeSP(SDNode *Op, SDValue N,
617                                           SDValue &Base, SDValue &OffImm) {
618  if (N.getOpcode() == ISD::FrameIndex) {
619    int FI = cast<FrameIndexSDNode>(N)->getIndex();
620    Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
621    OffImm = CurDAG->getTargetConstant(0, MVT::i32);
622    return true;
623  }
624
625  if (N.getOpcode() != ISD::ADD)
626    return false;
627
628  RegisterSDNode *LHSR = dyn_cast<RegisterSDNode>(N.getOperand(0));
629  if (N.getOperand(0).getOpcode() == ISD::FrameIndex ||
630      (LHSR && LHSR->getReg() == ARM::SP)) {
631    // If the RHS is + imm8 * scale, fold into addr mode.
632    if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
633      int RHSC = (int)RHS->getZExtValue();
634      if ((RHSC & 3) == 0) {  // The constant is implicitly multiplied.
635        RHSC >>= 2;
636        if (RHSC >= 0 && RHSC < 256) {
637          Base = N.getOperand(0);
638          if (Base.getOpcode() == ISD::FrameIndex) {
639            int FI = cast<FrameIndexSDNode>(Base)->getIndex();
640            Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
641          }
642          OffImm = CurDAG->getTargetConstant(RHSC, MVT::i32);
643          return true;
644        }
645      }
646    }
647  }
648
649  return false;
650}
651
652bool ARMDAGToDAGISel::SelectT2ShifterOperandReg(SDNode *Op, SDValue N,
653                                                SDValue &BaseReg,
654                                                SDValue &Opc) {
655  ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(N);
656
657  // Don't match base register only case. That is matched to a separate
658  // lower complexity pattern with explicit register operand.
659  if (ShOpcVal == ARM_AM::no_shift) return false;
660
661  BaseReg = N.getOperand(0);
662  unsigned ShImmVal = 0;
663  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
664    ShImmVal = RHS->getZExtValue() & 31;
665    Opc = getI32Imm(ARM_AM::getSORegOpc(ShOpcVal, ShImmVal));
666    return true;
667  }
668
669  return false;
670}
671
672bool ARMDAGToDAGISel::SelectT2AddrModeImm12(SDNode *Op, SDValue N,
673                                            SDValue &Base, SDValue &OffImm) {
674  // Match simple R + imm12 operands.
675
676  // Base only.
677  if (N.getOpcode() != ISD::ADD && N.getOpcode() != ISD::SUB) {
678    if (N.getOpcode() == ISD::FrameIndex) {
679      // Match frame index...
680      int FI = cast<FrameIndexSDNode>(N)->getIndex();
681      Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
682      OffImm  = CurDAG->getTargetConstant(0, MVT::i32);
683      return true;
684    } else if (N.getOpcode() == ARMISD::Wrapper &&
685               !(Subtarget->useMovt() &&
686                 N.getOperand(0).getOpcode() == ISD::TargetGlobalAddress)) {
687      Base = N.getOperand(0);
688      if (Base.getOpcode() == ISD::TargetConstantPool)
689        return false;  // We want to select t2LDRpci instead.
690    } else
691      Base = N;
692    OffImm  = CurDAG->getTargetConstant(0, MVT::i32);
693    return true;
694  }
695
696  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
697    if (SelectT2AddrModeImm8(Op, N, Base, OffImm))
698      // Let t2LDRi8 handle (R - imm8).
699      return false;
700
701    int RHSC = (int)RHS->getZExtValue();
702    if (N.getOpcode() == ISD::SUB)
703      RHSC = -RHSC;
704
705    if (RHSC >= 0 && RHSC < 0x1000) { // 12 bits (unsigned)
706      Base   = N.getOperand(0);
707      if (Base.getOpcode() == ISD::FrameIndex) {
708        int FI = cast<FrameIndexSDNode>(Base)->getIndex();
709        Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
710      }
711      OffImm = CurDAG->getTargetConstant(RHSC, MVT::i32);
712      return true;
713    }
714  }
715
716  // Base only.
717  Base = N;
718  OffImm  = CurDAG->getTargetConstant(0, MVT::i32);
719  return true;
720}
721
722bool ARMDAGToDAGISel::SelectT2AddrModeImm8(SDNode *Op, SDValue N,
723                                           SDValue &Base, SDValue &OffImm) {
724  // Match simple R - imm8 operands.
725  if (N.getOpcode() == ISD::ADD || N.getOpcode() == ISD::SUB) {
726    if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
727      int RHSC = (int)RHS->getSExtValue();
728      if (N.getOpcode() == ISD::SUB)
729        RHSC = -RHSC;
730
731      if ((RHSC >= -255) && (RHSC < 0)) { // 8 bits (always negative)
732        Base = N.getOperand(0);
733        if (Base.getOpcode() == ISD::FrameIndex) {
734          int FI = cast<FrameIndexSDNode>(Base)->getIndex();
735          Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
736        }
737        OffImm = CurDAG->getTargetConstant(RHSC, MVT::i32);
738        return true;
739      }
740    }
741  }
742
743  return false;
744}
745
746bool ARMDAGToDAGISel::SelectT2AddrModeImm8Offset(SDNode *Op, SDValue N,
747                                                 SDValue &OffImm){
748  unsigned Opcode = Op->getOpcode();
749  ISD::MemIndexedMode AM = (Opcode == ISD::LOAD)
750    ? cast<LoadSDNode>(Op)->getAddressingMode()
751    : cast<StoreSDNode>(Op)->getAddressingMode();
752  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N)) {
753    int RHSC = (int)RHS->getZExtValue();
754    if (RHSC >= 0 && RHSC < 0x100) { // 8 bits.
755      OffImm = ((AM == ISD::PRE_INC) || (AM == ISD::POST_INC))
756        ? CurDAG->getTargetConstant(RHSC, MVT::i32)
757        : CurDAG->getTargetConstant(-RHSC, MVT::i32);
758      return true;
759    }
760  }
761
762  return false;
763}
764
765bool ARMDAGToDAGISel::SelectT2AddrModeImm8s4(SDNode *Op, SDValue N,
766                                             SDValue &Base, SDValue &OffImm) {
767  if (N.getOpcode() == ISD::ADD) {
768    if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
769      int RHSC = (int)RHS->getZExtValue();
770      if (((RHSC & 0x3) == 0) &&
771          ((RHSC >= 0 && RHSC < 0x400) || (RHSC < 0 && RHSC > -0x400))) { // 8 bits.
772        Base   = N.getOperand(0);
773        OffImm = CurDAG->getTargetConstant(RHSC, MVT::i32);
774        return true;
775      }
776    }
777  } else if (N.getOpcode() == ISD::SUB) {
778    if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
779      int RHSC = (int)RHS->getZExtValue();
780      if (((RHSC & 0x3) == 0) && (RHSC >= 0 && RHSC < 0x400)) { // 8 bits.
781        Base   = N.getOperand(0);
782        OffImm = CurDAG->getTargetConstant(-RHSC, MVT::i32);
783        return true;
784      }
785    }
786  }
787
788  return false;
789}
790
791bool ARMDAGToDAGISel::SelectT2AddrModeSoReg(SDNode *Op, SDValue N,
792                                            SDValue &Base,
793                                            SDValue &OffReg, SDValue &ShImm) {
794  // (R - imm8) should be handled by t2LDRi8. The rest are handled by t2LDRi12.
795  if (N.getOpcode() != ISD::ADD)
796    return false;
797
798  // Leave (R + imm12) for t2LDRi12, (R - imm8) for t2LDRi8.
799  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
800    int RHSC = (int)RHS->getZExtValue();
801    if (RHSC >= 0 && RHSC < 0x1000) // 12 bits (unsigned)
802      return false;
803    else if (RHSC < 0 && RHSC >= -255) // 8 bits
804      return false;
805  }
806
807  // Look for (R + R) or (R + (R << [1,2,3])).
808  unsigned ShAmt = 0;
809  Base   = N.getOperand(0);
810  OffReg = N.getOperand(1);
811
812  // Swap if it is ((R << c) + R).
813  ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(OffReg);
814  if (ShOpcVal != ARM_AM::lsl) {
815    ShOpcVal = ARM_AM::getShiftOpcForNode(Base);
816    if (ShOpcVal == ARM_AM::lsl)
817      std::swap(Base, OffReg);
818  }
819
820  if (ShOpcVal == ARM_AM::lsl) {
821    // Check to see if the RHS of the shift is a constant, if not, we can't fold
822    // it.
823    if (ConstantSDNode *Sh = dyn_cast<ConstantSDNode>(OffReg.getOperand(1))) {
824      ShAmt = Sh->getZExtValue();
825      if (ShAmt >= 4) {
826        ShAmt = 0;
827        ShOpcVal = ARM_AM::no_shift;
828      } else
829        OffReg = OffReg.getOperand(0);
830    } else {
831      ShOpcVal = ARM_AM::no_shift;
832    }
833  }
834
835  ShImm = CurDAG->getTargetConstant(ShAmt, MVT::i32);
836
837  return true;
838}
839
840//===--------------------------------------------------------------------===//
841
842/// getAL - Returns a ARMCC::AL immediate node.
843static inline SDValue getAL(SelectionDAG *CurDAG) {
844  return CurDAG->getTargetConstant((uint64_t)ARMCC::AL, MVT::i32);
845}
846
847SDNode *ARMDAGToDAGISel::SelectARMIndexedLoad(SDNode *N) {
848  LoadSDNode *LD = cast<LoadSDNode>(N);
849  ISD::MemIndexedMode AM = LD->getAddressingMode();
850  if (AM == ISD::UNINDEXED)
851    return NULL;
852
853  EVT LoadedVT = LD->getMemoryVT();
854  SDValue Offset, AMOpc;
855  bool isPre = (AM == ISD::PRE_INC) || (AM == ISD::PRE_DEC);
856  unsigned Opcode = 0;
857  bool Match = false;
858  if (LoadedVT == MVT::i32 &&
859      SelectAddrMode2Offset(N, LD->getOffset(), Offset, AMOpc)) {
860    Opcode = isPre ? ARM::LDR_PRE : ARM::LDR_POST;
861    Match = true;
862  } else if (LoadedVT == MVT::i16 &&
863             SelectAddrMode3Offset(N, LD->getOffset(), Offset, AMOpc)) {
864    Match = true;
865    Opcode = (LD->getExtensionType() == ISD::SEXTLOAD)
866      ? (isPre ? ARM::LDRSH_PRE : ARM::LDRSH_POST)
867      : (isPre ? ARM::LDRH_PRE : ARM::LDRH_POST);
868  } else if (LoadedVT == MVT::i8 || LoadedVT == MVT::i1) {
869    if (LD->getExtensionType() == ISD::SEXTLOAD) {
870      if (SelectAddrMode3Offset(N, LD->getOffset(), Offset, AMOpc)) {
871        Match = true;
872        Opcode = isPre ? ARM::LDRSB_PRE : ARM::LDRSB_POST;
873      }
874    } else {
875      if (SelectAddrMode2Offset(N, LD->getOffset(), Offset, AMOpc)) {
876        Match = true;
877        Opcode = isPre ? ARM::LDRB_PRE : ARM::LDRB_POST;
878      }
879    }
880  }
881
882  if (Match) {
883    SDValue Chain = LD->getChain();
884    SDValue Base = LD->getBasePtr();
885    SDValue Ops[]= { Base, Offset, AMOpc, getAL(CurDAG),
886                     CurDAG->getRegister(0, MVT::i32), Chain };
887    return CurDAG->getMachineNode(Opcode, N->getDebugLoc(), MVT::i32, MVT::i32,
888                                  MVT::Other, Ops, 6);
889  }
890
891  return NULL;
892}
893
894SDNode *ARMDAGToDAGISel::SelectT2IndexedLoad(SDNode *N) {
895  LoadSDNode *LD = cast<LoadSDNode>(N);
896  ISD::MemIndexedMode AM = LD->getAddressingMode();
897  if (AM == ISD::UNINDEXED)
898    return NULL;
899
900  EVT LoadedVT = LD->getMemoryVT();
901  bool isSExtLd = LD->getExtensionType() == ISD::SEXTLOAD;
902  SDValue Offset;
903  bool isPre = (AM == ISD::PRE_INC) || (AM == ISD::PRE_DEC);
904  unsigned Opcode = 0;
905  bool Match = false;
906  if (SelectT2AddrModeImm8Offset(N, LD->getOffset(), Offset)) {
907    switch (LoadedVT.getSimpleVT().SimpleTy) {
908    case MVT::i32:
909      Opcode = isPre ? ARM::t2LDR_PRE : ARM::t2LDR_POST;
910      break;
911    case MVT::i16:
912      if (isSExtLd)
913        Opcode = isPre ? ARM::t2LDRSH_PRE : ARM::t2LDRSH_POST;
914      else
915        Opcode = isPre ? ARM::t2LDRH_PRE : ARM::t2LDRH_POST;
916      break;
917    case MVT::i8:
918    case MVT::i1:
919      if (isSExtLd)
920        Opcode = isPre ? ARM::t2LDRSB_PRE : ARM::t2LDRSB_POST;
921      else
922        Opcode = isPre ? ARM::t2LDRB_PRE : ARM::t2LDRB_POST;
923      break;
924    default:
925      return NULL;
926    }
927    Match = true;
928  }
929
930  if (Match) {
931    SDValue Chain = LD->getChain();
932    SDValue Base = LD->getBasePtr();
933    SDValue Ops[]= { Base, Offset, getAL(CurDAG),
934                     CurDAG->getRegister(0, MVT::i32), Chain };
935    return CurDAG->getMachineNode(Opcode, N->getDebugLoc(), MVT::i32, MVT::i32,
936                                  MVT::Other, Ops, 5);
937  }
938
939  return NULL;
940}
941
942SDNode *ARMDAGToDAGISel::SelectDYN_ALLOC(SDNode *N) {
943  DebugLoc dl = N->getDebugLoc();
944  EVT VT = N->getValueType(0);
945  SDValue Chain = N->getOperand(0);
946  SDValue Size = N->getOperand(1);
947  SDValue Align = N->getOperand(2);
948  SDValue SP = CurDAG->getRegister(ARM::SP, MVT::i32);
949  int32_t AlignVal = cast<ConstantSDNode>(Align)->getSExtValue();
950  if (AlignVal < 0)
951    // We need to align the stack. Use Thumb1 tAND which is the only thumb
952    // instruction that can read and write SP. This matches to a pseudo
953    // instruction that has a chain to ensure the result is written back to
954    // the stack pointer.
955    SP = SDValue(CurDAG->getMachineNode(ARM::tANDsp, dl, VT, SP, Align), 0);
956
957  bool isC = isa<ConstantSDNode>(Size);
958  uint32_t C = isC ? cast<ConstantSDNode>(Size)->getZExtValue() : ~0UL;
959  // Handle the most common case for both Thumb1 and Thumb2:
960  // tSUBspi - immediate is between 0 ... 508 inclusive.
961  if (C <= 508 && ((C & 3) == 0))
962    // FIXME: tSUBspi encode scale 4 implicitly.
963    return CurDAG->SelectNodeTo(N, ARM::tSUBspi_, VT, MVT::Other, SP,
964                                CurDAG->getTargetConstant(C/4, MVT::i32),
965                                Chain);
966
967  if (Subtarget->isThumb1Only()) {
968    // Use tADDspr since Thumb1 does not have a sub r, sp, r. ARMISelLowering
969    // should have negated the size operand already. FIXME: We can't insert
970    // new target independent node at this stage so we are forced to negate
971    // it earlier. Is there a better solution?
972    return CurDAG->SelectNodeTo(N, ARM::tADDspr_, VT, MVT::Other, SP, Size,
973                                Chain);
974  } else if (Subtarget->isThumb2()) {
975    if (isC && Predicate_t2_so_imm(Size.getNode())) {
976      // t2SUBrSPi
977      SDValue Ops[] = { SP, CurDAG->getTargetConstant(C, MVT::i32), Chain };
978      return CurDAG->SelectNodeTo(N, ARM::t2SUBrSPi_, VT, MVT::Other, Ops, 3);
979    } else if (isC && Predicate_imm0_4095(Size.getNode())) {
980      // t2SUBrSPi12
981      SDValue Ops[] = { SP, CurDAG->getTargetConstant(C, MVT::i32), Chain };
982      return CurDAG->SelectNodeTo(N, ARM::t2SUBrSPi12_, VT, MVT::Other, Ops, 3);
983    } else {
984      // t2SUBrSPs
985      SDValue Ops[] = { SP, Size,
986                        getI32Imm(ARM_AM::getSORegOpc(ARM_AM::lsl,0)), Chain };
987      return CurDAG->SelectNodeTo(N, ARM::t2SUBrSPs_, VT, MVT::Other, Ops, 4);
988    }
989  }
990
991  // FIXME: Add ADD / SUB sp instructions for ARM.
992  return 0;
993}
994
995/// PairDRegs - Insert a pair of double registers into an implicit def to
996/// form a quad register.
997SDNode *ARMDAGToDAGISel::PairDRegs(EVT VT, SDValue V0, SDValue V1) {
998  DebugLoc dl = V0.getNode()->getDebugLoc();
999  SDValue Undef =
1000    SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, dl, VT), 0);
1001  SDValue SubReg0 = CurDAG->getTargetConstant(ARM::DSUBREG_0, MVT::i32);
1002  SDValue SubReg1 = CurDAG->getTargetConstant(ARM::DSUBREG_1, MVT::i32);
1003  SDNode *Pair = CurDAG->getMachineNode(TargetOpcode::INSERT_SUBREG, dl,
1004                                        VT, Undef, V0, SubReg0);
1005  return CurDAG->getMachineNode(TargetOpcode::INSERT_SUBREG, dl,
1006                                VT, SDValue(Pair, 0), V1, SubReg1);
1007}
1008
1009/// GetNEONSubregVT - Given a type for a 128-bit NEON vector, return the type
1010/// for a 64-bit subregister of the vector.
1011static EVT GetNEONSubregVT(EVT VT) {
1012  switch (VT.getSimpleVT().SimpleTy) {
1013  default: llvm_unreachable("unhandled NEON type");
1014  case MVT::v16i8: return MVT::v8i8;
1015  case MVT::v8i16: return MVT::v4i16;
1016  case MVT::v4f32: return MVT::v2f32;
1017  case MVT::v4i32: return MVT::v2i32;
1018  case MVT::v2i64: return MVT::v1i64;
1019  }
1020}
1021
1022SDNode *ARMDAGToDAGISel::SelectVLD(SDNode *N, unsigned NumVecs,
1023                                   unsigned *DOpcodes, unsigned *QOpcodes0,
1024                                   unsigned *QOpcodes1) {
1025  assert(NumVecs >= 1 && NumVecs <= 4 && "VLD NumVecs out-of-range");
1026  DebugLoc dl = N->getDebugLoc();
1027
1028  SDValue MemAddr, Align;
1029  if (!SelectAddrMode6(N, N->getOperand(2), MemAddr, Align))
1030    return NULL;
1031
1032  SDValue Chain = N->getOperand(0);
1033  EVT VT = N->getValueType(0);
1034  bool is64BitVector = VT.is64BitVector();
1035
1036  unsigned OpcodeIndex;
1037  switch (VT.getSimpleVT().SimpleTy) {
1038  default: llvm_unreachable("unhandled vld type");
1039    // Double-register operations:
1040  case MVT::v8i8:  OpcodeIndex = 0; break;
1041  case MVT::v4i16: OpcodeIndex = 1; break;
1042  case MVT::v2f32:
1043  case MVT::v2i32: OpcodeIndex = 2; break;
1044  case MVT::v1i64: OpcodeIndex = 3; break;
1045    // Quad-register operations:
1046  case MVT::v16i8: OpcodeIndex = 0; break;
1047  case MVT::v8i16: OpcodeIndex = 1; break;
1048  case MVT::v4f32:
1049  case MVT::v4i32: OpcodeIndex = 2; break;
1050  case MVT::v2i64: OpcodeIndex = 3;
1051    assert(NumVecs == 1 && "v2i64 type only supported for VLD1");
1052    break;
1053  }
1054
1055  SDValue Pred = CurDAG->getTargetConstant(14, MVT::i32);
1056  SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
1057  if (is64BitVector) {
1058    unsigned Opc = DOpcodes[OpcodeIndex];
1059    const SDValue Ops[] = { MemAddr, Align, Pred, Reg0, Chain };
1060    std::vector<EVT> ResTys(NumVecs, VT);
1061    ResTys.push_back(MVT::Other);
1062    return CurDAG->getMachineNode(Opc, dl, ResTys, Ops, 5);
1063  }
1064
1065  EVT RegVT = GetNEONSubregVT(VT);
1066  if (NumVecs <= 2) {
1067    // Quad registers are directly supported for VLD1 and VLD2,
1068    // loading pairs of D regs.
1069    unsigned Opc = QOpcodes0[OpcodeIndex];
1070    const SDValue Ops[] = { MemAddr, Align, Pred, Reg0, Chain };
1071    std::vector<EVT> ResTys(2 * NumVecs, RegVT);
1072    ResTys.push_back(MVT::Other);
1073    SDNode *VLd = CurDAG->getMachineNode(Opc, dl, ResTys, Ops, 5);
1074    Chain = SDValue(VLd, 2 * NumVecs);
1075
1076    // Combine the even and odd subregs to produce the result.
1077    for (unsigned Vec = 0; Vec < NumVecs; ++Vec) {
1078      SDNode *Q = PairDRegs(VT, SDValue(VLd, 2*Vec), SDValue(VLd, 2*Vec+1));
1079      ReplaceUses(SDValue(N, Vec), SDValue(Q, 0));
1080    }
1081  } else {
1082    // Otherwise, quad registers are loaded with two separate instructions,
1083    // where one loads the even registers and the other loads the odd registers.
1084
1085    std::vector<EVT> ResTys(NumVecs, RegVT);
1086    ResTys.push_back(MemAddr.getValueType());
1087    ResTys.push_back(MVT::Other);
1088
1089    // Load the even subregs.
1090    unsigned Opc = QOpcodes0[OpcodeIndex];
1091    const SDValue OpsA[] = { MemAddr, Align, Reg0, Pred, Reg0, Chain };
1092    SDNode *VLdA = CurDAG->getMachineNode(Opc, dl, ResTys, OpsA, 6);
1093    Chain = SDValue(VLdA, NumVecs+1);
1094
1095    // Load the odd subregs.
1096    Opc = QOpcodes1[OpcodeIndex];
1097    const SDValue OpsB[] = { SDValue(VLdA, NumVecs),
1098                             Align, Reg0, Pred, Reg0, Chain };
1099    SDNode *VLdB = CurDAG->getMachineNode(Opc, dl, ResTys, OpsB, 6);
1100    Chain = SDValue(VLdB, NumVecs+1);
1101
1102    // Combine the even and odd subregs to produce the result.
1103    for (unsigned Vec = 0; Vec < NumVecs; ++Vec) {
1104      SDNode *Q = PairDRegs(VT, SDValue(VLdA, Vec), SDValue(VLdB, Vec));
1105      ReplaceUses(SDValue(N, Vec), SDValue(Q, 0));
1106    }
1107  }
1108  ReplaceUses(SDValue(N, NumVecs), Chain);
1109  return NULL;
1110}
1111
1112SDNode *ARMDAGToDAGISel::SelectVST(SDNode *N, unsigned NumVecs,
1113                                   unsigned *DOpcodes, unsigned *QOpcodes0,
1114                                   unsigned *QOpcodes1) {
1115  assert(NumVecs >=1 && NumVecs <= 4 && "VST NumVecs out-of-range");
1116  DebugLoc dl = N->getDebugLoc();
1117
1118  SDValue MemAddr, Align;
1119  if (!SelectAddrMode6(N, N->getOperand(2), MemAddr, Align))
1120    return NULL;
1121
1122  SDValue Chain = N->getOperand(0);
1123  EVT VT = N->getOperand(3).getValueType();
1124  bool is64BitVector = VT.is64BitVector();
1125
1126  unsigned OpcodeIndex;
1127  switch (VT.getSimpleVT().SimpleTy) {
1128  default: llvm_unreachable("unhandled vst type");
1129    // Double-register operations:
1130  case MVT::v8i8:  OpcodeIndex = 0; break;
1131  case MVT::v4i16: OpcodeIndex = 1; break;
1132  case MVT::v2f32:
1133  case MVT::v2i32: OpcodeIndex = 2; break;
1134  case MVT::v1i64: OpcodeIndex = 3; break;
1135    // Quad-register operations:
1136  case MVT::v16i8: OpcodeIndex = 0; break;
1137  case MVT::v8i16: OpcodeIndex = 1; break;
1138  case MVT::v4f32:
1139  case MVT::v4i32: OpcodeIndex = 2; break;
1140  case MVT::v2i64: OpcodeIndex = 3;
1141    assert(NumVecs == 1 && "v2i64 type only supported for VST1");
1142    break;
1143  }
1144
1145  SDValue Pred = CurDAG->getTargetConstant(14, MVT::i32);
1146  SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
1147
1148  SmallVector<SDValue, 10> Ops;
1149  Ops.push_back(MemAddr);
1150  Ops.push_back(Align);
1151
1152  if (is64BitVector) {
1153    unsigned Opc = DOpcodes[OpcodeIndex];
1154    for (unsigned Vec = 0; Vec < NumVecs; ++Vec)
1155      Ops.push_back(N->getOperand(Vec+3));
1156    Ops.push_back(Pred);
1157    Ops.push_back(Reg0); // predicate register
1158    Ops.push_back(Chain);
1159    return CurDAG->getMachineNode(Opc, dl, MVT::Other, Ops.data(), NumVecs+5);
1160  }
1161
1162  EVT RegVT = GetNEONSubregVT(VT);
1163  if (NumVecs <= 2) {
1164    // Quad registers are directly supported for VST1 and VST2,
1165    // storing pairs of D regs.
1166    unsigned Opc = QOpcodes0[OpcodeIndex];
1167    for (unsigned Vec = 0; Vec < NumVecs; ++Vec) {
1168      Ops.push_back(CurDAG->getTargetExtractSubreg(ARM::DSUBREG_0, dl, RegVT,
1169                                                   N->getOperand(Vec+3)));
1170      Ops.push_back(CurDAG->getTargetExtractSubreg(ARM::DSUBREG_1, dl, RegVT,
1171                                                   N->getOperand(Vec+3)));
1172    }
1173    Ops.push_back(Pred);
1174    Ops.push_back(Reg0); // predicate register
1175    Ops.push_back(Chain);
1176    return CurDAG->getMachineNode(Opc, dl, MVT::Other, Ops.data(),
1177                                  5 + 2 * NumVecs);
1178  }
1179
1180  // Otherwise, quad registers are stored with two separate instructions,
1181  // where one stores the even registers and the other stores the odd registers.
1182
1183  Ops.push_back(Reg0); // post-access address offset
1184
1185  // Store the even subregs.
1186  for (unsigned Vec = 0; Vec < NumVecs; ++Vec)
1187    Ops.push_back(CurDAG->getTargetExtractSubreg(ARM::DSUBREG_0, dl, RegVT,
1188                                                 N->getOperand(Vec+3)));
1189  Ops.push_back(Pred);
1190  Ops.push_back(Reg0); // predicate register
1191  Ops.push_back(Chain);
1192  unsigned Opc = QOpcodes0[OpcodeIndex];
1193  SDNode *VStA = CurDAG->getMachineNode(Opc, dl, MemAddr.getValueType(),
1194                                        MVT::Other, Ops.data(), NumVecs+6);
1195  Chain = SDValue(VStA, 1);
1196
1197  // Store the odd subregs.
1198  Ops[0] = SDValue(VStA, 0); // MemAddr
1199  for (unsigned Vec = 0; Vec < NumVecs; ++Vec)
1200    Ops[Vec+3] = CurDAG->getTargetExtractSubreg(ARM::DSUBREG_1, dl, RegVT,
1201                                                N->getOperand(Vec+3));
1202  Ops[NumVecs+5] = Chain;
1203  Opc = QOpcodes1[OpcodeIndex];
1204  SDNode *VStB = CurDAG->getMachineNode(Opc, dl, MemAddr.getValueType(),
1205                                        MVT::Other, Ops.data(), NumVecs+6);
1206  Chain = SDValue(VStB, 1);
1207  ReplaceUses(SDValue(N, 0), Chain);
1208  return NULL;
1209}
1210
1211SDNode *ARMDAGToDAGISel::SelectVLDSTLane(SDNode *N, bool IsLoad,
1212                                         unsigned NumVecs, unsigned *DOpcodes,
1213                                         unsigned *QOpcodes0,
1214                                         unsigned *QOpcodes1) {
1215  assert(NumVecs >=2 && NumVecs <= 4 && "VLDSTLane NumVecs out-of-range");
1216  DebugLoc dl = N->getDebugLoc();
1217
1218  SDValue MemAddr, Align;
1219  if (!SelectAddrMode6(N, N->getOperand(2), MemAddr, Align))
1220    return NULL;
1221
1222  SDValue Chain = N->getOperand(0);
1223  unsigned Lane =
1224    cast<ConstantSDNode>(N->getOperand(NumVecs+3))->getZExtValue();
1225  EVT VT = IsLoad ? N->getValueType(0) : N->getOperand(3).getValueType();
1226  bool is64BitVector = VT.is64BitVector();
1227
1228  // Quad registers are handled by load/store of subregs. Find the subreg info.
1229  unsigned NumElts = 0;
1230  int SubregIdx = 0;
1231  EVT RegVT = VT;
1232  if (!is64BitVector) {
1233    RegVT = GetNEONSubregVT(VT);
1234    NumElts = RegVT.getVectorNumElements();
1235    SubregIdx = (Lane < NumElts) ? ARM::DSUBREG_0 : ARM::DSUBREG_1;
1236  }
1237
1238  unsigned OpcodeIndex;
1239  switch (VT.getSimpleVT().SimpleTy) {
1240  default: llvm_unreachable("unhandled vld/vst lane type");
1241    // Double-register operations:
1242  case MVT::v8i8:  OpcodeIndex = 0; break;
1243  case MVT::v4i16: OpcodeIndex = 1; break;
1244  case MVT::v2f32:
1245  case MVT::v2i32: OpcodeIndex = 2; break;
1246    // Quad-register operations:
1247  case MVT::v8i16: OpcodeIndex = 0; break;
1248  case MVT::v4f32:
1249  case MVT::v4i32: OpcodeIndex = 1; break;
1250  }
1251
1252  SDValue Pred = CurDAG->getTargetConstant(14, MVT::i32);
1253  SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
1254
1255  SmallVector<SDValue, 10> Ops;
1256  Ops.push_back(MemAddr);
1257  Ops.push_back(Align);
1258
1259  unsigned Opc = 0;
1260  if (is64BitVector) {
1261    Opc = DOpcodes[OpcodeIndex];
1262    for (unsigned Vec = 0; Vec < NumVecs; ++Vec)
1263      Ops.push_back(N->getOperand(Vec+3));
1264  } else {
1265    // Check if this is loading the even or odd subreg of a Q register.
1266    if (Lane < NumElts) {
1267      Opc = QOpcodes0[OpcodeIndex];
1268    } else {
1269      Lane -= NumElts;
1270      Opc = QOpcodes1[OpcodeIndex];
1271    }
1272    // Extract the subregs of the input vector.
1273    for (unsigned Vec = 0; Vec < NumVecs; ++Vec)
1274      Ops.push_back(CurDAG->getTargetExtractSubreg(SubregIdx, dl, RegVT,
1275                                                   N->getOperand(Vec+3)));
1276  }
1277  Ops.push_back(getI32Imm(Lane));
1278  Ops.push_back(Pred);
1279  Ops.push_back(Reg0);
1280  Ops.push_back(Chain);
1281
1282  if (!IsLoad)
1283    return CurDAG->getMachineNode(Opc, dl, MVT::Other, Ops.data(), NumVecs+6);
1284
1285  std::vector<EVT> ResTys(NumVecs, RegVT);
1286  ResTys.push_back(MVT::Other);
1287  SDNode *VLdLn =
1288    CurDAG->getMachineNode(Opc, dl, ResTys, Ops.data(), NumVecs+6);
1289  // For a 64-bit vector load to D registers, nothing more needs to be done.
1290  if (is64BitVector)
1291    return VLdLn;
1292
1293  // For 128-bit vectors, take the 64-bit results of the load and insert them
1294  // as subregs into the result.
1295  for (unsigned Vec = 0; Vec < NumVecs; ++Vec) {
1296    SDValue QuadVec = CurDAG->getTargetInsertSubreg(SubregIdx, dl, VT,
1297                                                    N->getOperand(Vec+3),
1298                                                    SDValue(VLdLn, Vec));
1299    ReplaceUses(SDValue(N, Vec), QuadVec);
1300  }
1301
1302  Chain = SDValue(VLdLn, NumVecs);
1303  ReplaceUses(SDValue(N, NumVecs), Chain);
1304  return NULL;
1305}
1306
1307SDNode *ARMDAGToDAGISel::SelectV6T2BitfieldExtractOp(SDNode *N,
1308                                                     unsigned Opc) {
1309  if (!Subtarget->hasV6T2Ops())
1310    return NULL;
1311
1312  unsigned Shl_imm = 0;
1313  if (isOpcWithIntImmediate(N->getOperand(0).getNode(), ISD::SHL, Shl_imm)) {
1314    assert(Shl_imm > 0 && Shl_imm < 32 && "bad amount in shift node!");
1315    unsigned Srl_imm = 0;
1316    if (isInt32Immediate(N->getOperand(1), Srl_imm)) {
1317      assert(Srl_imm > 0 && Srl_imm < 32 && "bad amount in shift node!");
1318      unsigned Width = 32 - Srl_imm;
1319      int LSB = Srl_imm - Shl_imm;
1320      if (LSB < 0)
1321        return NULL;
1322      SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
1323      SDValue Ops[] = { N->getOperand(0).getOperand(0),
1324                        CurDAG->getTargetConstant(LSB, MVT::i32),
1325                        CurDAG->getTargetConstant(Width, MVT::i32),
1326                        getAL(CurDAG), Reg0 };
1327      return CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops, 5);
1328    }
1329  }
1330  return NULL;
1331}
1332
1333SDNode *ARMDAGToDAGISel::
1334SelectT2CMOVShiftOp(SDNode *N, SDValue FalseVal, SDValue TrueVal,
1335                    ARMCC::CondCodes CCVal, SDValue CCR, SDValue InFlag) {
1336  SDValue CPTmp0;
1337  SDValue CPTmp1;
1338  if (SelectT2ShifterOperandReg(N, TrueVal, CPTmp0, CPTmp1)) {
1339    unsigned SOVal = cast<ConstantSDNode>(CPTmp1)->getZExtValue();
1340    unsigned SOShOp = ARM_AM::getSORegShOp(SOVal);
1341    unsigned Opc = 0;
1342    switch (SOShOp) {
1343    case ARM_AM::lsl: Opc = ARM::t2MOVCClsl; break;
1344    case ARM_AM::lsr: Opc = ARM::t2MOVCClsr; break;
1345    case ARM_AM::asr: Opc = ARM::t2MOVCCasr; break;
1346    case ARM_AM::ror: Opc = ARM::t2MOVCCror; break;
1347    default:
1348      llvm_unreachable("Unknown so_reg opcode!");
1349      break;
1350    }
1351    SDValue SOShImm =
1352      CurDAG->getTargetConstant(ARM_AM::getSORegOffset(SOVal), MVT::i32);
1353    SDValue CC = CurDAG->getTargetConstant(CCVal, MVT::i32);
1354    SDValue Ops[] = { FalseVal, CPTmp0, SOShImm, CC, CCR, InFlag };
1355    return CurDAG->SelectNodeTo(N, Opc, MVT::i32,Ops, 6);
1356  }
1357  return 0;
1358}
1359
1360SDNode *ARMDAGToDAGISel::
1361SelectARMCMOVShiftOp(SDNode *N, SDValue FalseVal, SDValue TrueVal,
1362                     ARMCC::CondCodes CCVal, SDValue CCR, SDValue InFlag) {
1363  SDValue CPTmp0;
1364  SDValue CPTmp1;
1365  SDValue CPTmp2;
1366  if (SelectShifterOperandReg(N, TrueVal, CPTmp0, CPTmp1, CPTmp2)) {
1367    SDValue CC = CurDAG->getTargetConstant(CCVal, MVT::i32);
1368    SDValue Ops[] = { FalseVal, CPTmp0, CPTmp1, CPTmp2, CC, CCR, InFlag };
1369    return CurDAG->SelectNodeTo(N, ARM::MOVCCs, MVT::i32, Ops, 7);
1370  }
1371  return 0;
1372}
1373
1374SDNode *ARMDAGToDAGISel::
1375SelectT2CMOVSoImmOp(SDNode *N, SDValue FalseVal, SDValue TrueVal,
1376                    ARMCC::CondCodes CCVal, SDValue CCR, SDValue InFlag) {
1377  ConstantSDNode *T = dyn_cast<ConstantSDNode>(TrueVal);
1378  if (!T)
1379    return 0;
1380
1381  if (Predicate_t2_so_imm(TrueVal.getNode())) {
1382    SDValue True = CurDAG->getTargetConstant(T->getZExtValue(), MVT::i32);
1383    SDValue CC = CurDAG->getTargetConstant(CCVal, MVT::i32);
1384    SDValue Ops[] = { FalseVal, True, CC, CCR, InFlag };
1385    return CurDAG->SelectNodeTo(N,
1386                                ARM::t2MOVCCi, MVT::i32, Ops, 5);
1387  }
1388  return 0;
1389}
1390
1391SDNode *ARMDAGToDAGISel::
1392SelectARMCMOVSoImmOp(SDNode *N, SDValue FalseVal, SDValue TrueVal,
1393                     ARMCC::CondCodes CCVal, SDValue CCR, SDValue InFlag) {
1394  ConstantSDNode *T = dyn_cast<ConstantSDNode>(TrueVal);
1395  if (!T)
1396    return 0;
1397
1398  if (Predicate_so_imm(TrueVal.getNode())) {
1399    SDValue True = CurDAG->getTargetConstant(T->getZExtValue(), MVT::i32);
1400    SDValue CC = CurDAG->getTargetConstant(CCVal, MVT::i32);
1401    SDValue Ops[] = { FalseVal, True, CC, CCR, InFlag };
1402    return CurDAG->SelectNodeTo(N,
1403                                ARM::MOVCCi, MVT::i32, Ops, 5);
1404  }
1405  return 0;
1406}
1407
1408SDNode *ARMDAGToDAGISel::SelectCMOVOp(SDNode *N) {
1409  EVT VT = N->getValueType(0);
1410  SDValue FalseVal = N->getOperand(0);
1411  SDValue TrueVal  = N->getOperand(1);
1412  SDValue CC = N->getOperand(2);
1413  SDValue CCR = N->getOperand(3);
1414  SDValue InFlag = N->getOperand(4);
1415  assert(CC.getOpcode() == ISD::Constant);
1416  assert(CCR.getOpcode() == ISD::Register);
1417  ARMCC::CondCodes CCVal =
1418    (ARMCC::CondCodes)cast<ConstantSDNode>(CC)->getZExtValue();
1419
1420  if (!Subtarget->isThumb1Only() && VT == MVT::i32) {
1421    // Pattern: (ARMcmov:i32 GPR:i32:$false, so_reg:i32:$true, (imm:i32):$cc)
1422    // Emits: (MOVCCs:i32 GPR:i32:$false, so_reg:i32:$true, (imm:i32):$cc)
1423    // Pattern complexity = 18  cost = 1  size = 0
1424    SDValue CPTmp0;
1425    SDValue CPTmp1;
1426    SDValue CPTmp2;
1427    if (Subtarget->isThumb()) {
1428      SDNode *Res = SelectT2CMOVShiftOp(N, FalseVal, TrueVal,
1429                                        CCVal, CCR, InFlag);
1430      if (!Res)
1431        Res = SelectT2CMOVShiftOp(N, TrueVal, FalseVal,
1432                               ARMCC::getOppositeCondition(CCVal), CCR, InFlag);
1433      if (Res)
1434        return Res;
1435    } else {
1436      SDNode *Res = SelectARMCMOVShiftOp(N, FalseVal, TrueVal,
1437                                         CCVal, CCR, InFlag);
1438      if (!Res)
1439        Res = SelectARMCMOVShiftOp(N, TrueVal, FalseVal,
1440                               ARMCC::getOppositeCondition(CCVal), CCR, InFlag);
1441      if (Res)
1442        return Res;
1443    }
1444
1445    // Pattern: (ARMcmov:i32 GPR:i32:$false,
1446    //             (imm:i32)<<P:Predicate_so_imm>>:$true,
1447    //             (imm:i32):$cc)
1448    // Emits: (MOVCCi:i32 GPR:i32:$false,
1449    //           (so_imm:i32 (imm:i32):$true), (imm:i32):$cc)
1450    // Pattern complexity = 10  cost = 1  size = 0
1451    if (Subtarget->isThumb()) {
1452      SDNode *Res = SelectT2CMOVSoImmOp(N, FalseVal, TrueVal,
1453                                        CCVal, CCR, InFlag);
1454      if (!Res)
1455        Res = SelectT2CMOVSoImmOp(N, TrueVal, FalseVal,
1456                               ARMCC::getOppositeCondition(CCVal), CCR, InFlag);
1457      if (Res)
1458        return Res;
1459    } else {
1460      SDNode *Res = SelectARMCMOVSoImmOp(N, FalseVal, TrueVal,
1461                                         CCVal, CCR, InFlag);
1462      if (!Res)
1463        Res = SelectARMCMOVSoImmOp(N, TrueVal, FalseVal,
1464                               ARMCC::getOppositeCondition(CCVal), CCR, InFlag);
1465      if (Res)
1466        return Res;
1467    }
1468  }
1469
1470  // Pattern: (ARMcmov:i32 GPR:i32:$false, GPR:i32:$true, (imm:i32):$cc)
1471  // Emits: (MOVCCr:i32 GPR:i32:$false, GPR:i32:$true, (imm:i32):$cc)
1472  // Pattern complexity = 6  cost = 1  size = 0
1473  //
1474  // Pattern: (ARMcmov:i32 GPR:i32:$false, GPR:i32:$true, (imm:i32):$cc)
1475  // Emits: (tMOVCCr:i32 GPR:i32:$false, GPR:i32:$true, (imm:i32):$cc)
1476  // Pattern complexity = 6  cost = 11  size = 0
1477  //
1478  // Also FCPYScc and FCPYDcc.
1479  SDValue Tmp2 = CurDAG->getTargetConstant(CCVal, MVT::i32);
1480  SDValue Ops[] = { FalseVal, TrueVal, Tmp2, CCR, InFlag };
1481  unsigned Opc = 0;
1482  switch (VT.getSimpleVT().SimpleTy) {
1483  default: assert(false && "Illegal conditional move type!");
1484    break;
1485  case MVT::i32:
1486    Opc = Subtarget->isThumb()
1487      ? (Subtarget->hasThumb2() ? ARM::t2MOVCCr : ARM::tMOVCCr_pseudo)
1488      : ARM::MOVCCr;
1489    break;
1490  case MVT::f32:
1491    Opc = ARM::VMOVScc;
1492    break;
1493  case MVT::f64:
1494    Opc = ARM::VMOVDcc;
1495    break;
1496  }
1497  return CurDAG->SelectNodeTo(N, Opc, VT, Ops, 5);
1498}
1499
1500SDNode *ARMDAGToDAGISel::Select(SDNode *N) {
1501  DebugLoc dl = N->getDebugLoc();
1502
1503  if (N->isMachineOpcode())
1504    return NULL;   // Already selected.
1505
1506  switch (N->getOpcode()) {
1507  default: break;
1508  case ISD::Constant: {
1509    unsigned Val = cast<ConstantSDNode>(N)->getZExtValue();
1510    bool UseCP = true;
1511    if (Subtarget->hasThumb2())
1512      // Thumb2-aware targets have the MOVT instruction, so all immediates can
1513      // be done with MOV + MOVT, at worst.
1514      UseCP = 0;
1515    else {
1516      if (Subtarget->isThumb()) {
1517        UseCP = (Val > 255 &&                          // MOV
1518                 ~Val > 255 &&                         // MOV + MVN
1519                 !ARM_AM::isThumbImmShiftedVal(Val));  // MOV + LSL
1520      } else
1521        UseCP = (ARM_AM::getSOImmVal(Val) == -1 &&     // MOV
1522                 ARM_AM::getSOImmVal(~Val) == -1 &&    // MVN
1523                 !ARM_AM::isSOImmTwoPartVal(Val));     // two instrs.
1524    }
1525
1526    if (UseCP) {
1527      SDValue CPIdx =
1528        CurDAG->getTargetConstantPool(ConstantInt::get(
1529                                  Type::getInt32Ty(*CurDAG->getContext()), Val),
1530                                      TLI.getPointerTy());
1531
1532      SDNode *ResNode;
1533      if (Subtarget->isThumb1Only()) {
1534        SDValue Pred = CurDAG->getTargetConstant(14, MVT::i32);
1535        SDValue PredReg = CurDAG->getRegister(0, MVT::i32);
1536        SDValue Ops[] = { CPIdx, Pred, PredReg, CurDAG->getEntryNode() };
1537        ResNode = CurDAG->getMachineNode(ARM::tLDRcp, dl, MVT::i32, MVT::Other,
1538                                         Ops, 4);
1539      } else {
1540        SDValue Ops[] = {
1541          CPIdx,
1542          CurDAG->getRegister(0, MVT::i32),
1543          CurDAG->getTargetConstant(0, MVT::i32),
1544          getAL(CurDAG),
1545          CurDAG->getRegister(0, MVT::i32),
1546          CurDAG->getEntryNode()
1547        };
1548        ResNode=CurDAG->getMachineNode(ARM::LDRcp, dl, MVT::i32, MVT::Other,
1549                                       Ops, 6);
1550      }
1551      ReplaceUses(SDValue(N, 0), SDValue(ResNode, 0));
1552      return NULL;
1553    }
1554
1555    // Other cases are autogenerated.
1556    break;
1557  }
1558  case ISD::FrameIndex: {
1559    // Selects to ADDri FI, 0 which in turn will become ADDri SP, imm.
1560    int FI = cast<FrameIndexSDNode>(N)->getIndex();
1561    SDValue TFI = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
1562    if (Subtarget->isThumb1Only()) {
1563      return CurDAG->SelectNodeTo(N, ARM::tADDrSPi, MVT::i32, TFI,
1564                                  CurDAG->getTargetConstant(0, MVT::i32));
1565    } else {
1566      unsigned Opc = ((Subtarget->isThumb() && Subtarget->hasThumb2()) ?
1567                      ARM::t2ADDri : ARM::ADDri);
1568      SDValue Ops[] = { TFI, CurDAG->getTargetConstant(0, MVT::i32),
1569                        getAL(CurDAG), CurDAG->getRegister(0, MVT::i32),
1570                        CurDAG->getRegister(0, MVT::i32) };
1571      return CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops, 5);
1572    }
1573  }
1574  case ARMISD::DYN_ALLOC:
1575    return SelectDYN_ALLOC(N);
1576  case ISD::SRL:
1577    if (SDNode *I = SelectV6T2BitfieldExtractOp(N,
1578                      Subtarget->isThumb() ? ARM::t2UBFX : ARM::UBFX))
1579      return I;
1580    break;
1581  case ISD::SRA:
1582    if (SDNode *I = SelectV6T2BitfieldExtractOp(N,
1583                      Subtarget->isThumb() ? ARM::t2SBFX : ARM::SBFX))
1584      return I;
1585    break;
1586  case ISD::MUL:
1587    if (Subtarget->isThumb1Only())
1588      break;
1589    if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(N->getOperand(1))) {
1590      unsigned RHSV = C->getZExtValue();
1591      if (!RHSV) break;
1592      if (isPowerOf2_32(RHSV-1)) {  // 2^n+1?
1593        unsigned ShImm = Log2_32(RHSV-1);
1594        if (ShImm >= 32)
1595          break;
1596        SDValue V = N->getOperand(0);
1597        ShImm = ARM_AM::getSORegOpc(ARM_AM::lsl, ShImm);
1598        SDValue ShImmOp = CurDAG->getTargetConstant(ShImm, MVT::i32);
1599        SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
1600        if (Subtarget->isThumb()) {
1601          SDValue Ops[] = { V, V, ShImmOp, getAL(CurDAG), Reg0, Reg0 };
1602          return CurDAG->SelectNodeTo(N, ARM::t2ADDrs, MVT::i32, Ops, 6);
1603        } else {
1604          SDValue Ops[] = { V, V, Reg0, ShImmOp, getAL(CurDAG), Reg0, Reg0 };
1605          return CurDAG->SelectNodeTo(N, ARM::ADDrs, MVT::i32, Ops, 7);
1606        }
1607      }
1608      if (isPowerOf2_32(RHSV+1)) {  // 2^n-1?
1609        unsigned ShImm = Log2_32(RHSV+1);
1610        if (ShImm >= 32)
1611          break;
1612        SDValue V = N->getOperand(0);
1613        ShImm = ARM_AM::getSORegOpc(ARM_AM::lsl, ShImm);
1614        SDValue ShImmOp = CurDAG->getTargetConstant(ShImm, MVT::i32);
1615        SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
1616        if (Subtarget->isThumb()) {
1617          SDValue Ops[] = { V, V, ShImmOp, getAL(CurDAG), Reg0 };
1618          return CurDAG->SelectNodeTo(N, ARM::t2RSBrs, MVT::i32, Ops, 5);
1619        } else {
1620          SDValue Ops[] = { V, V, Reg0, ShImmOp, getAL(CurDAG), Reg0, Reg0 };
1621          return CurDAG->SelectNodeTo(N, ARM::RSBrs, MVT::i32, Ops, 7);
1622        }
1623      }
1624    }
1625    break;
1626  case ISD::AND: {
1627    // (and (or x, c2), c1) and top 16-bits of c1 and c2 match, lower 16-bits
1628    // of c1 are 0xffff, and lower 16-bit of c2 are 0. That is, the top 16-bits
1629    // are entirely contributed by c2 and lower 16-bits are entirely contributed
1630    // by x. That's equal to (or (and x, 0xffff), (and c1, 0xffff0000)).
1631    // Select it to: "movt x, ((c1 & 0xffff) >> 16)
1632    EVT VT = N->getValueType(0);
1633    if (VT != MVT::i32)
1634      break;
1635    unsigned Opc = (Subtarget->isThumb() && Subtarget->hasThumb2())
1636      ? ARM::t2MOVTi16
1637      : (Subtarget->hasV6T2Ops() ? ARM::MOVTi16 : 0);
1638    if (!Opc)
1639      break;
1640    SDValue N0 = N->getOperand(0), N1 = N->getOperand(1);
1641    ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
1642    if (!N1C)
1643      break;
1644    if (N0.getOpcode() == ISD::OR && N0.getNode()->hasOneUse()) {
1645      SDValue N2 = N0.getOperand(1);
1646      ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(N2);
1647      if (!N2C)
1648        break;
1649      unsigned N1CVal = N1C->getZExtValue();
1650      unsigned N2CVal = N2C->getZExtValue();
1651      if ((N1CVal & 0xffff0000U) == (N2CVal & 0xffff0000U) &&
1652          (N1CVal & 0xffffU) == 0xffffU &&
1653          (N2CVal & 0xffffU) == 0x0U) {
1654        SDValue Imm16 = CurDAG->getTargetConstant((N2CVal & 0xFFFF0000U) >> 16,
1655                                                  MVT::i32);
1656        SDValue Ops[] = { N0.getOperand(0), Imm16,
1657                          getAL(CurDAG), CurDAG->getRegister(0, MVT::i32) };
1658        return CurDAG->getMachineNode(Opc, dl, VT, Ops, 4);
1659      }
1660    }
1661    break;
1662  }
1663  case ARMISD::VMOVRRD:
1664    return CurDAG->getMachineNode(ARM::VMOVRRD, dl, MVT::i32, MVT::i32,
1665                                  N->getOperand(0), getAL(CurDAG),
1666                                  CurDAG->getRegister(0, MVT::i32));
1667  case ISD::UMUL_LOHI: {
1668    if (Subtarget->isThumb1Only())
1669      break;
1670    if (Subtarget->isThumb()) {
1671      SDValue Ops[] = { N->getOperand(0), N->getOperand(1),
1672                        getAL(CurDAG), CurDAG->getRegister(0, MVT::i32),
1673                        CurDAG->getRegister(0, MVT::i32) };
1674      return CurDAG->getMachineNode(ARM::t2UMULL, dl, MVT::i32, MVT::i32, Ops,4);
1675    } else {
1676      SDValue Ops[] = { N->getOperand(0), N->getOperand(1),
1677                        getAL(CurDAG), CurDAG->getRegister(0, MVT::i32),
1678                        CurDAG->getRegister(0, MVT::i32) };
1679      return CurDAG->getMachineNode(ARM::UMULL, dl, MVT::i32, MVT::i32, Ops, 5);
1680    }
1681  }
1682  case ISD::SMUL_LOHI: {
1683    if (Subtarget->isThumb1Only())
1684      break;
1685    if (Subtarget->isThumb()) {
1686      SDValue Ops[] = { N->getOperand(0), N->getOperand(1),
1687                        getAL(CurDAG), CurDAG->getRegister(0, MVT::i32) };
1688      return CurDAG->getMachineNode(ARM::t2SMULL, dl, MVT::i32, MVT::i32, Ops,4);
1689    } else {
1690      SDValue Ops[] = { N->getOperand(0), N->getOperand(1),
1691                        getAL(CurDAG), CurDAG->getRegister(0, MVT::i32),
1692                        CurDAG->getRegister(0, MVT::i32) };
1693      return CurDAG->getMachineNode(ARM::SMULL, dl, MVT::i32, MVT::i32, Ops, 5);
1694    }
1695  }
1696  case ISD::LOAD: {
1697    SDNode *ResNode = 0;
1698    if (Subtarget->isThumb() && Subtarget->hasThumb2())
1699      ResNode = SelectT2IndexedLoad(N);
1700    else
1701      ResNode = SelectARMIndexedLoad(N);
1702    if (ResNode)
1703      return ResNode;
1704
1705    // VLDMQ must be custom-selected for "v2f64 load" to set the AM5Opc value.
1706    if (Subtarget->hasVFP2() &&
1707        N->getValueType(0).getSimpleVT().SimpleTy == MVT::v2f64) {
1708      SDValue Chain = N->getOperand(0);
1709      SDValue AM5Opc =
1710        CurDAG->getTargetConstant(ARM_AM::getAM5Opc(ARM_AM::ia, 4), MVT::i32);
1711      SDValue Pred = CurDAG->getTargetConstant(14, MVT::i32);
1712      SDValue PredReg = CurDAG->getRegister(0, MVT::i32);
1713      SDValue Ops[] = { N->getOperand(1), AM5Opc, Pred, PredReg, Chain };
1714      return CurDAG->getMachineNode(ARM::VLDMQ, dl, MVT::v2f64, MVT::Other,
1715                                    Ops, 5);
1716    }
1717    // Other cases are autogenerated.
1718    break;
1719  }
1720  case ISD::STORE: {
1721    // VSTMQ must be custom-selected for "v2f64 store" to set the AM5Opc value.
1722    if (Subtarget->hasVFP2() &&
1723        N->getOperand(1).getValueType().getSimpleVT().SimpleTy == MVT::v2f64) {
1724      SDValue Chain = N->getOperand(0);
1725      SDValue AM5Opc =
1726        CurDAG->getTargetConstant(ARM_AM::getAM5Opc(ARM_AM::ia, 4), MVT::i32);
1727      SDValue Pred = CurDAG->getTargetConstant(14, MVT::i32);
1728      SDValue PredReg = CurDAG->getRegister(0, MVT::i32);
1729      SDValue Ops[] = { N->getOperand(1), N->getOperand(2),
1730                        AM5Opc, Pred, PredReg, Chain };
1731      return CurDAG->getMachineNode(ARM::VSTMQ, dl, MVT::Other, Ops, 6);
1732    }
1733    // Other cases are autogenerated.
1734    break;
1735  }
1736  case ARMISD::BRCOND: {
1737    // Pattern: (ARMbrcond:void (bb:Other):$dst, (imm:i32):$cc)
1738    // Emits: (Bcc:void (bb:Other):$dst, (imm:i32):$cc)
1739    // Pattern complexity = 6  cost = 1  size = 0
1740
1741    // Pattern: (ARMbrcond:void (bb:Other):$dst, (imm:i32):$cc)
1742    // Emits: (tBcc:void (bb:Other):$dst, (imm:i32):$cc)
1743    // Pattern complexity = 6  cost = 1  size = 0
1744
1745    // Pattern: (ARMbrcond:void (bb:Other):$dst, (imm:i32):$cc)
1746    // Emits: (t2Bcc:void (bb:Other):$dst, (imm:i32):$cc)
1747    // Pattern complexity = 6  cost = 1  size = 0
1748
1749    unsigned Opc = Subtarget->isThumb() ?
1750      ((Subtarget->hasThumb2()) ? ARM::t2Bcc : ARM::tBcc) : ARM::Bcc;
1751    SDValue Chain = N->getOperand(0);
1752    SDValue N1 = N->getOperand(1);
1753    SDValue N2 = N->getOperand(2);
1754    SDValue N3 = N->getOperand(3);
1755    SDValue InFlag = N->getOperand(4);
1756    assert(N1.getOpcode() == ISD::BasicBlock);
1757    assert(N2.getOpcode() == ISD::Constant);
1758    assert(N3.getOpcode() == ISD::Register);
1759
1760    SDValue Tmp2 = CurDAG->getTargetConstant(((unsigned)
1761                               cast<ConstantSDNode>(N2)->getZExtValue()),
1762                               MVT::i32);
1763    SDValue Ops[] = { N1, Tmp2, N3, Chain, InFlag };
1764    SDNode *ResNode = CurDAG->getMachineNode(Opc, dl, MVT::Other,
1765                                             MVT::Flag, Ops, 5);
1766    Chain = SDValue(ResNode, 0);
1767    if (N->getNumValues() == 2) {
1768      InFlag = SDValue(ResNode, 1);
1769      ReplaceUses(SDValue(N, 1), InFlag);
1770    }
1771    ReplaceUses(SDValue(N, 0),
1772                SDValue(Chain.getNode(), Chain.getResNo()));
1773    return NULL;
1774  }
1775  case ARMISD::CMOV:
1776    return SelectCMOVOp(N);
1777  case ARMISD::CNEG: {
1778    EVT VT = N->getValueType(0);
1779    SDValue N0 = N->getOperand(0);
1780    SDValue N1 = N->getOperand(1);
1781    SDValue N2 = N->getOperand(2);
1782    SDValue N3 = N->getOperand(3);
1783    SDValue InFlag = N->getOperand(4);
1784    assert(N2.getOpcode() == ISD::Constant);
1785    assert(N3.getOpcode() == ISD::Register);
1786
1787    SDValue Tmp2 = CurDAG->getTargetConstant(((unsigned)
1788                               cast<ConstantSDNode>(N2)->getZExtValue()),
1789                               MVT::i32);
1790    SDValue Ops[] = { N0, N1, Tmp2, N3, InFlag };
1791    unsigned Opc = 0;
1792    switch (VT.getSimpleVT().SimpleTy) {
1793    default: assert(false && "Illegal conditional move type!");
1794      break;
1795    case MVT::f32:
1796      Opc = ARM::VNEGScc;
1797      break;
1798    case MVT::f64:
1799      Opc = ARM::VNEGDcc;
1800      break;
1801    }
1802    return CurDAG->SelectNodeTo(N, Opc, VT, Ops, 5);
1803  }
1804
1805  case ARMISD::VZIP: {
1806    unsigned Opc = 0;
1807    EVT VT = N->getValueType(0);
1808    switch (VT.getSimpleVT().SimpleTy) {
1809    default: return NULL;
1810    case MVT::v8i8:  Opc = ARM::VZIPd8; break;
1811    case MVT::v4i16: Opc = ARM::VZIPd16; break;
1812    case MVT::v2f32:
1813    case MVT::v2i32: Opc = ARM::VZIPd32; break;
1814    case MVT::v16i8: Opc = ARM::VZIPq8; break;
1815    case MVT::v8i16: Opc = ARM::VZIPq16; break;
1816    case MVT::v4f32:
1817    case MVT::v4i32: Opc = ARM::VZIPq32; break;
1818    }
1819    SDValue Pred = CurDAG->getTargetConstant(14, MVT::i32);
1820    SDValue PredReg = CurDAG->getRegister(0, MVT::i32);
1821    SDValue Ops[] = { N->getOperand(0), N->getOperand(1), Pred, PredReg };
1822    return CurDAG->getMachineNode(Opc, dl, VT, VT, Ops, 4);
1823  }
1824  case ARMISD::VUZP: {
1825    unsigned Opc = 0;
1826    EVT VT = N->getValueType(0);
1827    switch (VT.getSimpleVT().SimpleTy) {
1828    default: return NULL;
1829    case MVT::v8i8:  Opc = ARM::VUZPd8; break;
1830    case MVT::v4i16: Opc = ARM::VUZPd16; break;
1831    case MVT::v2f32:
1832    case MVT::v2i32: Opc = ARM::VUZPd32; break;
1833    case MVT::v16i8: Opc = ARM::VUZPq8; break;
1834    case MVT::v8i16: Opc = ARM::VUZPq16; break;
1835    case MVT::v4f32:
1836    case MVT::v4i32: Opc = ARM::VUZPq32; break;
1837    }
1838    SDValue Pred = CurDAG->getTargetConstant(14, MVT::i32);
1839    SDValue PredReg = CurDAG->getRegister(0, MVT::i32);
1840    SDValue Ops[] = { N->getOperand(0), N->getOperand(1), Pred, PredReg };
1841    return CurDAG->getMachineNode(Opc, dl, VT, VT, Ops, 4);
1842  }
1843  case ARMISD::VTRN: {
1844    unsigned Opc = 0;
1845    EVT VT = N->getValueType(0);
1846    switch (VT.getSimpleVT().SimpleTy) {
1847    default: return NULL;
1848    case MVT::v8i8:  Opc = ARM::VTRNd8; break;
1849    case MVT::v4i16: Opc = ARM::VTRNd16; break;
1850    case MVT::v2f32:
1851    case MVT::v2i32: Opc = ARM::VTRNd32; break;
1852    case MVT::v16i8: Opc = ARM::VTRNq8; break;
1853    case MVT::v8i16: Opc = ARM::VTRNq16; break;
1854    case MVT::v4f32:
1855    case MVT::v4i32: Opc = ARM::VTRNq32; break;
1856    }
1857    SDValue Pred = CurDAG->getTargetConstant(14, MVT::i32);
1858    SDValue PredReg = CurDAG->getRegister(0, MVT::i32);
1859    SDValue Ops[] = { N->getOperand(0), N->getOperand(1), Pred, PredReg };
1860    return CurDAG->getMachineNode(Opc, dl, VT, VT, Ops, 4);
1861  }
1862
1863  case ISD::INTRINSIC_VOID:
1864  case ISD::INTRINSIC_W_CHAIN: {
1865    unsigned IntNo = cast<ConstantSDNode>(N->getOperand(1))->getZExtValue();
1866    switch (IntNo) {
1867    default:
1868      break;
1869
1870    case Intrinsic::arm_neon_vld1: {
1871      unsigned DOpcodes[] = { ARM::VLD1d8, ARM::VLD1d16,
1872                              ARM::VLD1d32, ARM::VLD1d64 };
1873      unsigned QOpcodes[] = { ARM::VLD1q8, ARM::VLD1q16,
1874                              ARM::VLD1q32, ARM::VLD1q64 };
1875      return SelectVLD(N, 1, DOpcodes, QOpcodes, 0);
1876    }
1877
1878    case Intrinsic::arm_neon_vld2: {
1879      unsigned DOpcodes[] = { ARM::VLD2d8, ARM::VLD2d16,
1880                              ARM::VLD2d32, ARM::VLD1q64 };
1881      unsigned QOpcodes[] = { ARM::VLD2q8, ARM::VLD2q16, ARM::VLD2q32 };
1882      return SelectVLD(N, 2, DOpcodes, QOpcodes, 0);
1883    }
1884
1885    case Intrinsic::arm_neon_vld3: {
1886      unsigned DOpcodes[] = { ARM::VLD3d8, ARM::VLD3d16,
1887                              ARM::VLD3d32, ARM::VLD1d64T };
1888      unsigned QOpcodes0[] = { ARM::VLD3q8_UPD,
1889                               ARM::VLD3q16_UPD,
1890                               ARM::VLD3q32_UPD };
1891      unsigned QOpcodes1[] = { ARM::VLD3q8odd_UPD,
1892                               ARM::VLD3q16odd_UPD,
1893                               ARM::VLD3q32odd_UPD };
1894      return SelectVLD(N, 3, DOpcodes, QOpcodes0, QOpcodes1);
1895    }
1896
1897    case Intrinsic::arm_neon_vld4: {
1898      unsigned DOpcodes[] = { ARM::VLD4d8, ARM::VLD4d16,
1899                              ARM::VLD4d32, ARM::VLD1d64Q };
1900      unsigned QOpcodes0[] = { ARM::VLD4q8_UPD,
1901                               ARM::VLD4q16_UPD,
1902                               ARM::VLD4q32_UPD };
1903      unsigned QOpcodes1[] = { ARM::VLD4q8odd_UPD,
1904                               ARM::VLD4q16odd_UPD,
1905                               ARM::VLD4q32odd_UPD };
1906      return SelectVLD(N, 4, DOpcodes, QOpcodes0, QOpcodes1);
1907    }
1908
1909    case Intrinsic::arm_neon_vld2lane: {
1910      unsigned DOpcodes[] = { ARM::VLD2LNd8, ARM::VLD2LNd16, ARM::VLD2LNd32 };
1911      unsigned QOpcodes0[] = { ARM::VLD2LNq16, ARM::VLD2LNq32 };
1912      unsigned QOpcodes1[] = { ARM::VLD2LNq16odd, ARM::VLD2LNq32odd };
1913      return SelectVLDSTLane(N, true, 2, DOpcodes, QOpcodes0, QOpcodes1);
1914    }
1915
1916    case Intrinsic::arm_neon_vld3lane: {
1917      unsigned DOpcodes[] = { ARM::VLD3LNd8, ARM::VLD3LNd16, ARM::VLD3LNd32 };
1918      unsigned QOpcodes0[] = { ARM::VLD3LNq16, ARM::VLD3LNq32 };
1919      unsigned QOpcodes1[] = { ARM::VLD3LNq16odd, ARM::VLD3LNq32odd };
1920      return SelectVLDSTLane(N, true, 3, DOpcodes, QOpcodes0, QOpcodes1);
1921    }
1922
1923    case Intrinsic::arm_neon_vld4lane: {
1924      unsigned DOpcodes[] = { ARM::VLD4LNd8, ARM::VLD4LNd16, ARM::VLD4LNd32 };
1925      unsigned QOpcodes0[] = { ARM::VLD4LNq16, ARM::VLD4LNq32 };
1926      unsigned QOpcodes1[] = { ARM::VLD4LNq16odd, ARM::VLD4LNq32odd };
1927      return SelectVLDSTLane(N, true, 4, DOpcodes, QOpcodes0, QOpcodes1);
1928    }
1929
1930    case Intrinsic::arm_neon_vst1: {
1931      unsigned DOpcodes[] = { ARM::VST1d8, ARM::VST1d16,
1932                              ARM::VST1d32, ARM::VST1d64 };
1933      unsigned QOpcodes[] = { ARM::VST1q8, ARM::VST1q16,
1934                              ARM::VST1q32, ARM::VST1q64 };
1935      return SelectVST(N, 1, DOpcodes, QOpcodes, 0);
1936    }
1937
1938    case Intrinsic::arm_neon_vst2: {
1939      unsigned DOpcodes[] = { ARM::VST2d8, ARM::VST2d16,
1940                              ARM::VST2d32, ARM::VST1q64 };
1941      unsigned QOpcodes[] = { ARM::VST2q8, ARM::VST2q16, ARM::VST2q32 };
1942      return SelectVST(N, 2, DOpcodes, QOpcodes, 0);
1943    }
1944
1945    case Intrinsic::arm_neon_vst3: {
1946      unsigned DOpcodes[] = { ARM::VST3d8, ARM::VST3d16,
1947                              ARM::VST3d32, ARM::VST1d64T };
1948      unsigned QOpcodes0[] = { ARM::VST3q8_UPD,
1949                               ARM::VST3q16_UPD,
1950                               ARM::VST3q32_UPD };
1951      unsigned QOpcodes1[] = { ARM::VST3q8odd_UPD,
1952                               ARM::VST3q16odd_UPD,
1953                               ARM::VST3q32odd_UPD };
1954      return SelectVST(N, 3, DOpcodes, QOpcodes0, QOpcodes1);
1955    }
1956
1957    case Intrinsic::arm_neon_vst4: {
1958      unsigned DOpcodes[] = { ARM::VST4d8, ARM::VST4d16,
1959                              ARM::VST4d32, ARM::VST1d64Q };
1960      unsigned QOpcodes0[] = { ARM::VST4q8_UPD,
1961                               ARM::VST4q16_UPD,
1962                               ARM::VST4q32_UPD };
1963      unsigned QOpcodes1[] = { ARM::VST4q8odd_UPD,
1964                               ARM::VST4q16odd_UPD,
1965                               ARM::VST4q32odd_UPD };
1966      return SelectVST(N, 4, DOpcodes, QOpcodes0, QOpcodes1);
1967    }
1968
1969    case Intrinsic::arm_neon_vst2lane: {
1970      unsigned DOpcodes[] = { ARM::VST2LNd8, ARM::VST2LNd16, ARM::VST2LNd32 };
1971      unsigned QOpcodes0[] = { ARM::VST2LNq16, ARM::VST2LNq32 };
1972      unsigned QOpcodes1[] = { ARM::VST2LNq16odd, ARM::VST2LNq32odd };
1973      return SelectVLDSTLane(N, false, 2, DOpcodes, QOpcodes0, QOpcodes1);
1974    }
1975
1976    case Intrinsic::arm_neon_vst3lane: {
1977      unsigned DOpcodes[] = { ARM::VST3LNd8, ARM::VST3LNd16, ARM::VST3LNd32 };
1978      unsigned QOpcodes0[] = { ARM::VST3LNq16, ARM::VST3LNq32 };
1979      unsigned QOpcodes1[] = { ARM::VST3LNq16odd, ARM::VST3LNq32odd };
1980      return SelectVLDSTLane(N, false, 3, DOpcodes, QOpcodes0, QOpcodes1);
1981    }
1982
1983    case Intrinsic::arm_neon_vst4lane: {
1984      unsigned DOpcodes[] = { ARM::VST4LNd8, ARM::VST4LNd16, ARM::VST4LNd32 };
1985      unsigned QOpcodes0[] = { ARM::VST4LNq16, ARM::VST4LNq32 };
1986      unsigned QOpcodes1[] = { ARM::VST4LNq16odd, ARM::VST4LNq32odd };
1987      return SelectVLDSTLane(N, false, 4, DOpcodes, QOpcodes0, QOpcodes1);
1988    }
1989    }
1990  }
1991  }
1992
1993  return SelectCode(N);
1994}
1995
1996bool ARMDAGToDAGISel::
1997SelectInlineAsmMemoryOperand(const SDValue &Op, char ConstraintCode,
1998                             std::vector<SDValue> &OutOps) {
1999  assert(ConstraintCode == 'm' && "unexpected asm memory constraint");
2000  // Require the address to be in a register.  That is safe for all ARM
2001  // variants and it is hard to do anything much smarter without knowing
2002  // how the operand is used.
2003  OutOps.push_back(Op);
2004  return false;
2005}
2006
2007/// createARMISelDag - This pass converts a legalized DAG into a
2008/// ARM-specific DAG, ready for instruction scheduling.
2009///
2010FunctionPass *llvm::createARMISelDag(ARMBaseTargetMachine &TM,
2011                                     CodeGenOpt::Level OptLevel) {
2012  return new ARMDAGToDAGISel(TM, OptLevel);
2013}
2014