ARMISelDAGToDAG.cpp revision 12c24690c7dc53cf6e6ca8cb062255e82d568edf 
1//===-- ARMISelDAGToDAG.cpp - A dag to dag inst selector for ARM ----------===//
2//
3//                     The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10// This file defines an instruction selector for the ARM target.
11//
12//===----------------------------------------------------------------------===//
13
14#include "ARM.h"
15#include "ARMAddressingModes.h"
16#include "ARMTargetMachine.h"
17#include "llvm/CallingConv.h"
18#include "llvm/Constants.h"
19#include "llvm/DerivedTypes.h"
20#include "llvm/Function.h"
21#include "llvm/Intrinsics.h"
22#include "llvm/LLVMContext.h"
23#include "llvm/CodeGen/MachineFrameInfo.h"
24#include "llvm/CodeGen/MachineFunction.h"
25#include "llvm/CodeGen/MachineInstrBuilder.h"
26#include "llvm/CodeGen/SelectionDAG.h"
27#include "llvm/CodeGen/SelectionDAGISel.h"
28#include "llvm/Target/TargetLowering.h"
29#include "llvm/Target/TargetOptions.h"
30#include "llvm/Support/CommandLine.h"
31#include "llvm/Support/Compiler.h"
32#include "llvm/Support/Debug.h"
33#include "llvm/Support/ErrorHandling.h"
34#include "llvm/Support/raw_ostream.h"
35
36using namespace llvm;
37
38static cl::opt<bool>
39UseRegSeq("neon-reg-sequence", cl::Hidden,
40          cl::desc("Use reg_sequence to model ld / st of multiple neon regs"));
41
42//===--------------------------------------------------------------------===//
43/// ARMDAGToDAGISel - ARM specific code to select ARM machine
44/// instructions for SelectionDAG operations.
45///
46namespace {
47class ARMDAGToDAGISel : public SelectionDAGISel {
48  ARMBaseTargetMachine &TM;
49
50  /// Subtarget - Keep a pointer to the ARMSubtarget around so that we can
51  /// make the right decision when generating code for different targets.
52  const ARMSubtarget *Subtarget;
53
54public:
55  explicit ARMDAGToDAGISel(ARMBaseTargetMachine &tm,
56                           CodeGenOpt::Level OptLevel)
57    : SelectionDAGISel(tm, OptLevel), TM(tm),
58    Subtarget(&TM.getSubtarget<ARMSubtarget>()) {
59  }
60
61  virtual const char *getPassName() const {
62    return "ARM Instruction Selection";
63  }
64
65  /// getI32Imm - Return a target constant of type i32 with the specified
66  /// value.
67  inline SDValue getI32Imm(unsigned Imm) {
68    return CurDAG->getTargetConstant(Imm, MVT::i32);
69  }
70
71  SDNode *Select(SDNode *N);
72
73  bool SelectShifterOperandReg(SDNode *Op, SDValue N, SDValue &A,
74                               SDValue &B, SDValue &C);
75  bool SelectAddrMode2(SDNode *Op, SDValue N, SDValue &Base,
76                       SDValue &Offset, SDValue &Opc);
77  bool SelectAddrMode2Offset(SDNode *Op, SDValue N,
78                             SDValue &Offset, SDValue &Opc);
79  bool SelectAddrMode3(SDNode *Op, SDValue N, SDValue &Base,
80                       SDValue &Offset, SDValue &Opc);
81  bool SelectAddrMode3Offset(SDNode *Op, SDValue N,
82                             SDValue &Offset, SDValue &Opc);
83  bool SelectAddrMode4(SDNode *Op, SDValue N, SDValue &Addr,
84                       SDValue &Mode);
85  bool SelectAddrMode5(SDNode *Op, SDValue N, SDValue &Base,
86                       SDValue &Offset);
87  bool SelectAddrMode6(SDNode *Op, SDValue N, SDValue &Addr, SDValue &Align);
88
89  bool SelectAddrModePC(SDNode *Op, SDValue N, SDValue &Offset,
90                        SDValue &Label);
91
92  bool SelectThumbAddrModeRR(SDNode *Op, SDValue N, SDValue &Base,
93                             SDValue &Offset);
94  bool SelectThumbAddrModeRI5(SDNode *Op, SDValue N, unsigned Scale,
95                              SDValue &Base, SDValue &OffImm,
96                              SDValue &Offset);
97  bool SelectThumbAddrModeS1(SDNode *Op, SDValue N, SDValue &Base,
98                             SDValue &OffImm, SDValue &Offset);
99  bool SelectThumbAddrModeS2(SDNode *Op, SDValue N, SDValue &Base,
100                             SDValue &OffImm, SDValue &Offset);
101  bool SelectThumbAddrModeS4(SDNode *Op, SDValue N, SDValue &Base,
102                             SDValue &OffImm, SDValue &Offset);
103  bool SelectThumbAddrModeSP(SDNode *Op, SDValue N, SDValue &Base,
104                             SDValue &OffImm);
105
106  bool SelectT2ShifterOperandReg(SDNode *Op, SDValue N,
107                                 SDValue &BaseReg, SDValue &Opc);
108  bool SelectT2AddrModeImm12(SDNode *Op, SDValue N, SDValue &Base,
109                             SDValue &OffImm);
110  bool SelectT2AddrModeImm8(SDNode *Op, SDValue N, SDValue &Base,
111                            SDValue &OffImm);
112  bool SelectT2AddrModeImm8Offset(SDNode *Op, SDValue N,
113                                 SDValue &OffImm);
114  bool SelectT2AddrModeImm8s4(SDNode *Op, SDValue N, SDValue &Base,
115                              SDValue &OffImm);
116  bool SelectT2AddrModeSoReg(SDNode *Op, SDValue N, SDValue &Base,
117                             SDValue &OffReg, SDValue &ShImm);
118
119  // Include the pieces autogenerated from the target description.
120#include "ARMGenDAGISel.inc"
121
122private:
123  /// SelectARMIndexedLoad - Indexed (pre/post inc/dec) load matching code for
124  /// ARM.
125  SDNode *SelectARMIndexedLoad(SDNode *N);
126  SDNode *SelectT2IndexedLoad(SDNode *N);
127
128  /// SelectVLD - Select NEON load intrinsics.  NumVecs should be
129  /// 1, 2, 3 or 4.  The opcode arrays specify the instructions used for
130  /// loads of D registers and even subregs and odd subregs of Q registers.
131  /// For NumVecs <= 2, QOpcodes1 is not used.
132  SDNode *SelectVLD(SDNode *N, unsigned NumVecs, unsigned *DOpcodes,
133                    unsigned *QOpcodes0, unsigned *QOpcodes1);
134
135  /// SelectVST - Select NEON store intrinsics.  NumVecs should
136  /// be 1, 2, 3 or 4.  The opcode arrays specify the instructions used for
137  /// stores of D registers and even subregs and odd subregs of Q registers.
138  /// For NumVecs <= 2, QOpcodes1 is not used.
139  SDNode *SelectVST(SDNode *N, unsigned NumVecs, unsigned *DOpcodes,
140                    unsigned *QOpcodes0, unsigned *QOpcodes1);
141
142  /// SelectVLDSTLane - Select NEON load/store lane intrinsics.  NumVecs should
143  /// be 2, 3 or 4.  The opcode arrays specify the instructions used for
144  /// load/store of D registers and even subregs and odd subregs of Q registers.
145  SDNode *SelectVLDSTLane(SDNode *N, bool IsLoad, unsigned NumVecs,
146                          unsigned *DOpcodes, unsigned *QOpcodes0,
147                          unsigned *QOpcodes1);
148
149  /// SelectV6T2BitfieldExtractOp - Select SBFX/UBFX instructions for ARM.
150  SDNode *SelectV6T2BitfieldExtractOp(SDNode *N, bool isSigned);
151
152  /// SelectCMOVOp - Select CMOV instructions for ARM.
153  SDNode *SelectCMOVOp(SDNode *N);
154  SDNode *SelectT2CMOVShiftOp(SDNode *N, SDValue FalseVal, SDValue TrueVal,
155                              ARMCC::CondCodes CCVal, SDValue CCR,
156                              SDValue InFlag);
157  SDNode *SelectARMCMOVShiftOp(SDNode *N, SDValue FalseVal, SDValue TrueVal,
158                               ARMCC::CondCodes CCVal, SDValue CCR,
159                               SDValue InFlag);
160  SDNode *SelectT2CMOVSoImmOp(SDNode *N, SDValue FalseVal, SDValue TrueVal,
161                              ARMCC::CondCodes CCVal, SDValue CCR,
162                              SDValue InFlag);
163  SDNode *SelectARMCMOVSoImmOp(SDNode *N, SDValue FalseVal, SDValue TrueVal,
164                               ARMCC::CondCodes CCVal, SDValue CCR,
165                               SDValue InFlag);
166
167  SDNode *SelectConcatVector(SDNode *N);
168
169  /// SelectInlineAsmMemoryOperand - Implement addressing mode selection for
170  /// inline asm expressions.
171  virtual bool SelectInlineAsmMemoryOperand(const SDValue &Op,
172                                            char ConstraintCode,
173                                            std::vector<SDValue> &OutOps);
174
175  /// PairDRegs - Form a quad register from a pair of D registers.
176  ///
177  SDNode *PairDRegs(EVT VT, SDValue V0, SDValue V1);
178
179  /// PairDRegs - Form a quad register pair from a pair of Q registers.
180  ///
181  SDNode *PairQRegs(EVT VT, SDValue V0, SDValue V1);
182
183  /// QuadDRegs - Form a quad register pair from a quad of D registers.
184  ///
185  SDNode *QuadDRegs(EVT VT, SDValue V0, SDValue V1, SDValue V2, SDValue V3);
186
187  /// OctoDRegs - Form 8 consecutive D registers.
188  ///
189  SDNode *OctoDRegs(EVT VT, SDValue V0, SDValue V1, SDValue V2, SDValue V3,
190                    SDValue V4, SDValue V5, SDValue V6, SDValue V7);
191};
192}
193
194/// isInt32Immediate - This method tests to see if the node is a 32-bit constant
195/// operand. If so Imm will receive the 32-bit value.
196static bool isInt32Immediate(SDNode *N, unsigned &Imm) {
197  if (N->getOpcode() == ISD::Constant && N->getValueType(0) == MVT::i32) {
198    Imm = cast<ConstantSDNode>(N)->getZExtValue();
199    return true;
200  }
201  return false;
202}
203
204// isInt32Immediate - This method tests to see if a constant operand.
205// If so Imm will receive the 32 bit value.
206static bool isInt32Immediate(SDValue N, unsigned &Imm) {
207  return isInt32Immediate(N.getNode(), Imm);
208}
209
210// isOpcWithIntImmediate - This method tests to see if the node is a specific
211// opcode and that it has a immediate integer right operand.
212// If so Imm will receive the 32 bit value.
213static bool isOpcWithIntImmediate(SDNode *N, unsigned Opc, unsigned& Imm) {
214  return N->getOpcode() == Opc &&
215         isInt32Immediate(N->getOperand(1).getNode(), Imm);
216}
217
218
219bool ARMDAGToDAGISel::SelectShifterOperandReg(SDNode *Op,
220                                              SDValue N,
221                                              SDValue &BaseReg,
222                                              SDValue &ShReg,
223                                              SDValue &Opc) {
224  ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(N);
225
226  // Don't match base register only case. That is matched to a separate
227  // lower complexity pattern with explicit register operand.
228  if (ShOpcVal == ARM_AM::no_shift) return false;
229
230  BaseReg = N.getOperand(0);
231  unsigned ShImmVal = 0;
232  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
233    ShReg = CurDAG->getRegister(0, MVT::i32);
234    ShImmVal = RHS->getZExtValue() & 31;
235  } else {
236    ShReg = N.getOperand(1);
237  }
238  Opc = CurDAG->getTargetConstant(ARM_AM::getSORegOpc(ShOpcVal, ShImmVal),
239                                  MVT::i32);
240  return true;
241}
242
243bool ARMDAGToDAGISel::SelectAddrMode2(SDNode *Op, SDValue N,
244                                      SDValue &Base, SDValue &Offset,
245                                      SDValue &Opc) {
246  if (N.getOpcode() == ISD::MUL) {
247    if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
248      // X * [3,5,9] -> X + X * [2,4,8] etc.
249      int RHSC = (int)RHS->getZExtValue();
250      if (RHSC & 1) {
251        RHSC = RHSC & ~1;
252        ARM_AM::AddrOpc AddSub = ARM_AM::add;
253        if (RHSC < 0) {
254          AddSub = ARM_AM::sub;
255          RHSC = - RHSC;
256        }
257        if (isPowerOf2_32(RHSC)) {
258          unsigned ShAmt = Log2_32(RHSC);
259          Base = Offset = N.getOperand(0);
260          Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, ShAmt,
261                                                            ARM_AM::lsl),
262                                          MVT::i32);
263          return true;
264        }
265      }
266    }
267  }
268
269  if (N.getOpcode() != ISD::ADD && N.getOpcode() != ISD::SUB) {
270    Base = N;
271    if (N.getOpcode() == ISD::FrameIndex) {
272      int FI = cast<FrameIndexSDNode>(N)->getIndex();
273      Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
274    } else if (N.getOpcode() == ARMISD::Wrapper &&
275               !(Subtarget->useMovt() &&
276                 N.getOperand(0).getOpcode() == ISD::TargetGlobalAddress)) {
277      Base = N.getOperand(0);
278    }
279    Offset = CurDAG->getRegister(0, MVT::i32);
280    Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(ARM_AM::add, 0,
281                                                      ARM_AM::no_shift),
282                                    MVT::i32);
283    return true;
284  }
285
286  // Match simple R +/- imm12 operands.
287  if (N.getOpcode() == ISD::ADD)
288    if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
289      int RHSC = (int)RHS->getZExtValue();
290      if ((RHSC >= 0 && RHSC < 0x1000) ||
291          (RHSC < 0 && RHSC > -0x1000)) { // 12 bits.
292        Base = N.getOperand(0);
293        if (Base.getOpcode() == ISD::FrameIndex) {
294          int FI = cast<FrameIndexSDNode>(Base)->getIndex();
295          Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
296        }
297        Offset = CurDAG->getRegister(0, MVT::i32);
298
299        ARM_AM::AddrOpc AddSub = ARM_AM::add;
300        if (RHSC < 0) {
301          AddSub = ARM_AM::sub;
302          RHSC = - RHSC;
303        }
304        Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, RHSC,
305                                                          ARM_AM::no_shift),
306                                        MVT::i32);
307        return true;
308      }
309    }
310
311  // Otherwise this is R +/- [possibly shifted] R.
312  ARM_AM::AddrOpc AddSub = N.getOpcode() == ISD::ADD ? ARM_AM::add:ARM_AM::sub;
313  ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(N.getOperand(1));
314  unsigned ShAmt = 0;
315
316  Base   = N.getOperand(0);
317  Offset = N.getOperand(1);
318
319  if (ShOpcVal != ARM_AM::no_shift) {
320    // Check to see if the RHS of the shift is a constant, if not, we can't fold
321    // it.
322    if (ConstantSDNode *Sh =
323           dyn_cast<ConstantSDNode>(N.getOperand(1).getOperand(1))) {
324      ShAmt = Sh->getZExtValue();
325      Offset = N.getOperand(1).getOperand(0);
326    } else {
327      ShOpcVal = ARM_AM::no_shift;
328    }
329  }
330
331  // Try matching (R shl C) + (R).
332  if (N.getOpcode() == ISD::ADD && ShOpcVal == ARM_AM::no_shift) {
333    ShOpcVal = ARM_AM::getShiftOpcForNode(N.getOperand(0));
334    if (ShOpcVal != ARM_AM::no_shift) {
335      // Check to see if the RHS of the shift is a constant, if not, we can't
336      // fold it.
337      if (ConstantSDNode *Sh =
338          dyn_cast<ConstantSDNode>(N.getOperand(0).getOperand(1))) {
339        ShAmt = Sh->getZExtValue();
340        Offset = N.getOperand(0).getOperand(0);
341        Base = N.getOperand(1);
342      } else {
343        ShOpcVal = ARM_AM::no_shift;
344      }
345    }
346  }
347
348  Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, ShAmt, ShOpcVal),
349                                  MVT::i32);
350  return true;
351}
352
353bool ARMDAGToDAGISel::SelectAddrMode2Offset(SDNode *Op, SDValue N,
354                                            SDValue &Offset, SDValue &Opc) {
355  unsigned Opcode = Op->getOpcode();
356  ISD::MemIndexedMode AM = (Opcode == ISD::LOAD)
357    ? cast<LoadSDNode>(Op)->getAddressingMode()
358    : cast<StoreSDNode>(Op)->getAddressingMode();
359  ARM_AM::AddrOpc AddSub = (AM == ISD::PRE_INC || AM == ISD::POST_INC)
360    ? ARM_AM::add : ARM_AM::sub;
361  if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(N)) {
362    int Val = (int)C->getZExtValue();
363    if (Val >= 0 && Val < 0x1000) { // 12 bits.
364      Offset = CurDAG->getRegister(0, MVT::i32);
365      Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, Val,
366                                                        ARM_AM::no_shift),
367                                      MVT::i32);
368      return true;
369    }
370  }
371
372  Offset = N;
373  ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(N);
374  unsigned ShAmt = 0;
375  if (ShOpcVal != ARM_AM::no_shift) {
376    // Check to see if the RHS of the shift is a constant, if not, we can't fold
377    // it.
378    if (ConstantSDNode *Sh = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
379      ShAmt = Sh->getZExtValue();
380      Offset = N.getOperand(0);
381    } else {
382      ShOpcVal = ARM_AM::no_shift;
383    }
384  }
385
386  Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, ShAmt, ShOpcVal),
387                                  MVT::i32);
388  return true;
389}
390
391
392bool ARMDAGToDAGISel::SelectAddrMode3(SDNode *Op, SDValue N,
393                                      SDValue &Base, SDValue &Offset,
394                                      SDValue &Opc) {
395  if (N.getOpcode() == ISD::SUB) {
396    // X - C  is canonicalize to X + -C, no need to handle it here.
397    Base = N.getOperand(0);
398    Offset = N.getOperand(1);
399    Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(ARM_AM::sub, 0),MVT::i32);
400    return true;
401  }
402
403  if (N.getOpcode() != ISD::ADD) {
404    Base = N;
405    if (N.getOpcode() == ISD::FrameIndex) {
406      int FI = cast<FrameIndexSDNode>(N)->getIndex();
407      Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
408    }
409    Offset = CurDAG->getRegister(0, MVT::i32);
410    Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(ARM_AM::add, 0),MVT::i32);
411    return true;
412  }
413
414  // If the RHS is +/- imm8, fold into addr mode.
415  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
416    int RHSC = (int)RHS->getZExtValue();
417    if ((RHSC >= 0 && RHSC < 256) ||
418        (RHSC < 0 && RHSC > -256)) { // note -256 itself isn't allowed.
419      Base = N.getOperand(0);
420      if (Base.getOpcode() == ISD::FrameIndex) {
421        int FI = cast<FrameIndexSDNode>(Base)->getIndex();
422        Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
423      }
424      Offset = CurDAG->getRegister(0, MVT::i32);
425
426      ARM_AM::AddrOpc AddSub = ARM_AM::add;
427      if (RHSC < 0) {
428        AddSub = ARM_AM::sub;
429        RHSC = - RHSC;
430      }
431      Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(AddSub, RHSC),MVT::i32);
432      return true;
433    }
434  }
435
436  Base = N.getOperand(0);
437  Offset = N.getOperand(1);
438  Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(ARM_AM::add, 0), MVT::i32);
439  return true;
440}
441
442bool ARMDAGToDAGISel::SelectAddrMode3Offset(SDNode *Op, SDValue N,
443                                            SDValue &Offset, SDValue &Opc) {
444  unsigned Opcode = Op->getOpcode();
445  ISD::MemIndexedMode AM = (Opcode == ISD::LOAD)
446    ? cast<LoadSDNode>(Op)->getAddressingMode()
447    : cast<StoreSDNode>(Op)->getAddressingMode();
448  ARM_AM::AddrOpc AddSub = (AM == ISD::PRE_INC || AM == ISD::POST_INC)
449    ? ARM_AM::add : ARM_AM::sub;
450  if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(N)) {
451    int Val = (int)C->getZExtValue();
452    if (Val >= 0 && Val < 256) {
453      Offset = CurDAG->getRegister(0, MVT::i32);
454      Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(AddSub, Val), MVT::i32);
455      return true;
456    }
457  }
458
459  Offset = N;
460  Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(AddSub, 0), MVT::i32);
461  return true;
462}
463
464bool ARMDAGToDAGISel::SelectAddrMode4(SDNode *Op, SDValue N,
465                                      SDValue &Addr, SDValue &Mode) {
466  Addr = N;
467  Mode = CurDAG->getTargetConstant(0, MVT::i32);
468  return true;
469}
470
471bool ARMDAGToDAGISel::SelectAddrMode5(SDNode *Op, SDValue N,
472                                      SDValue &Base, SDValue &Offset) {
473  if (N.getOpcode() != ISD::ADD) {
474    Base = N;
475    if (N.getOpcode() == ISD::FrameIndex) {
476      int FI = cast<FrameIndexSDNode>(N)->getIndex();
477      Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
478    } else if (N.getOpcode() == ARMISD::Wrapper &&
479               !(Subtarget->useMovt() &&
480                 N.getOperand(0).getOpcode() == ISD::TargetGlobalAddress)) {
481      Base = N.getOperand(0);
482    }
483    Offset = CurDAG->getTargetConstant(ARM_AM::getAM5Opc(ARM_AM::add, 0),
484                                       MVT::i32);
485    return true;
486  }
487
488  // If the RHS is +/- imm8, fold into addr mode.
489  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
490    int RHSC = (int)RHS->getZExtValue();
491    if ((RHSC & 3) == 0) {  // The constant is implicitly multiplied by 4.
492      RHSC >>= 2;
493      if ((RHSC >= 0 && RHSC < 256) ||
494          (RHSC < 0 && RHSC > -256)) { // note -256 itself isn't allowed.
495        Base = N.getOperand(0);
496        if (Base.getOpcode() == ISD::FrameIndex) {
497          int FI = cast<FrameIndexSDNode>(Base)->getIndex();
498          Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
499        }
500
501        ARM_AM::AddrOpc AddSub = ARM_AM::add;
502        if (RHSC < 0) {
503          AddSub = ARM_AM::sub;
504          RHSC = - RHSC;
505        }
506        Offset = CurDAG->getTargetConstant(ARM_AM::getAM5Opc(AddSub, RHSC),
507                                           MVT::i32);
508        return true;
509      }
510    }
511  }
512
513  Base = N;
514  Offset = CurDAG->getTargetConstant(ARM_AM::getAM5Opc(ARM_AM::add, 0),
515                                     MVT::i32);
516  return true;
517}
518
519bool ARMDAGToDAGISel::SelectAddrMode6(SDNode *Op, SDValue N,
520                                      SDValue &Addr, SDValue &Align) {
521  Addr = N;
522  // Default to no alignment.
523  Align = CurDAG->getTargetConstant(0, MVT::i32);
524  return true;
525}
526
527bool ARMDAGToDAGISel::SelectAddrModePC(SDNode *Op, SDValue N,
528                                       SDValue &Offset, SDValue &Label) {
529  if (N.getOpcode() == ARMISD::PIC_ADD && N.hasOneUse()) {
530    Offset = N.getOperand(0);
531    SDValue N1 = N.getOperand(1);
532    Label  = CurDAG->getTargetConstant(cast<ConstantSDNode>(N1)->getZExtValue(),
533                                       MVT::i32);
534    return true;
535  }
536  return false;
537}
538
539bool ARMDAGToDAGISel::SelectThumbAddrModeRR(SDNode *Op, SDValue N,
540                                            SDValue &Base, SDValue &Offset){
541  // FIXME dl should come from the parent load or store, not the address
542  DebugLoc dl = Op->getDebugLoc();
543  if (N.getOpcode() != ISD::ADD) {
544    ConstantSDNode *NC = dyn_cast<ConstantSDNode>(N);
545    if (!NC || NC->getZExtValue() != 0)
546      return false;
547
548    Base = Offset = N;
549    return true;
550  }
551
552  Base = N.getOperand(0);
553  Offset = N.getOperand(1);
554  return true;
555}
556
557bool
558ARMDAGToDAGISel::SelectThumbAddrModeRI5(SDNode *Op, SDValue N,
559                                        unsigned Scale, SDValue &Base,
560                                        SDValue &OffImm, SDValue &Offset) {
561  if (Scale == 4) {
562    SDValue TmpBase, TmpOffImm;
563    if (SelectThumbAddrModeSP(Op, N, TmpBase, TmpOffImm))
564      return false;  // We want to select tLDRspi / tSTRspi instead.
565    if (N.getOpcode() == ARMISD::Wrapper &&
566        N.getOperand(0).getOpcode() == ISD::TargetConstantPool)
567      return false;  // We want to select tLDRpci instead.
568  }
569
570  if (N.getOpcode() != ISD::ADD) {
571    if (N.getOpcode() == ARMISD::Wrapper &&
572        !(Subtarget->useMovt() &&
573          N.getOperand(0).getOpcode() == ISD::TargetGlobalAddress)) {
574      Base = N.getOperand(0);
575    } else
576      Base = N;
577
578    Offset = CurDAG->getRegister(0, MVT::i32);
579    OffImm = CurDAG->getTargetConstant(0, MVT::i32);
580    return true;
581  }
582
583  // Thumb does not have [sp, r] address mode.
584  RegisterSDNode *LHSR = dyn_cast<RegisterSDNode>(N.getOperand(0));
585  RegisterSDNode *RHSR = dyn_cast<RegisterSDNode>(N.getOperand(1));
586  if ((LHSR && LHSR->getReg() == ARM::SP) ||
587      (RHSR && RHSR->getReg() == ARM::SP)) {
588    Base = N;
589    Offset = CurDAG->getRegister(0, MVT::i32);
590    OffImm = CurDAG->getTargetConstant(0, MVT::i32);
591    return true;
592  }
593
594  // If the RHS is + imm5 * scale, fold into addr mode.
595  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
596    int RHSC = (int)RHS->getZExtValue();
597    if ((RHSC & (Scale-1)) == 0) {  // The constant is implicitly multiplied.
598      RHSC /= Scale;
599      if (RHSC >= 0 && RHSC < 32) {
600        Base = N.getOperand(0);
601        Offset = CurDAG->getRegister(0, MVT::i32);
602        OffImm = CurDAG->getTargetConstant(RHSC, MVT::i32);
603        return true;
604      }
605    }
606  }
607
608  Base = N.getOperand(0);
609  Offset = N.getOperand(1);
610  OffImm = CurDAG->getTargetConstant(0, MVT::i32);
611  return true;
612}
613
614bool ARMDAGToDAGISel::SelectThumbAddrModeS1(SDNode *Op, SDValue N,
615                                            SDValue &Base, SDValue &OffImm,
616                                            SDValue &Offset) {
617  return SelectThumbAddrModeRI5(Op, N, 1, Base, OffImm, Offset);
618}
619
620bool ARMDAGToDAGISel::SelectThumbAddrModeS2(SDNode *Op, SDValue N,
621                                            SDValue &Base, SDValue &OffImm,
622                                            SDValue &Offset) {
623  return SelectThumbAddrModeRI5(Op, N, 2, Base, OffImm, Offset);
624}
625
626bool ARMDAGToDAGISel::SelectThumbAddrModeS4(SDNode *Op, SDValue N,
627                                            SDValue &Base, SDValue &OffImm,
628                                            SDValue &Offset) {
629  return SelectThumbAddrModeRI5(Op, N, 4, Base, OffImm, Offset);
630}
631
632bool ARMDAGToDAGISel::SelectThumbAddrModeSP(SDNode *Op, SDValue N,
633                                           SDValue &Base, SDValue &OffImm) {
634  if (N.getOpcode() == ISD::FrameIndex) {
635    int FI = cast<FrameIndexSDNode>(N)->getIndex();
636    Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
637    OffImm = CurDAG->getTargetConstant(0, MVT::i32);
638    return true;
639  }
640
641  if (N.getOpcode() != ISD::ADD)
642    return false;
643
644  RegisterSDNode *LHSR = dyn_cast<RegisterSDNode>(N.getOperand(0));
645  if (N.getOperand(0).getOpcode() == ISD::FrameIndex ||
646      (LHSR && LHSR->getReg() == ARM::SP)) {
647    // If the RHS is + imm8 * scale, fold into addr mode.
648    if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
649      int RHSC = (int)RHS->getZExtValue();
650      if ((RHSC & 3) == 0) {  // The constant is implicitly multiplied.
651        RHSC >>= 2;
652        if (RHSC >= 0 && RHSC < 256) {
653          Base = N.getOperand(0);
654          if (Base.getOpcode() == ISD::FrameIndex) {
655            int FI = cast<FrameIndexSDNode>(Base)->getIndex();
656            Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
657          }
658          OffImm = CurDAG->getTargetConstant(RHSC, MVT::i32);
659          return true;
660        }
661      }
662    }
663  }
664
665  return false;
666}
667
668bool ARMDAGToDAGISel::SelectT2ShifterOperandReg(SDNode *Op, SDValue N,
669                                                SDValue &BaseReg,
670                                                SDValue &Opc) {
671  ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(N);
672
673  // Don't match base register only case. That is matched to a separate
674  // lower complexity pattern with explicit register operand.
675  if (ShOpcVal == ARM_AM::no_shift) return false;
676
677  BaseReg = N.getOperand(0);
678  unsigned ShImmVal = 0;
679  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
680    ShImmVal = RHS->getZExtValue() & 31;
681    Opc = getI32Imm(ARM_AM::getSORegOpc(ShOpcVal, ShImmVal));
682    return true;
683  }
684
685  return false;
686}
687
688bool ARMDAGToDAGISel::SelectT2AddrModeImm12(SDNode *Op, SDValue N,
689                                            SDValue &Base, SDValue &OffImm) {
690  // Match simple R + imm12 operands.
691
692  // Base only.
693  if (N.getOpcode() != ISD::ADD && N.getOpcode() != ISD::SUB) {
694    if (N.getOpcode() == ISD::FrameIndex) {
695      // Match frame index...
696      int FI = cast<FrameIndexSDNode>(N)->getIndex();
697      Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
698      OffImm  = CurDAG->getTargetConstant(0, MVT::i32);
699      return true;
700    } else if (N.getOpcode() == ARMISD::Wrapper &&
701               !(Subtarget->useMovt() &&
702                 N.getOperand(0).getOpcode() == ISD::TargetGlobalAddress)) {
703      Base = N.getOperand(0);
704      if (Base.getOpcode() == ISD::TargetConstantPool)
705        return false;  // We want to select t2LDRpci instead.
706    } else
707      Base = N;
708    OffImm  = CurDAG->getTargetConstant(0, MVT::i32);
709    return true;
710  }
711
712  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
713    if (SelectT2AddrModeImm8(Op, N, Base, OffImm))
714      // Let t2LDRi8 handle (R - imm8).
715      return false;
716
717    int RHSC = (int)RHS->getZExtValue();
718    if (N.getOpcode() == ISD::SUB)
719      RHSC = -RHSC;
720
721    if (RHSC >= 0 && RHSC < 0x1000) { // 12 bits (unsigned)
722      Base   = N.getOperand(0);
723      if (Base.getOpcode() == ISD::FrameIndex) {
724        int FI = cast<FrameIndexSDNode>(Base)->getIndex();
725        Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
726      }
727      OffImm = CurDAG->getTargetConstant(RHSC, MVT::i32);
728      return true;
729    }
730  }
731
732  // Base only.
733  Base = N;
734  OffImm  = CurDAG->getTargetConstant(0, MVT::i32);
735  return true;
736}
737
738bool ARMDAGToDAGISel::SelectT2AddrModeImm8(SDNode *Op, SDValue N,
739                                           SDValue &Base, SDValue &OffImm) {
740  // Match simple R - imm8 operands.
741  if (N.getOpcode() == ISD::ADD || N.getOpcode() == ISD::SUB) {
742    if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
743      int RHSC = (int)RHS->getSExtValue();
744      if (N.getOpcode() == ISD::SUB)
745        RHSC = -RHSC;
746
747      if ((RHSC >= -255) && (RHSC < 0)) { // 8 bits (always negative)
748        Base = N.getOperand(0);
749        if (Base.getOpcode() == ISD::FrameIndex) {
750          int FI = cast<FrameIndexSDNode>(Base)->getIndex();
751          Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
752        }
753        OffImm = CurDAG->getTargetConstant(RHSC, MVT::i32);
754        return true;
755      }
756    }
757  }
758
759  return false;
760}
761
762bool ARMDAGToDAGISel::SelectT2AddrModeImm8Offset(SDNode *Op, SDValue N,
763                                                 SDValue &OffImm){
764  unsigned Opcode = Op->getOpcode();
765  ISD::MemIndexedMode AM = (Opcode == ISD::LOAD)
766    ? cast<LoadSDNode>(Op)->getAddressingMode()
767    : cast<StoreSDNode>(Op)->getAddressingMode();
768  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N)) {
769    int RHSC = (int)RHS->getZExtValue();
770    if (RHSC >= 0 && RHSC < 0x100) { // 8 bits.
771      OffImm = ((AM == ISD::PRE_INC) || (AM == ISD::POST_INC))
772        ? CurDAG->getTargetConstant(RHSC, MVT::i32)
773        : CurDAG->getTargetConstant(-RHSC, MVT::i32);
774      return true;
775    }
776  }
777
778  return false;
779}
780
781bool ARMDAGToDAGISel::SelectT2AddrModeImm8s4(SDNode *Op, SDValue N,
782                                             SDValue &Base, SDValue &OffImm) {
783  if (N.getOpcode() == ISD::ADD) {
784    if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
785      int RHSC = (int)RHS->getZExtValue();
786      if (((RHSC & 0x3) == 0) &&
787          ((RHSC >= 0 && RHSC < 0x400) || (RHSC < 0 && RHSC > -0x400))) { // 8 bits.
788        Base   = N.getOperand(0);
789        OffImm = CurDAG->getTargetConstant(RHSC, MVT::i32);
790        return true;
791      }
792    }
793  } else if (N.getOpcode() == ISD::SUB) {
794    if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
795      int RHSC = (int)RHS->getZExtValue();
796      if (((RHSC & 0x3) == 0) && (RHSC >= 0 && RHSC < 0x400)) { // 8 bits.
797        Base   = N.getOperand(0);
798        OffImm = CurDAG->getTargetConstant(-RHSC, MVT::i32);
799        return true;
800      }
801    }
802  }
803
804  return false;
805}
806
807bool ARMDAGToDAGISel::SelectT2AddrModeSoReg(SDNode *Op, SDValue N,
808                                            SDValue &Base,
809                                            SDValue &OffReg, SDValue &ShImm) {
810  // (R - imm8) should be handled by t2LDRi8. The rest are handled by t2LDRi12.
811  if (N.getOpcode() != ISD::ADD)
812    return false;
813
814  // Leave (R + imm12) for t2LDRi12, (R - imm8) for t2LDRi8.
815  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
816    int RHSC = (int)RHS->getZExtValue();
817    if (RHSC >= 0 && RHSC < 0x1000) // 12 bits (unsigned)
818      return false;
819    else if (RHSC < 0 && RHSC >= -255) // 8 bits
820      return false;
821  }
822
823  // Look for (R + R) or (R + (R << [1,2,3])).
824  unsigned ShAmt = 0;
825  Base   = N.getOperand(0);
826  OffReg = N.getOperand(1);
827
828  // Swap if it is ((R << c) + R).
829  ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(OffReg);
830  if (ShOpcVal != ARM_AM::lsl) {
831    ShOpcVal = ARM_AM::getShiftOpcForNode(Base);
832    if (ShOpcVal == ARM_AM::lsl)
833      std::swap(Base, OffReg);
834  }
835
836  if (ShOpcVal == ARM_AM::lsl) {
837    // Check to see if the RHS of the shift is a constant, if not, we can't fold
838    // it.
839    if (ConstantSDNode *Sh = dyn_cast<ConstantSDNode>(OffReg.getOperand(1))) {
840      ShAmt = Sh->getZExtValue();
841      if (ShAmt >= 4) {
842        ShAmt = 0;
843        ShOpcVal = ARM_AM::no_shift;
844      } else
845        OffReg = OffReg.getOperand(0);
846    } else {
847      ShOpcVal = ARM_AM::no_shift;
848    }
849  }
850
851  ShImm = CurDAG->getTargetConstant(ShAmt, MVT::i32);
852
853  return true;
854}
855
856//===--------------------------------------------------------------------===//
857
858/// getAL - Returns a ARMCC::AL immediate node.
859static inline SDValue getAL(SelectionDAG *CurDAG) {
860  return CurDAG->getTargetConstant((uint64_t)ARMCC::AL, MVT::i32);
861}
862
863SDNode *ARMDAGToDAGISel::SelectARMIndexedLoad(SDNode *N) {
864  LoadSDNode *LD = cast<LoadSDNode>(N);
865  ISD::MemIndexedMode AM = LD->getAddressingMode();
866  if (AM == ISD::UNINDEXED)
867    return NULL;
868
869  EVT LoadedVT = LD->getMemoryVT();
870  SDValue Offset, AMOpc;
871  bool isPre = (AM == ISD::PRE_INC) || (AM == ISD::PRE_DEC);
872  unsigned Opcode = 0;
873  bool Match = false;
874  if (LoadedVT == MVT::i32 &&
875      SelectAddrMode2Offset(N, LD->getOffset(), Offset, AMOpc)) {
876    Opcode = isPre ? ARM::LDR_PRE : ARM::LDR_POST;
877    Match = true;
878  } else if (LoadedVT == MVT::i16 &&
879             SelectAddrMode3Offset(N, LD->getOffset(), Offset, AMOpc)) {
880    Match = true;
881    Opcode = (LD->getExtensionType() == ISD::SEXTLOAD)
882      ? (isPre ? ARM::LDRSH_PRE : ARM::LDRSH_POST)
883      : (isPre ? ARM::LDRH_PRE : ARM::LDRH_POST);
884  } else if (LoadedVT == MVT::i8 || LoadedVT == MVT::i1) {
885    if (LD->getExtensionType() == ISD::SEXTLOAD) {
886      if (SelectAddrMode3Offset(N, LD->getOffset(), Offset, AMOpc)) {
887        Match = true;
888        Opcode = isPre ? ARM::LDRSB_PRE : ARM::LDRSB_POST;
889      }
890    } else {
891      if (SelectAddrMode2Offset(N, LD->getOffset(), Offset, AMOpc)) {
892        Match = true;
893        Opcode = isPre ? ARM::LDRB_PRE : ARM::LDRB_POST;
894      }
895    }
896  }
897
898  if (Match) {
899    SDValue Chain = LD->getChain();
900    SDValue Base = LD->getBasePtr();
901    SDValue Ops[]= { Base, Offset, AMOpc, getAL(CurDAG),
902                     CurDAG->getRegister(0, MVT::i32), Chain };
903    return CurDAG->getMachineNode(Opcode, N->getDebugLoc(), MVT::i32, MVT::i32,
904                                  MVT::Other, Ops, 6);
905  }
906
907  return NULL;
908}
909
910SDNode *ARMDAGToDAGISel::SelectT2IndexedLoad(SDNode *N) {
911  LoadSDNode *LD = cast<LoadSDNode>(N);
912  ISD::MemIndexedMode AM = LD->getAddressingMode();
913  if (AM == ISD::UNINDEXED)
914    return NULL;
915
916  EVT LoadedVT = LD->getMemoryVT();
917  bool isSExtLd = LD->getExtensionType() == ISD::SEXTLOAD;
918  SDValue Offset;
919  bool isPre = (AM == ISD::PRE_INC) || (AM == ISD::PRE_DEC);
920  unsigned Opcode = 0;
921  bool Match = false;
922  if (SelectT2AddrModeImm8Offset(N, LD->getOffset(), Offset)) {
923    switch (LoadedVT.getSimpleVT().SimpleTy) {
924    case MVT::i32:
925      Opcode = isPre ? ARM::t2LDR_PRE : ARM::t2LDR_POST;
926      break;
927    case MVT::i16:
928      if (isSExtLd)
929        Opcode = isPre ? ARM::t2LDRSH_PRE : ARM::t2LDRSH_POST;
930      else
931        Opcode = isPre ? ARM::t2LDRH_PRE : ARM::t2LDRH_POST;
932      break;
933    case MVT::i8:
934    case MVT::i1:
935      if (isSExtLd)
936        Opcode = isPre ? ARM::t2LDRSB_PRE : ARM::t2LDRSB_POST;
937      else
938        Opcode = isPre ? ARM::t2LDRB_PRE : ARM::t2LDRB_POST;
939      break;
940    default:
941      return NULL;
942    }
943    Match = true;
944  }
945
946  if (Match) {
947    SDValue Chain = LD->getChain();
948    SDValue Base = LD->getBasePtr();
949    SDValue Ops[]= { Base, Offset, getAL(CurDAG),
950                     CurDAG->getRegister(0, MVT::i32), Chain };
951    return CurDAG->getMachineNode(Opcode, N->getDebugLoc(), MVT::i32, MVT::i32,
952                                  MVT::Other, Ops, 5);
953  }
954
955  return NULL;
956}
957
958/// PairDRegs - Form a quad register from a pair of D registers.
959///
960SDNode *ARMDAGToDAGISel::PairDRegs(EVT VT, SDValue V0, SDValue V1) {
961  DebugLoc dl = V0.getNode()->getDebugLoc();
962  SDValue SubReg0 = CurDAG->getTargetConstant(ARM::DSUBREG_0, MVT::i32);
963  SDValue SubReg1 = CurDAG->getTargetConstant(ARM::DSUBREG_1, MVT::i32);
964  if (llvm::ModelWithRegSequence()) {
965    const SDValue Ops[] = { V0, SubReg0, V1, SubReg1 };
966    return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops, 4);
967  }
968  SDValue Undef =
969    SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, dl, VT), 0);
970  SDNode *Pair = CurDAG->getMachineNode(TargetOpcode::INSERT_SUBREG, dl,
971                                        VT, Undef, V0, SubReg0);
972  return CurDAG->getMachineNode(TargetOpcode::INSERT_SUBREG, dl,
973                                VT, SDValue(Pair, 0), V1, SubReg1);
974}
975
976/// PairQRegs - Form 4 consecutive D registers from a pair of Q registers.
977///
978SDNode *ARMDAGToDAGISel::PairQRegs(EVT VT, SDValue V0, SDValue V1) {
979  DebugLoc dl = V0.getNode()->getDebugLoc();
980  SDValue SubReg0 = CurDAG->getTargetConstant(ARM::QSUBREG_0, MVT::i32);
981  SDValue SubReg1 = CurDAG->getTargetConstant(ARM::QSUBREG_1, MVT::i32);
982  const SDValue Ops[] = { V0, SubReg0, V1, SubReg1 };
983  return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops, 4);
984}
985
986/// QuadDRegs - Form 4 consecutive D registers.
987///
988SDNode *ARMDAGToDAGISel::QuadDRegs(EVT VT, SDValue V0, SDValue V1,
989                                   SDValue V2, SDValue V3) {
990  DebugLoc dl = V0.getNode()->getDebugLoc();
991  SDValue SubReg0 = CurDAG->getTargetConstant(ARM::DSUBREG_0, MVT::i32);
992  SDValue SubReg1 = CurDAG->getTargetConstant(ARM::DSUBREG_1, MVT::i32);
993  SDValue SubReg2 = CurDAG->getTargetConstant(ARM::DSUBREG_2, MVT::i32);
994  SDValue SubReg3 = CurDAG->getTargetConstant(ARM::DSUBREG_3, MVT::i32);
995  const SDValue Ops[] = { V0, SubReg0, V1, SubReg1, V2, SubReg2, V3, SubReg3 };
996  return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops, 8);
997}
998
999/// OctoDRegs - Form 8 consecutive D registers.
1000///
1001SDNode *ARMDAGToDAGISel::OctoDRegs(EVT VT, SDValue V0, SDValue V1,
1002                                   SDValue V2, SDValue V3,
1003                                   SDValue V4, SDValue V5,
1004                                   SDValue V6, SDValue V7) {
1005  DebugLoc dl = V0.getNode()->getDebugLoc();
1006  SDValue SubReg0 = CurDAG->getTargetConstant(ARM::DSUBREG_0, MVT::i32);
1007  SDValue SubReg1 = CurDAG->getTargetConstant(ARM::DSUBREG_1, MVT::i32);
1008  SDValue SubReg2 = CurDAG->getTargetConstant(ARM::DSUBREG_2, MVT::i32);
1009  SDValue SubReg3 = CurDAG->getTargetConstant(ARM::DSUBREG_3, MVT::i32);
1010  SDValue SubReg4 = CurDAG->getTargetConstant(ARM::DSUBREG_4, MVT::i32);
1011  SDValue SubReg5 = CurDAG->getTargetConstant(ARM::DSUBREG_5, MVT::i32);
1012  SDValue SubReg6 = CurDAG->getTargetConstant(ARM::DSUBREG_6, MVT::i32);
1013  SDValue SubReg7 = CurDAG->getTargetConstant(ARM::DSUBREG_7, MVT::i32);
1014  const SDValue Ops[] ={ V0, SubReg0, V1, SubReg1, V2, SubReg2, V3, SubReg3,
1015                         V4, SubReg4, V5, SubReg5, V6, SubReg6, V7, SubReg7 };
1016  return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops, 16);
1017}
1018
1019/// GetNEONSubregVT - Given a type for a 128-bit NEON vector, return the type
1020/// for a 64-bit subregister of the vector.
1021static EVT GetNEONSubregVT(EVT VT) {
1022  switch (VT.getSimpleVT().SimpleTy) {
1023  default: llvm_unreachable("unhandled NEON type");
1024  case MVT::v16i8: return MVT::v8i8;
1025  case MVT::v8i16: return MVT::v4i16;
1026  case MVT::v4f32: return MVT::v2f32;
1027  case MVT::v4i32: return MVT::v2i32;
1028  case MVT::v2i64: return MVT::v1i64;
1029  }
1030}
1031
1032SDNode *ARMDAGToDAGISel::SelectVLD(SDNode *N, unsigned NumVecs,
1033                                   unsigned *DOpcodes, unsigned *QOpcodes0,
1034                                   unsigned *QOpcodes1) {
1035  assert(NumVecs >= 1 && NumVecs <= 4 && "VLD NumVecs out-of-range");
1036  DebugLoc dl = N->getDebugLoc();
1037
1038  SDValue MemAddr, Align;
1039  if (!SelectAddrMode6(N, N->getOperand(2), MemAddr, Align))
1040    return NULL;
1041
1042  SDValue Chain = N->getOperand(0);
1043  EVT VT = N->getValueType(0);
1044  bool is64BitVector = VT.is64BitVector();
1045
1046  unsigned OpcodeIndex;
1047  switch (VT.getSimpleVT().SimpleTy) {
1048  default: llvm_unreachable("unhandled vld type");
1049    // Double-register operations:
1050  case MVT::v8i8:  OpcodeIndex = 0; break;
1051  case MVT::v4i16: OpcodeIndex = 1; break;
1052  case MVT::v2f32:
1053  case MVT::v2i32: OpcodeIndex = 2; break;
1054  case MVT::v1i64: OpcodeIndex = 3; break;
1055    // Quad-register operations:
1056  case MVT::v16i8: OpcodeIndex = 0; break;
1057  case MVT::v8i16: OpcodeIndex = 1; break;
1058  case MVT::v4f32:
1059  case MVT::v4i32: OpcodeIndex = 2; break;
1060  case MVT::v2i64: OpcodeIndex = 3;
1061    assert(NumVecs == 1 && "v2i64 type only supported for VLD1");
1062    break;
1063  }
1064
1065  SDValue Pred = getAL(CurDAG);
1066  SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
1067  if (is64BitVector) {
1068    unsigned Opc = DOpcodes[OpcodeIndex];
1069    const SDValue Ops[] = { MemAddr, Align, Pred, Reg0, Chain };
1070    std::vector<EVT> ResTys(NumVecs, VT);
1071    ResTys.push_back(MVT::Other);
1072    SDNode *VLd = CurDAG->getMachineNode(Opc, dl, ResTys, Ops, 5);
1073    if (!llvm::ModelWithRegSequence() || NumVecs < 2)
1074      return VLd;
1075
1076    assert(NumVecs <= 4);
1077    SDValue RegSeq;
1078    SDValue V0 = SDValue(VLd, 0);
1079    SDValue V1 = SDValue(VLd, 1);
1080
1081    // Form a REG_SEQUENCE to force register allocation.
1082    if (NumVecs == 2)
1083      RegSeq = SDValue(PairDRegs(MVT::v2i64, V0, V1), 0);
1084    else {
1085      SDValue V2 = SDValue(VLd, 2);
1086      // If it's a vld3, form a quad D-register but discard the last part.
1087      SDValue V3 = (NumVecs == 3)
1088          ? SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF,dl,VT), 0)
1089          : SDValue(VLd, 3);
1090      RegSeq = SDValue(QuadDRegs(MVT::v4i64, V0, V1, V2, V3), 0);
1091    }
1092
1093    for (unsigned Vec = 0; Vec < NumVecs; ++Vec) {
1094      SDValue D = CurDAG->getTargetExtractSubreg(ARM::DSUBREG_0+Vec,
1095                                                 dl, VT, RegSeq);
1096      ReplaceUses(SDValue(N, Vec), D);
1097    }
1098    ReplaceUses(SDValue(N, NumVecs), SDValue(VLd, NumVecs));
1099    return NULL;
1100  }
1101
1102  EVT RegVT = GetNEONSubregVT(VT);
1103  if (NumVecs <= 2) {
1104    // Quad registers are directly supported for VLD1 and VLD2,
1105    // loading pairs of D regs.
1106    unsigned Opc = QOpcodes0[OpcodeIndex];
1107    const SDValue Ops[] = { MemAddr, Align, Pred, Reg0, Chain };
1108    std::vector<EVT> ResTys(2 * NumVecs, RegVT);
1109    ResTys.push_back(MVT::Other);
1110    SDNode *VLd = CurDAG->getMachineNode(Opc, dl, ResTys, Ops, 5);
1111    Chain = SDValue(VLd, 2 * NumVecs);
1112
1113    // Combine the even and odd subregs to produce the result.
1114    if (llvm::ModelWithRegSequence()) {
1115      if (NumVecs == 1) {
1116        SDNode *Q = PairDRegs(VT, SDValue(VLd, 0), SDValue(VLd, 1));
1117        ReplaceUses(SDValue(N, 0), SDValue(Q, 0));
1118      } else {
1119        SDValue QQ = SDValue(QuadDRegs(MVT::v4i64,
1120                                       SDValue(VLd, 0), SDValue(VLd, 1),
1121                                       SDValue(VLd, 2), SDValue(VLd, 3)), 0);
1122        SDValue Q0 = CurDAG->getTargetExtractSubreg(ARM::QSUBREG_0, dl, VT, QQ);
1123        SDValue Q1 = CurDAG->getTargetExtractSubreg(ARM::QSUBREG_1, dl, VT, QQ);
1124        ReplaceUses(SDValue(N, 0), Q0);
1125        ReplaceUses(SDValue(N, 1), Q1);
1126      }
1127    } else {
1128      for (unsigned Vec = 0; Vec < NumVecs; ++Vec) {
1129        SDNode *Q = PairDRegs(VT, SDValue(VLd, 2*Vec), SDValue(VLd, 2*Vec+1));
1130        ReplaceUses(SDValue(N, Vec), SDValue(Q, 0));
1131      }
1132    }
1133  } else {
1134    // Otherwise, quad registers are loaded with two separate instructions,
1135    // where one loads the even registers and the other loads the odd registers.
1136
1137    std::vector<EVT> ResTys(NumVecs, RegVT);
1138    ResTys.push_back(MemAddr.getValueType());
1139    ResTys.push_back(MVT::Other);
1140
1141    // Load the even subregs.
1142    unsigned Opc = QOpcodes0[OpcodeIndex];
1143    const SDValue OpsA[] = { MemAddr, Align, Reg0, Pred, Reg0, Chain };
1144    SDNode *VLdA = CurDAG->getMachineNode(Opc, dl, ResTys, OpsA, 6);
1145    Chain = SDValue(VLdA, NumVecs+1);
1146
1147    // Load the odd subregs.
1148    Opc = QOpcodes1[OpcodeIndex];
1149    const SDValue OpsB[] = { SDValue(VLdA, NumVecs),
1150                             Align, Reg0, Pred, Reg0, Chain };
1151    SDNode *VLdB = CurDAG->getMachineNode(Opc, dl, ResTys, OpsB, 6);
1152    Chain = SDValue(VLdB, NumVecs+1);
1153
1154    if (llvm::ModelWithRegSequence()) {
1155      SDValue V0 = SDValue(VLdA, 0);
1156      SDValue V1 = SDValue(VLdB, 0);
1157      SDValue V2 = SDValue(VLdA, 1);
1158      SDValue V3 = SDValue(VLdB, 1);
1159      SDValue V4 = SDValue(VLdA, 2);
1160      SDValue V5 = SDValue(VLdB, 2);
1161      SDValue V6 = (NumVecs == 3)
1162          ? SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF,dl,RegVT),
1163                    0)
1164          : SDValue(VLdA, 3);
1165      SDValue V7 = (NumVecs == 3)
1166          ? SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF,dl,RegVT),
1167                    0)
1168          : SDValue(VLdB, 3);
1169      SDValue RegSeq = SDValue(OctoDRegs(MVT::v8i64, V0, V1, V2, V3,
1170                                         V4, V5, V6, V7), 0);
1171
1172      // Extract out the 3 / 4 Q registers.
1173      for (unsigned Vec = 0; Vec < NumVecs; ++Vec) {
1174        SDValue Q = CurDAG->getTargetExtractSubreg(ARM::QSUBREG_0+Vec,
1175                                                   dl, VT, RegSeq);
1176        ReplaceUses(SDValue(N, Vec), Q);
1177      }
1178    } else {
1179      // Combine the even and odd subregs to produce the result.
1180      for (unsigned Vec = 0; Vec < NumVecs; ++Vec) {
1181        SDNode *Q = PairDRegs(VT, SDValue(VLdA, Vec), SDValue(VLdB, Vec));
1182        ReplaceUses(SDValue(N, Vec), SDValue(Q, 0));
1183      }
1184    }
1185  }
1186  ReplaceUses(SDValue(N, NumVecs), Chain);
1187  return NULL;
1188}
1189
1190SDNode *ARMDAGToDAGISel::SelectVST(SDNode *N, unsigned NumVecs,
1191                                   unsigned *DOpcodes, unsigned *QOpcodes0,
1192                                   unsigned *QOpcodes1) {
1193  assert(NumVecs >=1 && NumVecs <= 4 && "VST NumVecs out-of-range");
1194  DebugLoc dl = N->getDebugLoc();
1195
1196  SDValue MemAddr, Align;
1197  if (!SelectAddrMode6(N, N->getOperand(2), MemAddr, Align))
1198    return NULL;
1199
1200  SDValue Chain = N->getOperand(0);
1201  EVT VT = N->getOperand(3).getValueType();
1202  bool is64BitVector = VT.is64BitVector();
1203
1204  unsigned OpcodeIndex;
1205  switch (VT.getSimpleVT().SimpleTy) {
1206  default: llvm_unreachable("unhandled vst type");
1207    // Double-register operations:
1208  case MVT::v8i8:  OpcodeIndex = 0; break;
1209  case MVT::v4i16: OpcodeIndex = 1; break;
1210  case MVT::v2f32:
1211  case MVT::v2i32: OpcodeIndex = 2; break;
1212  case MVT::v1i64: OpcodeIndex = 3; break;
1213    // Quad-register operations:
1214  case MVT::v16i8: OpcodeIndex = 0; break;
1215  case MVT::v8i16: OpcodeIndex = 1; break;
1216  case MVT::v4f32:
1217  case MVT::v4i32: OpcodeIndex = 2; break;
1218  case MVT::v2i64: OpcodeIndex = 3;
1219    assert(NumVecs == 1 && "v2i64 type only supported for VST1");
1220    break;
1221  }
1222
1223  SDValue Pred = getAL(CurDAG);
1224  SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
1225
1226  SmallVector<SDValue, 10> Ops;
1227  Ops.push_back(MemAddr);
1228  Ops.push_back(Align);
1229
1230  if (is64BitVector) {
1231    if (llvm::ModelWithRegSequence() && NumVecs >= 2) {
1232      assert(NumVecs <= 4);
1233      SDValue RegSeq;
1234      SDValue V0 = N->getOperand(0+3);
1235      SDValue V1 = N->getOperand(1+3);
1236
1237      // Form a REG_SEQUENCE to force register allocation.
1238      if (NumVecs == 2)
1239        RegSeq = SDValue(PairDRegs(MVT::v2i64, V0, V1), 0);
1240      else {
1241        SDValue V2 = N->getOperand(2+3);
1242        // If it's a vld3, form a quad D-register and leave the last part as
1243        // an undef.
1244        SDValue V3 = (NumVecs == 3)
1245          ? SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF,dl,VT), 0)
1246          : N->getOperand(3+3);
1247        RegSeq = SDValue(QuadDRegs(MVT::v4i64, V0, V1, V2, V3), 0);
1248      }
1249
1250      // Now extract the D registers back out.
1251      Ops.push_back(CurDAG->getTargetExtractSubreg(ARM::DSUBREG_0, dl, VT,
1252                                                   RegSeq));
1253      Ops.push_back(CurDAG->getTargetExtractSubreg(ARM::DSUBREG_1, dl, VT,
1254                                                   RegSeq));
1255      if (NumVecs > 2)
1256        Ops.push_back(CurDAG->getTargetExtractSubreg(ARM::DSUBREG_2, dl, VT,
1257                                                     RegSeq));
1258      if (NumVecs > 3)
1259        Ops.push_back(CurDAG->getTargetExtractSubreg(ARM::DSUBREG_3, dl, VT,
1260                                                     RegSeq));
1261    } else {
1262      for (unsigned Vec = 0; Vec < NumVecs; ++Vec)
1263        Ops.push_back(N->getOperand(Vec+3));
1264    }
1265    Ops.push_back(Pred);
1266    Ops.push_back(Reg0); // predicate register
1267    Ops.push_back(Chain);
1268    unsigned Opc = DOpcodes[OpcodeIndex];
1269    return CurDAG->getMachineNode(Opc, dl, MVT::Other, Ops.data(), NumVecs+5);
1270  }
1271
1272  EVT RegVT = GetNEONSubregVT(VT);
1273  if (NumVecs <= 2) {
1274    // Quad registers are directly supported for VST1 and VST2,
1275    // storing pairs of D regs.
1276    unsigned Opc = QOpcodes0[OpcodeIndex];
1277    if (llvm::ModelWithRegSequence() && NumVecs == 2) {
1278      // First extract the pair of Q registers.
1279      SDValue Q0 = N->getOperand(3);
1280      SDValue Q1 = N->getOperand(4);
1281
1282      // Form a QQ register.
1283      SDValue QQ = SDValue(PairQRegs(MVT::v4i64, Q0, Q1), 0);
1284
1285      // Now extract the D registers back out.
1286      Ops.push_back(CurDAG->getTargetExtractSubreg(ARM::DSUBREG_0, dl, RegVT,
1287                                                   QQ));
1288      Ops.push_back(CurDAG->getTargetExtractSubreg(ARM::DSUBREG_1, dl, RegVT,
1289                                                   QQ));
1290      Ops.push_back(CurDAG->getTargetExtractSubreg(ARM::DSUBREG_2, dl, RegVT,
1291                                                   QQ));
1292      Ops.push_back(CurDAG->getTargetExtractSubreg(ARM::DSUBREG_3, dl, RegVT,
1293                                                   QQ));
1294      Ops.push_back(Pred);
1295      Ops.push_back(Reg0); // predicate register
1296      Ops.push_back(Chain);
1297      return CurDAG->getMachineNode(Opc, dl, MVT::Other, Ops.data(), 5 + 4);
1298    } else {
1299      for (unsigned Vec = 0; Vec < NumVecs; ++Vec) {
1300        Ops.push_back(CurDAG->getTargetExtractSubreg(ARM::DSUBREG_0, dl, RegVT,
1301                                                     N->getOperand(Vec+3)));
1302        Ops.push_back(CurDAG->getTargetExtractSubreg(ARM::DSUBREG_1, dl, RegVT,
1303                                                     N->getOperand(Vec+3)));
1304      }
1305      Ops.push_back(Pred);
1306      Ops.push_back(Reg0); // predicate register
1307      Ops.push_back(Chain);
1308      return CurDAG->getMachineNode(Opc, dl, MVT::Other, Ops.data(),
1309                                    5 + 2 * NumVecs);
1310    }
1311  }
1312
1313  // Otherwise, quad registers are stored with two separate instructions,
1314  // where one stores the even registers and the other stores the odd registers.
1315  if (llvm::ModelWithRegSequence()) {
1316    assert(NumVecs <= 4);
1317
1318    // Form the QQQQ REG_SEQUENCE.
1319    SDValue V[8];
1320    for (unsigned Vec = 0, i = 0; Vec < NumVecs; ++Vec, i+=2) {
1321      V[i]   = CurDAG->getTargetExtractSubreg(ARM::DSUBREG_0, dl, RegVT,
1322                                              N->getOperand(Vec+3));
1323      V[i+1] = CurDAG->getTargetExtractSubreg(ARM::DSUBREG_1, dl, RegVT,
1324                                              N->getOperand(Vec+3));
1325    }
1326    if (NumVecs == 3)
1327      V[6] = V[7] =
1328        SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF,dl,RegVT), 0);
1329    SDValue RegSeq = SDValue(OctoDRegs(MVT::v8i64, V[0], V[1], V[2], V[3],
1330                                       V[4], V[5], V[6], V[7]), 0);
1331
1332    // Store the even D registers.
1333    Ops.push_back(Reg0); // post-access address offset
1334    for (unsigned Vec = 0; Vec < NumVecs; ++Vec)
1335      Ops.push_back(CurDAG->getTargetExtractSubreg(ARM::DSUBREG_0+Vec*2, dl,
1336                                                   RegVT, RegSeq));
1337    Ops.push_back(Pred);
1338    Ops.push_back(Reg0); // predicate register
1339    Ops.push_back(Chain);
1340    unsigned Opc = QOpcodes0[OpcodeIndex];
1341    SDNode *VStA = CurDAG->getMachineNode(Opc, dl, MemAddr.getValueType(),
1342                                          MVT::Other, Ops.data(), NumVecs+6);
1343    Chain = SDValue(VStA, 1);
1344
1345    // Store the odd D registers.
1346    Ops[0] = SDValue(VStA, 0); // MemAddr
1347    for (unsigned Vec = 0; Vec < NumVecs; ++Vec)
1348      Ops[Vec+3] = CurDAG->getTargetExtractSubreg(ARM::DSUBREG_1+Vec*2, dl,
1349                                                  RegVT, RegSeq);
1350    Ops[NumVecs+5] = Chain;
1351    Opc = QOpcodes1[OpcodeIndex];
1352    SDNode *VStB = CurDAG->getMachineNode(Opc, dl, MemAddr.getValueType(),
1353                                          MVT::Other, Ops.data(), NumVecs+6);
1354    Chain = SDValue(VStB, 1);
1355    ReplaceUses(SDValue(N, 0), Chain);
1356    return NULL;
1357  } else {
1358    Ops.push_back(Reg0); // post-access address offset
1359
1360    // Store the even subregs.
1361    for (unsigned Vec = 0; Vec < NumVecs; ++Vec)
1362      Ops.push_back(CurDAG->getTargetExtractSubreg(ARM::DSUBREG_0, dl, RegVT,
1363                                                   N->getOperand(Vec+3)));
1364    Ops.push_back(Pred);
1365    Ops.push_back(Reg0); // predicate register
1366    Ops.push_back(Chain);
1367    unsigned Opc = QOpcodes0[OpcodeIndex];
1368    SDNode *VStA = CurDAG->getMachineNode(Opc, dl, MemAddr.getValueType(),
1369                                          MVT::Other, Ops.data(), NumVecs+6);
1370    Chain = SDValue(VStA, 1);
1371
1372    // Store the odd subregs.
1373    Ops[0] = SDValue(VStA, 0); // MemAddr
1374    for (unsigned Vec = 0; Vec < NumVecs; ++Vec)
1375      Ops[Vec+3] = CurDAG->getTargetExtractSubreg(ARM::DSUBREG_1, dl, RegVT,
1376                                                  N->getOperand(Vec+3));
1377    Ops[NumVecs+5] = Chain;
1378    Opc = QOpcodes1[OpcodeIndex];
1379    SDNode *VStB = CurDAG->getMachineNode(Opc, dl, MemAddr.getValueType(),
1380                                          MVT::Other, Ops.data(), NumVecs+6);
1381    Chain = SDValue(VStB, 1);
1382    ReplaceUses(SDValue(N, 0), Chain);
1383    return NULL;
1384  }
1385}
1386
1387SDNode *ARMDAGToDAGISel::SelectVLDSTLane(SDNode *N, bool IsLoad,
1388                                         unsigned NumVecs, unsigned *DOpcodes,
1389                                         unsigned *QOpcodes0,
1390                                         unsigned *QOpcodes1) {
1391  assert(NumVecs >=2 && NumVecs <= 4 && "VLDSTLane NumVecs out-of-range");
1392  DebugLoc dl = N->getDebugLoc();
1393
1394  SDValue MemAddr, Align;
1395  if (!SelectAddrMode6(N, N->getOperand(2), MemAddr, Align))
1396    return NULL;
1397
1398  SDValue Chain = N->getOperand(0);
1399  unsigned Lane =
1400    cast<ConstantSDNode>(N->getOperand(NumVecs+3))->getZExtValue();
1401  EVT VT = IsLoad ? N->getValueType(0) : N->getOperand(3).getValueType();
1402  bool is64BitVector = VT.is64BitVector();
1403
1404  // Quad registers are handled by load/store of subregs. Find the subreg info.
1405  unsigned NumElts = 0;
1406  int SubregIdx = 0;
1407  EVT RegVT = VT;
1408  if (!is64BitVector) {
1409    RegVT = GetNEONSubregVT(VT);
1410    NumElts = RegVT.getVectorNumElements();
1411    SubregIdx = (Lane < NumElts) ? ARM::DSUBREG_0 : ARM::DSUBREG_1;
1412  }
1413
1414  unsigned OpcodeIndex;
1415  switch (VT.getSimpleVT().SimpleTy) {
1416  default: llvm_unreachable("unhandled vld/vst lane type");
1417    // Double-register operations:
1418  case MVT::v8i8:  OpcodeIndex = 0; break;
1419  case MVT::v4i16: OpcodeIndex = 1; break;
1420  case MVT::v2f32:
1421  case MVT::v2i32: OpcodeIndex = 2; break;
1422    // Quad-register operations:
1423  case MVT::v8i16: OpcodeIndex = 0; break;
1424  case MVT::v4f32:
1425  case MVT::v4i32: OpcodeIndex = 1; break;
1426  }
1427
1428  SDValue Pred = getAL(CurDAG);
1429  SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
1430
1431  SmallVector<SDValue, 10> Ops;
1432  Ops.push_back(MemAddr);
1433  Ops.push_back(Align);
1434
1435  unsigned Opc = 0;
1436  if (is64BitVector) {
1437    Opc = DOpcodes[OpcodeIndex];
1438    for (unsigned Vec = 0; Vec < NumVecs; ++Vec)
1439      Ops.push_back(N->getOperand(Vec+3));
1440  } else {
1441    // Check if this is loading the even or odd subreg of a Q register.
1442    if (Lane < NumElts) {
1443      Opc = QOpcodes0[OpcodeIndex];
1444    } else {
1445      Lane -= NumElts;
1446      Opc = QOpcodes1[OpcodeIndex];
1447    }
1448    // Extract the subregs of the input vector.
1449    for (unsigned Vec = 0; Vec < NumVecs; ++Vec)
1450      Ops.push_back(CurDAG->getTargetExtractSubreg(SubregIdx, dl, RegVT,
1451                                                   N->getOperand(Vec+3)));
1452  }
1453  Ops.push_back(getI32Imm(Lane));
1454  Ops.push_back(Pred);
1455  Ops.push_back(Reg0);
1456  Ops.push_back(Chain);
1457
1458  if (!IsLoad)
1459    return CurDAG->getMachineNode(Opc, dl, MVT::Other, Ops.data(), NumVecs+6);
1460
1461  std::vector<EVT> ResTys(NumVecs, RegVT);
1462  ResTys.push_back(MVT::Other);
1463  SDNode *VLdLn =
1464    CurDAG->getMachineNode(Opc, dl, ResTys, Ops.data(), NumVecs+6);
1465  // For a 64-bit vector load to D registers, nothing more needs to be done.
1466  if (is64BitVector)
1467    return VLdLn;
1468
1469  // For 128-bit vectors, take the 64-bit results of the load and insert them
1470  // as subregs into the result.
1471  for (unsigned Vec = 0; Vec < NumVecs; ++Vec) {
1472    SDValue QuadVec = CurDAG->getTargetInsertSubreg(SubregIdx, dl, VT,
1473                                                    N->getOperand(Vec+3),
1474                                                    SDValue(VLdLn, Vec));
1475    ReplaceUses(SDValue(N, Vec), QuadVec);
1476  }
1477
1478  Chain = SDValue(VLdLn, NumVecs);
1479  ReplaceUses(SDValue(N, NumVecs), Chain);
1480  return NULL;
1481}
1482
1483SDNode *ARMDAGToDAGISel::SelectV6T2BitfieldExtractOp(SDNode *N,
1484                                                     bool isSigned) {
1485  if (!Subtarget->hasV6T2Ops())
1486    return NULL;
1487
1488  unsigned Opc = isSigned ? (Subtarget->isThumb() ? ARM::t2SBFX : ARM::SBFX)
1489    : (Subtarget->isThumb() ? ARM::t2UBFX : ARM::UBFX);
1490
1491
1492  // For unsigned extracts, check for a shift right and mask
1493  unsigned And_imm = 0;
1494  if (N->getOpcode() == ISD::AND) {
1495    if (isOpcWithIntImmediate(N, ISD::AND, And_imm)) {
1496
1497      // The immediate is a mask of the low bits iff imm & (imm+1) == 0
1498      if (And_imm & (And_imm + 1))
1499        return NULL;
1500
1501      unsigned Srl_imm = 0;
1502      if (isOpcWithIntImmediate(N->getOperand(0).getNode(), ISD::SRL,
1503                                Srl_imm)) {
1504        assert(Srl_imm > 0 && Srl_imm < 32 && "bad amount in shift node!");
1505
1506        unsigned Width = CountTrailingOnes_32(And_imm);
1507        unsigned LSB = Srl_imm;
1508        SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
1509        SDValue Ops[] = { N->getOperand(0).getOperand(0),
1510                          CurDAG->getTargetConstant(LSB, MVT::i32),
1511                          CurDAG->getTargetConstant(Width, MVT::i32),
1512          getAL(CurDAG), Reg0 };
1513        return CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops, 5);
1514      }
1515    }
1516    return NULL;
1517  }
1518
1519  // Otherwise, we're looking for a shift of a shift
1520  unsigned Shl_imm = 0;
1521  if (isOpcWithIntImmediate(N->getOperand(0).getNode(), ISD::SHL, Shl_imm)) {
1522    assert(Shl_imm > 0 && Shl_imm < 32 && "bad amount in shift node!");
1523    unsigned Srl_imm = 0;
1524    if (isInt32Immediate(N->getOperand(1), Srl_imm)) {
1525      assert(Srl_imm > 0 && Srl_imm < 32 && "bad amount in shift node!");
1526      unsigned Width = 32 - Srl_imm;
1527      int LSB = Srl_imm - Shl_imm;
1528      if (LSB < 0)
1529        return NULL;
1530      SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
1531      SDValue Ops[] = { N->getOperand(0).getOperand(0),
1532                        CurDAG->getTargetConstant(LSB, MVT::i32),
1533                        CurDAG->getTargetConstant(Width, MVT::i32),
1534                        getAL(CurDAG), Reg0 };
1535      return CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops, 5);
1536    }
1537  }
1538  return NULL;
1539}
1540
1541SDNode *ARMDAGToDAGISel::
1542SelectT2CMOVShiftOp(SDNode *N, SDValue FalseVal, SDValue TrueVal,
1543                    ARMCC::CondCodes CCVal, SDValue CCR, SDValue InFlag) {
1544  SDValue CPTmp0;
1545  SDValue CPTmp1;
1546  if (SelectT2ShifterOperandReg(N, TrueVal, CPTmp0, CPTmp1)) {
1547    unsigned SOVal = cast<ConstantSDNode>(CPTmp1)->getZExtValue();
1548    unsigned SOShOp = ARM_AM::getSORegShOp(SOVal);
1549    unsigned Opc = 0;
1550    switch (SOShOp) {
1551    case ARM_AM::lsl: Opc = ARM::t2MOVCClsl; break;
1552    case ARM_AM::lsr: Opc = ARM::t2MOVCClsr; break;
1553    case ARM_AM::asr: Opc = ARM::t2MOVCCasr; break;
1554    case ARM_AM::ror: Opc = ARM::t2MOVCCror; break;
1555    default:
1556      llvm_unreachable("Unknown so_reg opcode!");
1557      break;
1558    }
1559    SDValue SOShImm =
1560      CurDAG->getTargetConstant(ARM_AM::getSORegOffset(SOVal), MVT::i32);
1561    SDValue CC = CurDAG->getTargetConstant(CCVal, MVT::i32);
1562    SDValue Ops[] = { FalseVal, CPTmp0, SOShImm, CC, CCR, InFlag };
1563    return CurDAG->SelectNodeTo(N, Opc, MVT::i32,Ops, 6);
1564  }
1565  return 0;
1566}
1567
1568SDNode *ARMDAGToDAGISel::
1569SelectARMCMOVShiftOp(SDNode *N, SDValue FalseVal, SDValue TrueVal,
1570                     ARMCC::CondCodes CCVal, SDValue CCR, SDValue InFlag) {
1571  SDValue CPTmp0;
1572  SDValue CPTmp1;
1573  SDValue CPTmp2;
1574  if (SelectShifterOperandReg(N, TrueVal, CPTmp0, CPTmp1, CPTmp2)) {
1575    SDValue CC = CurDAG->getTargetConstant(CCVal, MVT::i32);
1576    SDValue Ops[] = { FalseVal, CPTmp0, CPTmp1, CPTmp2, CC, CCR, InFlag };
1577    return CurDAG->SelectNodeTo(N, ARM::MOVCCs, MVT::i32, Ops, 7);
1578  }
1579  return 0;
1580}
1581
1582SDNode *ARMDAGToDAGISel::
1583SelectT2CMOVSoImmOp(SDNode *N, SDValue FalseVal, SDValue TrueVal,
1584                    ARMCC::CondCodes CCVal, SDValue CCR, SDValue InFlag) {
1585  ConstantSDNode *T = dyn_cast<ConstantSDNode>(TrueVal);
1586  if (!T)
1587    return 0;
1588
1589  if (Predicate_t2_so_imm(TrueVal.getNode())) {
1590    SDValue True = CurDAG->getTargetConstant(T->getZExtValue(), MVT::i32);
1591    SDValue CC = CurDAG->getTargetConstant(CCVal, MVT::i32);
1592    SDValue Ops[] = { FalseVal, True, CC, CCR, InFlag };
1593    return CurDAG->SelectNodeTo(N,
1594                                ARM::t2MOVCCi, MVT::i32, Ops, 5);
1595  }
1596  return 0;
1597}
1598
1599SDNode *ARMDAGToDAGISel::
1600SelectARMCMOVSoImmOp(SDNode *N, SDValue FalseVal, SDValue TrueVal,
1601                     ARMCC::CondCodes CCVal, SDValue CCR, SDValue InFlag) {
1602  ConstantSDNode *T = dyn_cast<ConstantSDNode>(TrueVal);
1603  if (!T)
1604    return 0;
1605
1606  if (Predicate_so_imm(TrueVal.getNode())) {
1607    SDValue True = CurDAG->getTargetConstant(T->getZExtValue(), MVT::i32);
1608    SDValue CC = CurDAG->getTargetConstant(CCVal, MVT::i32);
1609    SDValue Ops[] = { FalseVal, True, CC, CCR, InFlag };
1610    return CurDAG->SelectNodeTo(N,
1611                                ARM::MOVCCi, MVT::i32, Ops, 5);
1612  }
1613  return 0;
1614}
1615
1616SDNode *ARMDAGToDAGISel::SelectCMOVOp(SDNode *N) {
1617  EVT VT = N->getValueType(0);
1618  SDValue FalseVal = N->getOperand(0);
1619  SDValue TrueVal  = N->getOperand(1);
1620  SDValue CC = N->getOperand(2);
1621  SDValue CCR = N->getOperand(3);
1622  SDValue InFlag = N->getOperand(4);
1623  assert(CC.getOpcode() == ISD::Constant);
1624  assert(CCR.getOpcode() == ISD::Register);
1625  ARMCC::CondCodes CCVal =
1626    (ARMCC::CondCodes)cast<ConstantSDNode>(CC)->getZExtValue();
1627
1628  if (!Subtarget->isThumb1Only() && VT == MVT::i32) {
1629    // Pattern: (ARMcmov:i32 GPR:i32:$false, so_reg:i32:$true, (imm:i32):$cc)
1630    // Emits: (MOVCCs:i32 GPR:i32:$false, so_reg:i32:$true, (imm:i32):$cc)
1631    // Pattern complexity = 18  cost = 1  size = 0
1632    SDValue CPTmp0;
1633    SDValue CPTmp1;
1634    SDValue CPTmp2;
1635    if (Subtarget->isThumb()) {
1636      SDNode *Res = SelectT2CMOVShiftOp(N, FalseVal, TrueVal,
1637                                        CCVal, CCR, InFlag);
1638      if (!Res)
1639        Res = SelectT2CMOVShiftOp(N, TrueVal, FalseVal,
1640                               ARMCC::getOppositeCondition(CCVal), CCR, InFlag);
1641      if (Res)
1642        return Res;
1643    } else {
1644      SDNode *Res = SelectARMCMOVShiftOp(N, FalseVal, TrueVal,
1645                                         CCVal, CCR, InFlag);
1646      if (!Res)
1647        Res = SelectARMCMOVShiftOp(N, TrueVal, FalseVal,
1648                               ARMCC::getOppositeCondition(CCVal), CCR, InFlag);
1649      if (Res)
1650        return Res;
1651    }
1652
1653    // Pattern: (ARMcmov:i32 GPR:i32:$false,
1654    //             (imm:i32)<<P:Predicate_so_imm>>:$true,
1655    //             (imm:i32):$cc)
1656    // Emits: (MOVCCi:i32 GPR:i32:$false,
1657    //           (so_imm:i32 (imm:i32):$true), (imm:i32):$cc)
1658    // Pattern complexity = 10  cost = 1  size = 0
1659    if (Subtarget->isThumb()) {
1660      SDNode *Res = SelectT2CMOVSoImmOp(N, FalseVal, TrueVal,
1661                                        CCVal, CCR, InFlag);
1662      if (!Res)
1663        Res = SelectT2CMOVSoImmOp(N, TrueVal, FalseVal,
1664                               ARMCC::getOppositeCondition(CCVal), CCR, InFlag);
1665      if (Res)
1666        return Res;
1667    } else {
1668      SDNode *Res = SelectARMCMOVSoImmOp(N, FalseVal, TrueVal,
1669                                         CCVal, CCR, InFlag);
1670      if (!Res)
1671        Res = SelectARMCMOVSoImmOp(N, TrueVal, FalseVal,
1672                               ARMCC::getOppositeCondition(CCVal), CCR, InFlag);
1673      if (Res)
1674        return Res;
1675    }
1676  }
1677
1678  // Pattern: (ARMcmov:i32 GPR:i32:$false, GPR:i32:$true, (imm:i32):$cc)
1679  // Emits: (MOVCCr:i32 GPR:i32:$false, GPR:i32:$true, (imm:i32):$cc)
1680  // Pattern complexity = 6  cost = 1  size = 0
1681  //
1682  // Pattern: (ARMcmov:i32 GPR:i32:$false, GPR:i32:$true, (imm:i32):$cc)
1683  // Emits: (tMOVCCr:i32 GPR:i32:$false, GPR:i32:$true, (imm:i32):$cc)
1684  // Pattern complexity = 6  cost = 11  size = 0
1685  //
1686  // Also FCPYScc and FCPYDcc.
1687  SDValue Tmp2 = CurDAG->getTargetConstant(CCVal, MVT::i32);
1688  SDValue Ops[] = { FalseVal, TrueVal, Tmp2, CCR, InFlag };
1689  unsigned Opc = 0;
1690  switch (VT.getSimpleVT().SimpleTy) {
1691  default: assert(false && "Illegal conditional move type!");
1692    break;
1693  case MVT::i32:
1694    Opc = Subtarget->isThumb()
1695      ? (Subtarget->hasThumb2() ? ARM::t2MOVCCr : ARM::tMOVCCr_pseudo)
1696      : ARM::MOVCCr;
1697    break;
1698  case MVT::f32:
1699    Opc = ARM::VMOVScc;
1700    break;
1701  case MVT::f64:
1702    Opc = ARM::VMOVDcc;
1703    break;
1704  }
1705  return CurDAG->SelectNodeTo(N, Opc, VT, Ops, 5);
1706}
1707
1708SDNode *ARMDAGToDAGISel::SelectConcatVector(SDNode *N) {
1709  // The only time a CONCAT_VECTORS operation can have legal types is when
1710  // two 64-bit vectors are concatenated to a 128-bit vector.
1711  EVT VT = N->getValueType(0);
1712  if (!VT.is128BitVector() || N->getNumOperands() != 2)
1713    llvm_unreachable("unexpected CONCAT_VECTORS");
1714  DebugLoc dl = N->getDebugLoc();
1715  SDValue V0 = N->getOperand(0);
1716  SDValue V1 = N->getOperand(1);
1717  SDValue SubReg0 = CurDAG->getTargetConstant(ARM::DSUBREG_0, MVT::i32);
1718  SDValue SubReg1 = CurDAG->getTargetConstant(ARM::DSUBREG_1, MVT::i32);
1719  const SDValue Ops[] = { V0, SubReg0, V1, SubReg1 };
1720  return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops, 4);
1721}
1722
1723SDNode *ARMDAGToDAGISel::Select(SDNode *N) {
1724  DebugLoc dl = N->getDebugLoc();
1725
1726  if (N->isMachineOpcode())
1727    return NULL;   // Already selected.
1728
1729  switch (N->getOpcode()) {
1730  default: break;
1731  case ISD::Constant: {
1732    unsigned Val = cast<ConstantSDNode>(N)->getZExtValue();
1733    bool UseCP = true;
1734    if (Subtarget->hasThumb2())
1735      // Thumb2-aware targets have the MOVT instruction, so all immediates can
1736      // be done with MOV + MOVT, at worst.
1737      UseCP = 0;
1738    else {
1739      if (Subtarget->isThumb()) {
1740        UseCP = (Val > 255 &&                          // MOV
1741                 ~Val > 255 &&                         // MOV + MVN
1742                 !ARM_AM::isThumbImmShiftedVal(Val));  // MOV + LSL
1743      } else
1744        UseCP = (ARM_AM::getSOImmVal(Val) == -1 &&     // MOV
1745                 ARM_AM::getSOImmVal(~Val) == -1 &&    // MVN
1746                 !ARM_AM::isSOImmTwoPartVal(Val));     // two instrs.
1747    }
1748
1749    if (UseCP) {
1750      SDValue CPIdx =
1751        CurDAG->getTargetConstantPool(ConstantInt::get(
1752                                  Type::getInt32Ty(*CurDAG->getContext()), Val),
1753                                      TLI.getPointerTy());
1754
1755      SDNode *ResNode;
1756      if (Subtarget->isThumb1Only()) {
1757        SDValue Pred = getAL(CurDAG);
1758        SDValue PredReg = CurDAG->getRegister(0, MVT::i32);
1759        SDValue Ops[] = { CPIdx, Pred, PredReg, CurDAG->getEntryNode() };
1760        ResNode = CurDAG->getMachineNode(ARM::tLDRcp, dl, MVT::i32, MVT::Other,
1761                                         Ops, 4);
1762      } else {
1763        SDValue Ops[] = {
1764          CPIdx,
1765          CurDAG->getRegister(0, MVT::i32),
1766          CurDAG->getTargetConstant(0, MVT::i32),
1767          getAL(CurDAG),
1768          CurDAG->getRegister(0, MVT::i32),
1769          CurDAG->getEntryNode()
1770        };
1771        ResNode=CurDAG->getMachineNode(ARM::LDRcp, dl, MVT::i32, MVT::Other,
1772                                       Ops, 6);
1773      }
1774      ReplaceUses(SDValue(N, 0), SDValue(ResNode, 0));
1775      return NULL;
1776    }
1777
1778    // Other cases are autogenerated.
1779    break;
1780  }
1781  case ISD::FrameIndex: {
1782    // Selects to ADDri FI, 0 which in turn will become ADDri SP, imm.
1783    int FI = cast<FrameIndexSDNode>(N)->getIndex();
1784    SDValue TFI = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
1785    if (Subtarget->isThumb1Only()) {
1786      return CurDAG->SelectNodeTo(N, ARM::tADDrSPi, MVT::i32, TFI,
1787                                  CurDAG->getTargetConstant(0, MVT::i32));
1788    } else {
1789      unsigned Opc = ((Subtarget->isThumb() && Subtarget->hasThumb2()) ?
1790                      ARM::t2ADDri : ARM::ADDri);
1791      SDValue Ops[] = { TFI, CurDAG->getTargetConstant(0, MVT::i32),
1792                        getAL(CurDAG), CurDAG->getRegister(0, MVT::i32),
1793                        CurDAG->getRegister(0, MVT::i32) };
1794      return CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops, 5);
1795    }
1796  }
1797  case ISD::SRL:
1798    if (SDNode *I = SelectV6T2BitfieldExtractOp(N, false))
1799      return I;
1800    break;
1801  case ISD::SRA:
1802    if (SDNode *I = SelectV6T2BitfieldExtractOp(N, true))
1803      return I;
1804    break;
1805  case ISD::MUL:
1806    if (Subtarget->isThumb1Only())
1807      break;
1808    if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(N->getOperand(1))) {
1809      unsigned RHSV = C->getZExtValue();
1810      if (!RHSV) break;
1811      if (isPowerOf2_32(RHSV-1)) {  // 2^n+1?
1812        unsigned ShImm = Log2_32(RHSV-1);
1813        if (ShImm >= 32)
1814          break;
1815        SDValue V = N->getOperand(0);
1816        ShImm = ARM_AM::getSORegOpc(ARM_AM::lsl, ShImm);
1817        SDValue ShImmOp = CurDAG->getTargetConstant(ShImm, MVT::i32);
1818        SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
1819        if (Subtarget->isThumb()) {
1820          SDValue Ops[] = { V, V, ShImmOp, getAL(CurDAG), Reg0, Reg0 };
1821          return CurDAG->SelectNodeTo(N, ARM::t2ADDrs, MVT::i32, Ops, 6);
1822        } else {
1823          SDValue Ops[] = { V, V, Reg0, ShImmOp, getAL(CurDAG), Reg0, Reg0 };
1824          return CurDAG->SelectNodeTo(N, ARM::ADDrs, MVT::i32, Ops, 7);
1825        }
1826      }
1827      if (isPowerOf2_32(RHSV+1)) {  // 2^n-1?
1828        unsigned ShImm = Log2_32(RHSV+1);
1829        if (ShImm >= 32)
1830          break;
1831        SDValue V = N->getOperand(0);
1832        ShImm = ARM_AM::getSORegOpc(ARM_AM::lsl, ShImm);
1833        SDValue ShImmOp = CurDAG->getTargetConstant(ShImm, MVT::i32);
1834        SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
1835        if (Subtarget->isThumb()) {
1836          SDValue Ops[] = { V, V, ShImmOp, getAL(CurDAG), Reg0 };
1837          return CurDAG->SelectNodeTo(N, ARM::t2RSBrs, MVT::i32, Ops, 5);
1838        } else {
1839          SDValue Ops[] = { V, V, Reg0, ShImmOp, getAL(CurDAG), Reg0, Reg0 };
1840          return CurDAG->SelectNodeTo(N, ARM::RSBrs, MVT::i32, Ops, 7);
1841        }
1842      }
1843    }
1844    break;
1845  case ISD::AND: {
1846    // Check for unsigned bitfield extract
1847    if (SDNode *I = SelectV6T2BitfieldExtractOp(N, false))
1848      return I;
1849
1850    // (and (or x, c2), c1) and top 16-bits of c1 and c2 match, lower 16-bits
1851    // of c1 are 0xffff, and lower 16-bit of c2 are 0. That is, the top 16-bits
1852    // are entirely contributed by c2 and lower 16-bits are entirely contributed
1853    // by x. That's equal to (or (and x, 0xffff), (and c1, 0xffff0000)).
1854    // Select it to: "movt x, ((c1 & 0xffff) >> 16)
1855    EVT VT = N->getValueType(0);
1856    if (VT != MVT::i32)
1857      break;
1858    unsigned Opc = (Subtarget->isThumb() && Subtarget->hasThumb2())
1859      ? ARM::t2MOVTi16
1860      : (Subtarget->hasV6T2Ops() ? ARM::MOVTi16 : 0);
1861    if (!Opc)
1862      break;
1863    SDValue N0 = N->getOperand(0), N1 = N->getOperand(1);
1864    ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
1865    if (!N1C)
1866      break;
1867    if (N0.getOpcode() == ISD::OR && N0.getNode()->hasOneUse()) {
1868      SDValue N2 = N0.getOperand(1);
1869      ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(N2);
1870      if (!N2C)
1871        break;
1872      unsigned N1CVal = N1C->getZExtValue();
1873      unsigned N2CVal = N2C->getZExtValue();
1874      if ((N1CVal & 0xffff0000U) == (N2CVal & 0xffff0000U) &&
1875          (N1CVal & 0xffffU) == 0xffffU &&
1876          (N2CVal & 0xffffU) == 0x0U) {
1877        SDValue Imm16 = CurDAG->getTargetConstant((N2CVal & 0xFFFF0000U) >> 16,
1878                                                  MVT::i32);
1879        SDValue Ops[] = { N0.getOperand(0), Imm16,
1880                          getAL(CurDAG), CurDAG->getRegister(0, MVT::i32) };
1881        return CurDAG->getMachineNode(Opc, dl, VT, Ops, 4);
1882      }
1883    }
1884    break;
1885  }
1886  case ARMISD::VMOVRRD:
1887    return CurDAG->getMachineNode(ARM::VMOVRRD, dl, MVT::i32, MVT::i32,
1888                                  N->getOperand(0), getAL(CurDAG),
1889                                  CurDAG->getRegister(0, MVT::i32));
1890  case ISD::UMUL_LOHI: {
1891    if (Subtarget->isThumb1Only())
1892      break;
1893    if (Subtarget->isThumb()) {
1894      SDValue Ops[] = { N->getOperand(0), N->getOperand(1),
1895                        getAL(CurDAG), CurDAG->getRegister(0, MVT::i32),
1896                        CurDAG->getRegister(0, MVT::i32) };
1897      return CurDAG->getMachineNode(ARM::t2UMULL, dl, MVT::i32, MVT::i32, Ops,4);
1898    } else {
1899      SDValue Ops[] = { N->getOperand(0), N->getOperand(1),
1900                        getAL(CurDAG), CurDAG->getRegister(0, MVT::i32),
1901                        CurDAG->getRegister(0, MVT::i32) };
1902      return CurDAG->getMachineNode(ARM::UMULL, dl, MVT::i32, MVT::i32, Ops, 5);
1903    }
1904  }
1905  case ISD::SMUL_LOHI: {
1906    if (Subtarget->isThumb1Only())
1907      break;
1908    if (Subtarget->isThumb()) {
1909      SDValue Ops[] = { N->getOperand(0), N->getOperand(1),
1910                        getAL(CurDAG), CurDAG->getRegister(0, MVT::i32) };
1911      return CurDAG->getMachineNode(ARM::t2SMULL, dl, MVT::i32, MVT::i32, Ops,4);
1912    } else {
1913      SDValue Ops[] = { N->getOperand(0), N->getOperand(1),
1914                        getAL(CurDAG), CurDAG->getRegister(0, MVT::i32),
1915                        CurDAG->getRegister(0, MVT::i32) };
1916      return CurDAG->getMachineNode(ARM::SMULL, dl, MVT::i32, MVT::i32, Ops, 5);
1917    }
1918  }
1919  case ISD::LOAD: {
1920    SDNode *ResNode = 0;
1921    if (Subtarget->isThumb() && Subtarget->hasThumb2())
1922      ResNode = SelectT2IndexedLoad(N);
1923    else
1924      ResNode = SelectARMIndexedLoad(N);
1925    if (ResNode)
1926      return ResNode;
1927
1928    // VLDMQ must be custom-selected for "v2f64 load" to set the AM5Opc value.
1929    if (Subtarget->hasVFP2() &&
1930        N->getValueType(0).getSimpleVT().SimpleTy == MVT::v2f64) {
1931      SDValue Chain = N->getOperand(0);
1932      SDValue AM5Opc =
1933        CurDAG->getTargetConstant(ARM_AM::getAM5Opc(ARM_AM::ia, 4), MVT::i32);
1934      SDValue Pred = getAL(CurDAG);
1935      SDValue PredReg = CurDAG->getRegister(0, MVT::i32);
1936      SDValue Ops[] = { N->getOperand(1), AM5Opc, Pred, PredReg, Chain };
1937      return CurDAG->getMachineNode(ARM::VLDMQ, dl, MVT::v2f64, MVT::Other,
1938                                    Ops, 5);
1939    }
1940    // Other cases are autogenerated.
1941    break;
1942  }
1943  case ISD::STORE: {
1944    // VSTMQ must be custom-selected for "v2f64 store" to set the AM5Opc value.
1945    if (Subtarget->hasVFP2() &&
1946        N->getOperand(1).getValueType().getSimpleVT().SimpleTy == MVT::v2f64) {
1947      SDValue Chain = N->getOperand(0);
1948      SDValue AM5Opc =
1949        CurDAG->getTargetConstant(ARM_AM::getAM5Opc(ARM_AM::ia, 4), MVT::i32);
1950      SDValue Pred = getAL(CurDAG);
1951      SDValue PredReg = CurDAG->getRegister(0, MVT::i32);
1952      SDValue Ops[] = { N->getOperand(1), N->getOperand(2),
1953                        AM5Opc, Pred, PredReg, Chain };
1954      return CurDAG->getMachineNode(ARM::VSTMQ, dl, MVT::Other, Ops, 6);
1955    }
1956    // Other cases are autogenerated.
1957    break;
1958  }
1959  case ARMISD::BRCOND: {
1960    // Pattern: (ARMbrcond:void (bb:Other):$dst, (imm:i32):$cc)
1961    // Emits: (Bcc:void (bb:Other):$dst, (imm:i32):$cc)
1962    // Pattern complexity = 6  cost = 1  size = 0
1963
1964    // Pattern: (ARMbrcond:void (bb:Other):$dst, (imm:i32):$cc)
1965    // Emits: (tBcc:void (bb:Other):$dst, (imm:i32):$cc)
1966    // Pattern complexity = 6  cost = 1  size = 0
1967
1968    // Pattern: (ARMbrcond:void (bb:Other):$dst, (imm:i32):$cc)
1969    // Emits: (t2Bcc:void (bb:Other):$dst, (imm:i32):$cc)
1970    // Pattern complexity = 6  cost = 1  size = 0
1971
1972    unsigned Opc = Subtarget->isThumb() ?
1973      ((Subtarget->hasThumb2()) ? ARM::t2Bcc : ARM::tBcc) : ARM::Bcc;
1974    SDValue Chain = N->getOperand(0);
1975    SDValue N1 = N->getOperand(1);
1976    SDValue N2 = N->getOperand(2);
1977    SDValue N3 = N->getOperand(3);
1978    SDValue InFlag = N->getOperand(4);
1979    assert(N1.getOpcode() == ISD::BasicBlock);
1980    assert(N2.getOpcode() == ISD::Constant);
1981    assert(N3.getOpcode() == ISD::Register);
1982
1983    SDValue Tmp2 = CurDAG->getTargetConstant(((unsigned)
1984                               cast<ConstantSDNode>(N2)->getZExtValue()),
1985                               MVT::i32);
1986    SDValue Ops[] = { N1, Tmp2, N3, Chain, InFlag };
1987    SDNode *ResNode = CurDAG->getMachineNode(Opc, dl, MVT::Other,
1988                                             MVT::Flag, Ops, 5);
1989    Chain = SDValue(ResNode, 0);
1990    if (N->getNumValues() == 2) {
1991      InFlag = SDValue(ResNode, 1);
1992      ReplaceUses(SDValue(N, 1), InFlag);
1993    }
1994    ReplaceUses(SDValue(N, 0),
1995                SDValue(Chain.getNode(), Chain.getResNo()));
1996    return NULL;
1997  }
1998  case ARMISD::CMOV:
1999    return SelectCMOVOp(N);
2000  case ARMISD::CNEG: {
2001    EVT VT = N->getValueType(0);
2002    SDValue N0 = N->getOperand(0);
2003    SDValue N1 = N->getOperand(1);
2004    SDValue N2 = N->getOperand(2);
2005    SDValue N3 = N->getOperand(3);
2006    SDValue InFlag = N->getOperand(4);
2007    assert(N2.getOpcode() == ISD::Constant);
2008    assert(N3.getOpcode() == ISD::Register);
2009
2010    SDValue Tmp2 = CurDAG->getTargetConstant(((unsigned)
2011                               cast<ConstantSDNode>(N2)->getZExtValue()),
2012                               MVT::i32);
2013    SDValue Ops[] = { N0, N1, Tmp2, N3, InFlag };
2014    unsigned Opc = 0;
2015    switch (VT.getSimpleVT().SimpleTy) {
2016    default: assert(false && "Illegal conditional move type!");
2017      break;
2018    case MVT::f32:
2019      Opc = ARM::VNEGScc;
2020      break;
2021    case MVT::f64:
2022      Opc = ARM::VNEGDcc;
2023      break;
2024    }
2025    return CurDAG->SelectNodeTo(N, Opc, VT, Ops, 5);
2026  }
2027
2028  case ARMISD::VZIP: {
2029    unsigned Opc = 0;
2030    EVT VT = N->getValueType(0);
2031    switch (VT.getSimpleVT().SimpleTy) {
2032    default: return NULL;
2033    case MVT::v8i8:  Opc = ARM::VZIPd8; break;
2034    case MVT::v4i16: Opc = ARM::VZIPd16; break;
2035    case MVT::v2f32:
2036    case MVT::v2i32: Opc = ARM::VZIPd32; break;
2037    case MVT::v16i8: Opc = ARM::VZIPq8; break;
2038    case MVT::v8i16: Opc = ARM::VZIPq16; break;
2039    case MVT::v4f32:
2040    case MVT::v4i32: Opc = ARM::VZIPq32; break;
2041    }
2042    SDValue Pred = getAL(CurDAG);
2043    SDValue PredReg = CurDAG->getRegister(0, MVT::i32);
2044    SDValue Ops[] = { N->getOperand(0), N->getOperand(1), Pred, PredReg };
2045    return CurDAG->getMachineNode(Opc, dl, VT, VT, Ops, 4);
2046  }
2047  case ARMISD::VUZP: {
2048    unsigned Opc = 0;
2049    EVT VT = N->getValueType(0);
2050    switch (VT.getSimpleVT().SimpleTy) {
2051    default: return NULL;
2052    case MVT::v8i8:  Opc = ARM::VUZPd8; break;
2053    case MVT::v4i16: Opc = ARM::VUZPd16; break;
2054    case MVT::v2f32:
2055    case MVT::v2i32: Opc = ARM::VUZPd32; break;
2056    case MVT::v16i8: Opc = ARM::VUZPq8; break;
2057    case MVT::v8i16: Opc = ARM::VUZPq16; break;
2058    case MVT::v4f32:
2059    case MVT::v4i32: Opc = ARM::VUZPq32; break;
2060    }
2061    SDValue Pred = getAL(CurDAG);
2062    SDValue PredReg = CurDAG->getRegister(0, MVT::i32);
2063    SDValue Ops[] = { N->getOperand(0), N->getOperand(1), Pred, PredReg };
2064    return CurDAG->getMachineNode(Opc, dl, VT, VT, Ops, 4);
2065  }
2066  case ARMISD::VTRN: {
2067    unsigned Opc = 0;
2068    EVT VT = N->getValueType(0);
2069    switch (VT.getSimpleVT().SimpleTy) {
2070    default: return NULL;
2071    case MVT::v8i8:  Opc = ARM::VTRNd8; break;
2072    case MVT::v4i16: Opc = ARM::VTRNd16; break;
2073    case MVT::v2f32:
2074    case MVT::v2i32: Opc = ARM::VTRNd32; break;
2075    case MVT::v16i8: Opc = ARM::VTRNq8; break;
2076    case MVT::v8i16: Opc = ARM::VTRNq16; break;
2077    case MVT::v4f32:
2078    case MVT::v4i32: Opc = ARM::VTRNq32; break;
2079    }
2080    SDValue Pred = getAL(CurDAG);
2081    SDValue PredReg = CurDAG->getRegister(0, MVT::i32);
2082    SDValue Ops[] = { N->getOperand(0), N->getOperand(1), Pred, PredReg };
2083    return CurDAG->getMachineNode(Opc, dl, VT, VT, Ops, 4);
2084  }
2085
2086  case ISD::INTRINSIC_VOID:
2087  case ISD::INTRINSIC_W_CHAIN: {
2088    unsigned IntNo = cast<ConstantSDNode>(N->getOperand(1))->getZExtValue();
2089    switch (IntNo) {
2090    default:
2091      break;
2092
2093    case Intrinsic::arm_neon_vld1: {
2094      unsigned DOpcodes[] = { ARM::VLD1d8, ARM::VLD1d16,
2095                              ARM::VLD1d32, ARM::VLD1d64 };
2096      unsigned QOpcodes[] = { ARM::VLD1q8, ARM::VLD1q16,
2097                              ARM::VLD1q32, ARM::VLD1q64 };
2098      return SelectVLD(N, 1, DOpcodes, QOpcodes, 0);
2099    }
2100
2101    case Intrinsic::arm_neon_vld2: {
2102      unsigned DOpcodes[] = { ARM::VLD2d8, ARM::VLD2d16,
2103                              ARM::VLD2d32, ARM::VLD1q64 };
2104      unsigned QOpcodes[] = { ARM::VLD2q8, ARM::VLD2q16, ARM::VLD2q32 };
2105      return SelectVLD(N, 2, DOpcodes, QOpcodes, 0);
2106    }
2107
2108    case Intrinsic::arm_neon_vld3: {
2109      unsigned DOpcodes[] = { ARM::VLD3d8, ARM::VLD3d16,
2110                              ARM::VLD3d32, ARM::VLD1d64T };
2111      unsigned QOpcodes0[] = { ARM::VLD3q8_UPD,
2112                               ARM::VLD3q16_UPD,
2113                               ARM::VLD3q32_UPD };
2114      unsigned QOpcodes1[] = { ARM::VLD3q8odd_UPD,
2115                               ARM::VLD3q16odd_UPD,
2116                               ARM::VLD3q32odd_UPD };
2117      return SelectVLD(N, 3, DOpcodes, QOpcodes0, QOpcodes1);
2118    }
2119
2120    case Intrinsic::arm_neon_vld4: {
2121      unsigned DOpcodes[] = { ARM::VLD4d8, ARM::VLD4d16,
2122                              ARM::VLD4d32, ARM::VLD1d64Q };
2123      unsigned QOpcodes0[] = { ARM::VLD4q8_UPD,
2124                               ARM::VLD4q16_UPD,
2125                               ARM::VLD4q32_UPD };
2126      unsigned QOpcodes1[] = { ARM::VLD4q8odd_UPD,
2127                               ARM::VLD4q16odd_UPD,
2128                               ARM::VLD4q32odd_UPD };
2129      return SelectVLD(N, 4, DOpcodes, QOpcodes0, QOpcodes1);
2130    }
2131
2132    case Intrinsic::arm_neon_vld2lane: {
2133      unsigned DOpcodes[] = { ARM::VLD2LNd8, ARM::VLD2LNd16, ARM::VLD2LNd32 };
2134      unsigned QOpcodes0[] = { ARM::VLD2LNq16, ARM::VLD2LNq32 };
2135      unsigned QOpcodes1[] = { ARM::VLD2LNq16odd, ARM::VLD2LNq32odd };
2136      return SelectVLDSTLane(N, true, 2, DOpcodes, QOpcodes0, QOpcodes1);
2137    }
2138
2139    case Intrinsic::arm_neon_vld3lane: {
2140      unsigned DOpcodes[] = { ARM::VLD3LNd8, ARM::VLD3LNd16, ARM::VLD3LNd32 };
2141      unsigned QOpcodes0[] = { ARM::VLD3LNq16, ARM::VLD3LNq32 };
2142      unsigned QOpcodes1[] = { ARM::VLD3LNq16odd, ARM::VLD3LNq32odd };
2143      return SelectVLDSTLane(N, true, 3, DOpcodes, QOpcodes0, QOpcodes1);
2144    }
2145
2146    case Intrinsic::arm_neon_vld4lane: {
2147      unsigned DOpcodes[] = { ARM::VLD4LNd8, ARM::VLD4LNd16, ARM::VLD4LNd32 };
2148      unsigned QOpcodes0[] = { ARM::VLD4LNq16, ARM::VLD4LNq32 };
2149      unsigned QOpcodes1[] = { ARM::VLD4LNq16odd, ARM::VLD4LNq32odd };
2150      return SelectVLDSTLane(N, true, 4, DOpcodes, QOpcodes0, QOpcodes1);
2151    }
2152
2153    case Intrinsic::arm_neon_vst1: {
2154      unsigned DOpcodes[] = { ARM::VST1d8, ARM::VST1d16,
2155                              ARM::VST1d32, ARM::VST1d64 };
2156      unsigned QOpcodes[] = { ARM::VST1q8, ARM::VST1q16,
2157                              ARM::VST1q32, ARM::VST1q64 };
2158      return SelectVST(N, 1, DOpcodes, QOpcodes, 0);
2159    }
2160
2161    case Intrinsic::arm_neon_vst2: {
2162      unsigned DOpcodes[] = { ARM::VST2d8, ARM::VST2d16,
2163                              ARM::VST2d32, ARM::VST1q64 };
2164      unsigned QOpcodes[] = { ARM::VST2q8, ARM::VST2q16, ARM::VST2q32 };
2165      return SelectVST(N, 2, DOpcodes, QOpcodes, 0);
2166    }
2167
2168    case Intrinsic::arm_neon_vst3: {
2169      unsigned DOpcodes[] = { ARM::VST3d8, ARM::VST3d16,
2170                              ARM::VST3d32, ARM::VST1d64T };
2171      unsigned QOpcodes0[] = { ARM::VST3q8_UPD,
2172                               ARM::VST3q16_UPD,
2173                               ARM::VST3q32_UPD };
2174      unsigned QOpcodes1[] = { ARM::VST3q8odd_UPD,
2175                               ARM::VST3q16odd_UPD,
2176                               ARM::VST3q32odd_UPD };
2177      return SelectVST(N, 3, DOpcodes, QOpcodes0, QOpcodes1);
2178    }
2179
2180    case Intrinsic::arm_neon_vst4: {
2181      unsigned DOpcodes[] = { ARM::VST4d8, ARM::VST4d16,
2182                              ARM::VST4d32, ARM::VST1d64Q };
2183      unsigned QOpcodes0[] = { ARM::VST4q8_UPD,
2184                               ARM::VST4q16_UPD,
2185                               ARM::VST4q32_UPD };
2186      unsigned QOpcodes1[] = { ARM::VST4q8odd_UPD,
2187                               ARM::VST4q16odd_UPD,
2188                               ARM::VST4q32odd_UPD };
2189      return SelectVST(N, 4, DOpcodes, QOpcodes0, QOpcodes1);
2190    }
2191
2192    case Intrinsic::arm_neon_vst2lane: {
2193      unsigned DOpcodes[] = { ARM::VST2LNd8, ARM::VST2LNd16, ARM::VST2LNd32 };
2194      unsigned QOpcodes0[] = { ARM::VST2LNq16, ARM::VST2LNq32 };
2195      unsigned QOpcodes1[] = { ARM::VST2LNq16odd, ARM::VST2LNq32odd };
2196      return SelectVLDSTLane(N, false, 2, DOpcodes, QOpcodes0, QOpcodes1);
2197    }
2198
2199    case Intrinsic::arm_neon_vst3lane: {
2200      unsigned DOpcodes[] = { ARM::VST3LNd8, ARM::VST3LNd16, ARM::VST3LNd32 };
2201      unsigned QOpcodes0[] = { ARM::VST3LNq16, ARM::VST3LNq32 };
2202      unsigned QOpcodes1[] = { ARM::VST3LNq16odd, ARM::VST3LNq32odd };
2203      return SelectVLDSTLane(N, false, 3, DOpcodes, QOpcodes0, QOpcodes1);
2204    }
2205
2206    case Intrinsic::arm_neon_vst4lane: {
2207      unsigned DOpcodes[] = { ARM::VST4LNd8, ARM::VST4LNd16, ARM::VST4LNd32 };
2208      unsigned QOpcodes0[] = { ARM::VST4LNq16, ARM::VST4LNq32 };
2209      unsigned QOpcodes1[] = { ARM::VST4LNq16odd, ARM::VST4LNq32odd };
2210      return SelectVLDSTLane(N, false, 4, DOpcodes, QOpcodes0, QOpcodes1);
2211    }
2212    }
2213    break;
2214  }
2215
2216  case ISD::CONCAT_VECTORS:
2217    return SelectConcatVector(N);
2218  }
2219
2220  return SelectCode(N);
2221}
2222
2223bool ARMDAGToDAGISel::
2224SelectInlineAsmMemoryOperand(const SDValue &Op, char ConstraintCode,
2225                             std::vector<SDValue> &OutOps) {
2226  assert(ConstraintCode == 'm' && "unexpected asm memory constraint");
2227  // Require the address to be in a register.  That is safe for all ARM
2228  // variants and it is hard to do anything much smarter without knowing
2229  // how the operand is used.
2230  OutOps.push_back(Op);
2231  return false;
2232}
2233
2234/// createARMISelDag - This pass converts a legalized DAG into a
2235/// ARM-specific DAG, ready for instruction scheduling.
2236///
2237FunctionPass *llvm::createARMISelDag(ARMBaseTargetMachine &TM,
2238                                     CodeGenOpt::Level OptLevel) {
2239  return new ARMDAGToDAGISel(TM, OptLevel);
2240}
2241
2242/// ModelWithRegSequence - Return true if isel should use REG_SEQUENCE to model
2243/// operations involving sub-registers.
2244bool llvm::ModelWithRegSequence() {
2245  return UseRegSeq;
2246}
2247