ARMISelDAGToDAG.cpp revision 7f687195175f01d820eea70e8a647a61d5b99fce 
1//===-- ARMISelDAGToDAG.cpp - A dag to dag inst selector for ARM ----------===//
2//
3//                     The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10// This file defines an instruction selector for the ARM target.
11//
12//===----------------------------------------------------------------------===//
13
14#include "ARM.h"
15#include "ARMAddressingModes.h"
16#include "ARMTargetMachine.h"
17#include "llvm/CallingConv.h"
18#include "llvm/Constants.h"
19#include "llvm/DerivedTypes.h"
20#include "llvm/Function.h"
21#include "llvm/Intrinsics.h"
22#include "llvm/LLVMContext.h"
23#include "llvm/CodeGen/MachineFrameInfo.h"
24#include "llvm/CodeGen/MachineFunction.h"
25#include "llvm/CodeGen/MachineInstrBuilder.h"
26#include "llvm/CodeGen/SelectionDAG.h"
27#include "llvm/CodeGen/SelectionDAGISel.h"
28#include "llvm/Target/TargetLowering.h"
29#include "llvm/Target/TargetOptions.h"
30#include "llvm/Support/CommandLine.h"
31#include "llvm/Support/Compiler.h"
32#include "llvm/Support/Debug.h"
33#include "llvm/Support/ErrorHandling.h"
34#include "llvm/Support/raw_ostream.h"
35
36using namespace llvm;
37
38static cl::opt<bool>
39UseRegSeq("neon-reg-sequence", cl::Hidden,
40          cl::desc("Use reg_sequence to model ld / st of multiple neon regs"));
41
42//===--------------------------------------------------------------------===//
43/// ARMDAGToDAGISel - ARM specific code to select ARM machine
44/// instructions for SelectionDAG operations.
45///
46namespace {
47class ARMDAGToDAGISel : public SelectionDAGISel {
48  ARMBaseTargetMachine &TM;
49
50  /// Subtarget - Keep a pointer to the ARMSubtarget around so that we can
51  /// make the right decision when generating code for different targets.
52  const ARMSubtarget *Subtarget;
53
54public:
55  explicit ARMDAGToDAGISel(ARMBaseTargetMachine &tm,
56                           CodeGenOpt::Level OptLevel)
57    : SelectionDAGISel(tm, OptLevel), TM(tm),
58    Subtarget(&TM.getSubtarget<ARMSubtarget>()) {
59  }
60
61  virtual const char *getPassName() const {
62    return "ARM Instruction Selection";
63  }
64
65  /// getI32Imm - Return a target constant of type i32 with the specified
66  /// value.
67  inline SDValue getI32Imm(unsigned Imm) {
68    return CurDAG->getTargetConstant(Imm, MVT::i32);
69  }
70
71  SDNode *Select(SDNode *N);
72
73  bool SelectShifterOperandReg(SDNode *Op, SDValue N, SDValue &A,
74                               SDValue &B, SDValue &C);
75  bool SelectAddrMode2(SDNode *Op, SDValue N, SDValue &Base,
76                       SDValue &Offset, SDValue &Opc);
77  bool SelectAddrMode2Offset(SDNode *Op, SDValue N,
78                             SDValue &Offset, SDValue &Opc);
79  bool SelectAddrMode3(SDNode *Op, SDValue N, SDValue &Base,
80                       SDValue &Offset, SDValue &Opc);
81  bool SelectAddrMode3Offset(SDNode *Op, SDValue N,
82                             SDValue &Offset, SDValue &Opc);
83  bool SelectAddrMode4(SDNode *Op, SDValue N, SDValue &Addr,
84                       SDValue &Mode);
85  bool SelectAddrMode5(SDNode *Op, SDValue N, SDValue &Base,
86                       SDValue &Offset);
87  bool SelectAddrMode6(SDNode *Op, SDValue N, SDValue &Addr, SDValue &Align);
88
89  bool SelectAddrModePC(SDNode *Op, SDValue N, SDValue &Offset,
90                        SDValue &Label);
91
92  bool SelectThumbAddrModeRR(SDNode *Op, SDValue N, SDValue &Base,
93                             SDValue &Offset);
94  bool SelectThumbAddrModeRI5(SDNode *Op, SDValue N, unsigned Scale,
95                              SDValue &Base, SDValue &OffImm,
96                              SDValue &Offset);
97  bool SelectThumbAddrModeS1(SDNode *Op, SDValue N, SDValue &Base,
98                             SDValue &OffImm, SDValue &Offset);
99  bool SelectThumbAddrModeS2(SDNode *Op, SDValue N, SDValue &Base,
100                             SDValue &OffImm, SDValue &Offset);
101  bool SelectThumbAddrModeS4(SDNode *Op, SDValue N, SDValue &Base,
102                             SDValue &OffImm, SDValue &Offset);
103  bool SelectThumbAddrModeSP(SDNode *Op, SDValue N, SDValue &Base,
104                             SDValue &OffImm);
105
106  bool SelectT2ShifterOperandReg(SDNode *Op, SDValue N,
107                                 SDValue &BaseReg, SDValue &Opc);
108  bool SelectT2AddrModeImm12(SDNode *Op, SDValue N, SDValue &Base,
109                             SDValue &OffImm);
110  bool SelectT2AddrModeImm8(SDNode *Op, SDValue N, SDValue &Base,
111                            SDValue &OffImm);
112  bool SelectT2AddrModeImm8Offset(SDNode *Op, SDValue N,
113                                 SDValue &OffImm);
114  bool SelectT2AddrModeImm8s4(SDNode *Op, SDValue N, SDValue &Base,
115                              SDValue &OffImm);
116  bool SelectT2AddrModeSoReg(SDNode *Op, SDValue N, SDValue &Base,
117                             SDValue &OffReg, SDValue &ShImm);
118
119  // Include the pieces autogenerated from the target description.
120#include "ARMGenDAGISel.inc"
121
122private:
123  /// SelectARMIndexedLoad - Indexed (pre/post inc/dec) load matching code for
124  /// ARM.
125  SDNode *SelectARMIndexedLoad(SDNode *N);
126  SDNode *SelectT2IndexedLoad(SDNode *N);
127
128  /// SelectVLD - Select NEON load intrinsics.  NumVecs should be
129  /// 1, 2, 3 or 4.  The opcode arrays specify the instructions used for
130  /// loads of D registers and even subregs and odd subregs of Q registers.
131  /// For NumVecs <= 2, QOpcodes1 is not used.
132  SDNode *SelectVLD(SDNode *N, unsigned NumVecs, unsigned *DOpcodes,
133                    unsigned *QOpcodes0, unsigned *QOpcodes1);
134
135  /// SelectVST - Select NEON store intrinsics.  NumVecs should
136  /// be 1, 2, 3 or 4.  The opcode arrays specify the instructions used for
137  /// stores of D registers and even subregs and odd subregs of Q registers.
138  /// For NumVecs <= 2, QOpcodes1 is not used.
139  SDNode *SelectVST(SDNode *N, unsigned NumVecs, unsigned *DOpcodes,
140                    unsigned *QOpcodes0, unsigned *QOpcodes1);
141
142  /// SelectVLDSTLane - Select NEON load/store lane intrinsics.  NumVecs should
143  /// be 2, 3 or 4.  The opcode arrays specify the instructions used for
144  /// load/store of D registers and even subregs and odd subregs of Q registers.
145  SDNode *SelectVLDSTLane(SDNode *N, bool IsLoad, unsigned NumVecs,
146                          unsigned *DOpcodes, unsigned *QOpcodes0,
147                          unsigned *QOpcodes1);
148
149  /// SelectV6T2BitfieldExtractOp - Select SBFX/UBFX instructions for ARM.
150  SDNode *SelectV6T2BitfieldExtractOp(SDNode *N, bool isSigned);
151
152  /// SelectCMOVOp - Select CMOV instructions for ARM.
153  SDNode *SelectCMOVOp(SDNode *N);
154  SDNode *SelectT2CMOVShiftOp(SDNode *N, SDValue FalseVal, SDValue TrueVal,
155                              ARMCC::CondCodes CCVal, SDValue CCR,
156                              SDValue InFlag);
157  SDNode *SelectARMCMOVShiftOp(SDNode *N, SDValue FalseVal, SDValue TrueVal,
158                               ARMCC::CondCodes CCVal, SDValue CCR,
159                               SDValue InFlag);
160  SDNode *SelectT2CMOVSoImmOp(SDNode *N, SDValue FalseVal, SDValue TrueVal,
161                              ARMCC::CondCodes CCVal, SDValue CCR,
162                              SDValue InFlag);
163  SDNode *SelectARMCMOVSoImmOp(SDNode *N, SDValue FalseVal, SDValue TrueVal,
164                               ARMCC::CondCodes CCVal, SDValue CCR,
165                               SDValue InFlag);
166
167  SDNode *SelectConcatVector(SDNode *N);
168
169  /// SelectInlineAsmMemoryOperand - Implement addressing mode selection for
170  /// inline asm expressions.
171  virtual bool SelectInlineAsmMemoryOperand(const SDValue &Op,
172                                            char ConstraintCode,
173                                            std::vector<SDValue> &OutOps);
174
175  /// PairDRegs - Form a quad register from a pair of D registers.
176  ///
177  SDNode *PairDRegs(EVT VT, SDValue V0, SDValue V1);
178
179  /// PairDRegs - Form a quad register pair from a pair of Q registers.
180  ///
181  SDNode *PairQRegs(EVT VT, SDValue V0, SDValue V1);
182
183  /// QuadDRegs - Form a quad register pair from a quad of D registers.
184  ///
185  SDNode *QuadDRegs(EVT VT, SDValue V0, SDValue V1, SDValue V2, SDValue V3);
186};
187}
188
189/// isInt32Immediate - This method tests to see if the node is a 32-bit constant
190/// operand. If so Imm will receive the 32-bit value.
191static bool isInt32Immediate(SDNode *N, unsigned &Imm) {
192  if (N->getOpcode() == ISD::Constant && N->getValueType(0) == MVT::i32) {
193    Imm = cast<ConstantSDNode>(N)->getZExtValue();
194    return true;
195  }
196  return false;
197}
198
199// isInt32Immediate - This method tests to see if a constant operand.
200// If so Imm will receive the 32 bit value.
201static bool isInt32Immediate(SDValue N, unsigned &Imm) {
202  return isInt32Immediate(N.getNode(), Imm);
203}
204
205// isOpcWithIntImmediate - This method tests to see if the node is a specific
206// opcode and that it has a immediate integer right operand.
207// If so Imm will receive the 32 bit value.
208static bool isOpcWithIntImmediate(SDNode *N, unsigned Opc, unsigned& Imm) {
209  return N->getOpcode() == Opc &&
210         isInt32Immediate(N->getOperand(1).getNode(), Imm);
211}
212
213
214bool ARMDAGToDAGISel::SelectShifterOperandReg(SDNode *Op,
215                                              SDValue N,
216                                              SDValue &BaseReg,
217                                              SDValue &ShReg,
218                                              SDValue &Opc) {
219  ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(N);
220
221  // Don't match base register only case. That is matched to a separate
222  // lower complexity pattern with explicit register operand.
223  if (ShOpcVal == ARM_AM::no_shift) return false;
224
225  BaseReg = N.getOperand(0);
226  unsigned ShImmVal = 0;
227  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
228    ShReg = CurDAG->getRegister(0, MVT::i32);
229    ShImmVal = RHS->getZExtValue() & 31;
230  } else {
231    ShReg = N.getOperand(1);
232  }
233  Opc = CurDAG->getTargetConstant(ARM_AM::getSORegOpc(ShOpcVal, ShImmVal),
234                                  MVT::i32);
235  return true;
236}
237
238bool ARMDAGToDAGISel::SelectAddrMode2(SDNode *Op, SDValue N,
239                                      SDValue &Base, SDValue &Offset,
240                                      SDValue &Opc) {
241  if (N.getOpcode() == ISD::MUL) {
242    if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
243      // X * [3,5,9] -> X + X * [2,4,8] etc.
244      int RHSC = (int)RHS->getZExtValue();
245      if (RHSC & 1) {
246        RHSC = RHSC & ~1;
247        ARM_AM::AddrOpc AddSub = ARM_AM::add;
248        if (RHSC < 0) {
249          AddSub = ARM_AM::sub;
250          RHSC = - RHSC;
251        }
252        if (isPowerOf2_32(RHSC)) {
253          unsigned ShAmt = Log2_32(RHSC);
254          Base = Offset = N.getOperand(0);
255          Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, ShAmt,
256                                                            ARM_AM::lsl),
257                                          MVT::i32);
258          return true;
259        }
260      }
261    }
262  }
263
264  if (N.getOpcode() != ISD::ADD && N.getOpcode() != ISD::SUB) {
265    Base = N;
266    if (N.getOpcode() == ISD::FrameIndex) {
267      int FI = cast<FrameIndexSDNode>(N)->getIndex();
268      Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
269    } else if (N.getOpcode() == ARMISD::Wrapper &&
270               !(Subtarget->useMovt() &&
271                 N.getOperand(0).getOpcode() == ISD::TargetGlobalAddress)) {
272      Base = N.getOperand(0);
273    }
274    Offset = CurDAG->getRegister(0, MVT::i32);
275    Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(ARM_AM::add, 0,
276                                                      ARM_AM::no_shift),
277                                    MVT::i32);
278    return true;
279  }
280
281  // Match simple R +/- imm12 operands.
282  if (N.getOpcode() == ISD::ADD)
283    if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
284      int RHSC = (int)RHS->getZExtValue();
285      if ((RHSC >= 0 && RHSC < 0x1000) ||
286          (RHSC < 0 && RHSC > -0x1000)) { // 12 bits.
287        Base = N.getOperand(0);
288        if (Base.getOpcode() == ISD::FrameIndex) {
289          int FI = cast<FrameIndexSDNode>(Base)->getIndex();
290          Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
291        }
292        Offset = CurDAG->getRegister(0, MVT::i32);
293
294        ARM_AM::AddrOpc AddSub = ARM_AM::add;
295        if (RHSC < 0) {
296          AddSub = ARM_AM::sub;
297          RHSC = - RHSC;
298        }
299        Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, RHSC,
300                                                          ARM_AM::no_shift),
301                                        MVT::i32);
302        return true;
303      }
304    }
305
306  // Otherwise this is R +/- [possibly shifted] R.
307  ARM_AM::AddrOpc AddSub = N.getOpcode() == ISD::ADD ? ARM_AM::add:ARM_AM::sub;
308  ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(N.getOperand(1));
309  unsigned ShAmt = 0;
310
311  Base   = N.getOperand(0);
312  Offset = N.getOperand(1);
313
314  if (ShOpcVal != ARM_AM::no_shift) {
315    // Check to see if the RHS of the shift is a constant, if not, we can't fold
316    // it.
317    if (ConstantSDNode *Sh =
318           dyn_cast<ConstantSDNode>(N.getOperand(1).getOperand(1))) {
319      ShAmt = Sh->getZExtValue();
320      Offset = N.getOperand(1).getOperand(0);
321    } else {
322      ShOpcVal = ARM_AM::no_shift;
323    }
324  }
325
326  // Try matching (R shl C) + (R).
327  if (N.getOpcode() == ISD::ADD && ShOpcVal == ARM_AM::no_shift) {
328    ShOpcVal = ARM_AM::getShiftOpcForNode(N.getOperand(0));
329    if (ShOpcVal != ARM_AM::no_shift) {
330      // Check to see if the RHS of the shift is a constant, if not, we can't
331      // fold it.
332      if (ConstantSDNode *Sh =
333          dyn_cast<ConstantSDNode>(N.getOperand(0).getOperand(1))) {
334        ShAmt = Sh->getZExtValue();
335        Offset = N.getOperand(0).getOperand(0);
336        Base = N.getOperand(1);
337      } else {
338        ShOpcVal = ARM_AM::no_shift;
339      }
340    }
341  }
342
343  Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, ShAmt, ShOpcVal),
344                                  MVT::i32);
345  return true;
346}
347
348bool ARMDAGToDAGISel::SelectAddrMode2Offset(SDNode *Op, SDValue N,
349                                            SDValue &Offset, SDValue &Opc) {
350  unsigned Opcode = Op->getOpcode();
351  ISD::MemIndexedMode AM = (Opcode == ISD::LOAD)
352    ? cast<LoadSDNode>(Op)->getAddressingMode()
353    : cast<StoreSDNode>(Op)->getAddressingMode();
354  ARM_AM::AddrOpc AddSub = (AM == ISD::PRE_INC || AM == ISD::POST_INC)
355    ? ARM_AM::add : ARM_AM::sub;
356  if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(N)) {
357    int Val = (int)C->getZExtValue();
358    if (Val >= 0 && Val < 0x1000) { // 12 bits.
359      Offset = CurDAG->getRegister(0, MVT::i32);
360      Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, Val,
361                                                        ARM_AM::no_shift),
362                                      MVT::i32);
363      return true;
364    }
365  }
366
367  Offset = N;
368  ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(N);
369  unsigned ShAmt = 0;
370  if (ShOpcVal != ARM_AM::no_shift) {
371    // Check to see if the RHS of the shift is a constant, if not, we can't fold
372    // it.
373    if (ConstantSDNode *Sh = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
374      ShAmt = Sh->getZExtValue();
375      Offset = N.getOperand(0);
376    } else {
377      ShOpcVal = ARM_AM::no_shift;
378    }
379  }
380
381  Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, ShAmt, ShOpcVal),
382                                  MVT::i32);
383  return true;
384}
385
386
387bool ARMDAGToDAGISel::SelectAddrMode3(SDNode *Op, SDValue N,
388                                      SDValue &Base, SDValue &Offset,
389                                      SDValue &Opc) {
390  if (N.getOpcode() == ISD::SUB) {
391    // X - C  is canonicalize to X + -C, no need to handle it here.
392    Base = N.getOperand(0);
393    Offset = N.getOperand(1);
394    Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(ARM_AM::sub, 0),MVT::i32);
395    return true;
396  }
397
398  if (N.getOpcode() != ISD::ADD) {
399    Base = N;
400    if (N.getOpcode() == ISD::FrameIndex) {
401      int FI = cast<FrameIndexSDNode>(N)->getIndex();
402      Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
403    }
404    Offset = CurDAG->getRegister(0, MVT::i32);
405    Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(ARM_AM::add, 0),MVT::i32);
406    return true;
407  }
408
409  // If the RHS is +/- imm8, fold into addr mode.
410  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
411    int RHSC = (int)RHS->getZExtValue();
412    if ((RHSC >= 0 && RHSC < 256) ||
413        (RHSC < 0 && RHSC > -256)) { // note -256 itself isn't allowed.
414      Base = N.getOperand(0);
415      if (Base.getOpcode() == ISD::FrameIndex) {
416        int FI = cast<FrameIndexSDNode>(Base)->getIndex();
417        Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
418      }
419      Offset = CurDAG->getRegister(0, MVT::i32);
420
421      ARM_AM::AddrOpc AddSub = ARM_AM::add;
422      if (RHSC < 0) {
423        AddSub = ARM_AM::sub;
424        RHSC = - RHSC;
425      }
426      Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(AddSub, RHSC),MVT::i32);
427      return true;
428    }
429  }
430
431  Base = N.getOperand(0);
432  Offset = N.getOperand(1);
433  Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(ARM_AM::add, 0), MVT::i32);
434  return true;
435}
436
437bool ARMDAGToDAGISel::SelectAddrMode3Offset(SDNode *Op, SDValue N,
438                                            SDValue &Offset, SDValue &Opc) {
439  unsigned Opcode = Op->getOpcode();
440  ISD::MemIndexedMode AM = (Opcode == ISD::LOAD)
441    ? cast<LoadSDNode>(Op)->getAddressingMode()
442    : cast<StoreSDNode>(Op)->getAddressingMode();
443  ARM_AM::AddrOpc AddSub = (AM == ISD::PRE_INC || AM == ISD::POST_INC)
444    ? ARM_AM::add : ARM_AM::sub;
445  if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(N)) {
446    int Val = (int)C->getZExtValue();
447    if (Val >= 0 && Val < 256) {
448      Offset = CurDAG->getRegister(0, MVT::i32);
449      Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(AddSub, Val), MVT::i32);
450      return true;
451    }
452  }
453
454  Offset = N;
455  Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(AddSub, 0), MVT::i32);
456  return true;
457}
458
459bool ARMDAGToDAGISel::SelectAddrMode4(SDNode *Op, SDValue N,
460                                      SDValue &Addr, SDValue &Mode) {
461  Addr = N;
462  Mode = CurDAG->getTargetConstant(0, MVT::i32);
463  return true;
464}
465
466bool ARMDAGToDAGISel::SelectAddrMode5(SDNode *Op, SDValue N,
467                                      SDValue &Base, SDValue &Offset) {
468  if (N.getOpcode() != ISD::ADD) {
469    Base = N;
470    if (N.getOpcode() == ISD::FrameIndex) {
471      int FI = cast<FrameIndexSDNode>(N)->getIndex();
472      Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
473    } else if (N.getOpcode() == ARMISD::Wrapper &&
474               !(Subtarget->useMovt() &&
475                 N.getOperand(0).getOpcode() == ISD::TargetGlobalAddress)) {
476      Base = N.getOperand(0);
477    }
478    Offset = CurDAG->getTargetConstant(ARM_AM::getAM5Opc(ARM_AM::add, 0),
479                                       MVT::i32);
480    return true;
481  }
482
483  // If the RHS is +/- imm8, fold into addr mode.
484  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
485    int RHSC = (int)RHS->getZExtValue();
486    if ((RHSC & 3) == 0) {  // The constant is implicitly multiplied by 4.
487      RHSC >>= 2;
488      if ((RHSC >= 0 && RHSC < 256) ||
489          (RHSC < 0 && RHSC > -256)) { // note -256 itself isn't allowed.
490        Base = N.getOperand(0);
491        if (Base.getOpcode() == ISD::FrameIndex) {
492          int FI = cast<FrameIndexSDNode>(Base)->getIndex();
493          Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
494        }
495
496        ARM_AM::AddrOpc AddSub = ARM_AM::add;
497        if (RHSC < 0) {
498          AddSub = ARM_AM::sub;
499          RHSC = - RHSC;
500        }
501        Offset = CurDAG->getTargetConstant(ARM_AM::getAM5Opc(AddSub, RHSC),
502                                           MVT::i32);
503        return true;
504      }
505    }
506  }
507
508  Base = N;
509  Offset = CurDAG->getTargetConstant(ARM_AM::getAM5Opc(ARM_AM::add, 0),
510                                     MVT::i32);
511  return true;
512}
513
514bool ARMDAGToDAGISel::SelectAddrMode6(SDNode *Op, SDValue N,
515                                      SDValue &Addr, SDValue &Align) {
516  Addr = N;
517  // Default to no alignment.
518  Align = CurDAG->getTargetConstant(0, MVT::i32);
519  return true;
520}
521
522bool ARMDAGToDAGISel::SelectAddrModePC(SDNode *Op, SDValue N,
523                                       SDValue &Offset, SDValue &Label) {
524  if (N.getOpcode() == ARMISD::PIC_ADD && N.hasOneUse()) {
525    Offset = N.getOperand(0);
526    SDValue N1 = N.getOperand(1);
527    Label  = CurDAG->getTargetConstant(cast<ConstantSDNode>(N1)->getZExtValue(),
528                                       MVT::i32);
529    return true;
530  }
531  return false;
532}
533
534bool ARMDAGToDAGISel::SelectThumbAddrModeRR(SDNode *Op, SDValue N,
535                                            SDValue &Base, SDValue &Offset){
536  // FIXME dl should come from the parent load or store, not the address
537  DebugLoc dl = Op->getDebugLoc();
538  if (N.getOpcode() != ISD::ADD) {
539    ConstantSDNode *NC = dyn_cast<ConstantSDNode>(N);
540    if (!NC || NC->getZExtValue() != 0)
541      return false;
542
543    Base = Offset = N;
544    return true;
545  }
546
547  Base = N.getOperand(0);
548  Offset = N.getOperand(1);
549  return true;
550}
551
552bool
553ARMDAGToDAGISel::SelectThumbAddrModeRI5(SDNode *Op, SDValue N,
554                                        unsigned Scale, SDValue &Base,
555                                        SDValue &OffImm, SDValue &Offset) {
556  if (Scale == 4) {
557    SDValue TmpBase, TmpOffImm;
558    if (SelectThumbAddrModeSP(Op, N, TmpBase, TmpOffImm))
559      return false;  // We want to select tLDRspi / tSTRspi instead.
560    if (N.getOpcode() == ARMISD::Wrapper &&
561        N.getOperand(0).getOpcode() == ISD::TargetConstantPool)
562      return false;  // We want to select tLDRpci instead.
563  }
564
565  if (N.getOpcode() != ISD::ADD) {
566    if (N.getOpcode() == ARMISD::Wrapper &&
567        !(Subtarget->useMovt() &&
568          N.getOperand(0).getOpcode() == ISD::TargetGlobalAddress)) {
569      Base = N.getOperand(0);
570    } else
571      Base = N;
572
573    Offset = CurDAG->getRegister(0, MVT::i32);
574    OffImm = CurDAG->getTargetConstant(0, MVT::i32);
575    return true;
576  }
577
578  // Thumb does not have [sp, r] address mode.
579  RegisterSDNode *LHSR = dyn_cast<RegisterSDNode>(N.getOperand(0));
580  RegisterSDNode *RHSR = dyn_cast<RegisterSDNode>(N.getOperand(1));
581  if ((LHSR && LHSR->getReg() == ARM::SP) ||
582      (RHSR && RHSR->getReg() == ARM::SP)) {
583    Base = N;
584    Offset = CurDAG->getRegister(0, MVT::i32);
585    OffImm = CurDAG->getTargetConstant(0, MVT::i32);
586    return true;
587  }
588
589  // If the RHS is + imm5 * scale, fold into addr mode.
590  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
591    int RHSC = (int)RHS->getZExtValue();
592    if ((RHSC & (Scale-1)) == 0) {  // The constant is implicitly multiplied.
593      RHSC /= Scale;
594      if (RHSC >= 0 && RHSC < 32) {
595        Base = N.getOperand(0);
596        Offset = CurDAG->getRegister(0, MVT::i32);
597        OffImm = CurDAG->getTargetConstant(RHSC, MVT::i32);
598        return true;
599      }
600    }
601  }
602
603  Base = N.getOperand(0);
604  Offset = N.getOperand(1);
605  OffImm = CurDAG->getTargetConstant(0, MVT::i32);
606  return true;
607}
608
609bool ARMDAGToDAGISel::SelectThumbAddrModeS1(SDNode *Op, SDValue N,
610                                            SDValue &Base, SDValue &OffImm,
611                                            SDValue &Offset) {
612  return SelectThumbAddrModeRI5(Op, N, 1, Base, OffImm, Offset);
613}
614
615bool ARMDAGToDAGISel::SelectThumbAddrModeS2(SDNode *Op, SDValue N,
616                                            SDValue &Base, SDValue &OffImm,
617                                            SDValue &Offset) {
618  return SelectThumbAddrModeRI5(Op, N, 2, Base, OffImm, Offset);
619}
620
621bool ARMDAGToDAGISel::SelectThumbAddrModeS4(SDNode *Op, SDValue N,
622                                            SDValue &Base, SDValue &OffImm,
623                                            SDValue &Offset) {
624  return SelectThumbAddrModeRI5(Op, N, 4, Base, OffImm, Offset);
625}
626
627bool ARMDAGToDAGISel::SelectThumbAddrModeSP(SDNode *Op, SDValue N,
628                                           SDValue &Base, SDValue &OffImm) {
629  if (N.getOpcode() == ISD::FrameIndex) {
630    int FI = cast<FrameIndexSDNode>(N)->getIndex();
631    Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
632    OffImm = CurDAG->getTargetConstant(0, MVT::i32);
633    return true;
634  }
635
636  if (N.getOpcode() != ISD::ADD)
637    return false;
638
639  RegisterSDNode *LHSR = dyn_cast<RegisterSDNode>(N.getOperand(0));
640  if (N.getOperand(0).getOpcode() == ISD::FrameIndex ||
641      (LHSR && LHSR->getReg() == ARM::SP)) {
642    // If the RHS is + imm8 * scale, fold into addr mode.
643    if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
644      int RHSC = (int)RHS->getZExtValue();
645      if ((RHSC & 3) == 0) {  // The constant is implicitly multiplied.
646        RHSC >>= 2;
647        if (RHSC >= 0 && RHSC < 256) {
648          Base = N.getOperand(0);
649          if (Base.getOpcode() == ISD::FrameIndex) {
650            int FI = cast<FrameIndexSDNode>(Base)->getIndex();
651            Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
652          }
653          OffImm = CurDAG->getTargetConstant(RHSC, MVT::i32);
654          return true;
655        }
656      }
657    }
658  }
659
660  return false;
661}
662
663bool ARMDAGToDAGISel::SelectT2ShifterOperandReg(SDNode *Op, SDValue N,
664                                                SDValue &BaseReg,
665                                                SDValue &Opc) {
666  ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(N);
667
668  // Don't match base register only case. That is matched to a separate
669  // lower complexity pattern with explicit register operand.
670  if (ShOpcVal == ARM_AM::no_shift) return false;
671
672  BaseReg = N.getOperand(0);
673  unsigned ShImmVal = 0;
674  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
675    ShImmVal = RHS->getZExtValue() & 31;
676    Opc = getI32Imm(ARM_AM::getSORegOpc(ShOpcVal, ShImmVal));
677    return true;
678  }
679
680  return false;
681}
682
683bool ARMDAGToDAGISel::SelectT2AddrModeImm12(SDNode *Op, SDValue N,
684                                            SDValue &Base, SDValue &OffImm) {
685  // Match simple R + imm12 operands.
686
687  // Base only.
688  if (N.getOpcode() != ISD::ADD && N.getOpcode() != ISD::SUB) {
689    if (N.getOpcode() == ISD::FrameIndex) {
690      // Match frame index...
691      int FI = cast<FrameIndexSDNode>(N)->getIndex();
692      Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
693      OffImm  = CurDAG->getTargetConstant(0, MVT::i32);
694      return true;
695    } else if (N.getOpcode() == ARMISD::Wrapper &&
696               !(Subtarget->useMovt() &&
697                 N.getOperand(0).getOpcode() == ISD::TargetGlobalAddress)) {
698      Base = N.getOperand(0);
699      if (Base.getOpcode() == ISD::TargetConstantPool)
700        return false;  // We want to select t2LDRpci instead.
701    } else
702      Base = N;
703    OffImm  = CurDAG->getTargetConstant(0, MVT::i32);
704    return true;
705  }
706
707  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
708    if (SelectT2AddrModeImm8(Op, N, Base, OffImm))
709      // Let t2LDRi8 handle (R - imm8).
710      return false;
711
712    int RHSC = (int)RHS->getZExtValue();
713    if (N.getOpcode() == ISD::SUB)
714      RHSC = -RHSC;
715
716    if (RHSC >= 0 && RHSC < 0x1000) { // 12 bits (unsigned)
717      Base   = N.getOperand(0);
718      if (Base.getOpcode() == ISD::FrameIndex) {
719        int FI = cast<FrameIndexSDNode>(Base)->getIndex();
720        Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
721      }
722      OffImm = CurDAG->getTargetConstant(RHSC, MVT::i32);
723      return true;
724    }
725  }
726
727  // Base only.
728  Base = N;
729  OffImm  = CurDAG->getTargetConstant(0, MVT::i32);
730  return true;
731}
732
733bool ARMDAGToDAGISel::SelectT2AddrModeImm8(SDNode *Op, SDValue N,
734                                           SDValue &Base, SDValue &OffImm) {
735  // Match simple R - imm8 operands.
736  if (N.getOpcode() == ISD::ADD || N.getOpcode() == ISD::SUB) {
737    if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
738      int RHSC = (int)RHS->getSExtValue();
739      if (N.getOpcode() == ISD::SUB)
740        RHSC = -RHSC;
741
742      if ((RHSC >= -255) && (RHSC < 0)) { // 8 bits (always negative)
743        Base = N.getOperand(0);
744        if (Base.getOpcode() == ISD::FrameIndex) {
745          int FI = cast<FrameIndexSDNode>(Base)->getIndex();
746          Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
747        }
748        OffImm = CurDAG->getTargetConstant(RHSC, MVT::i32);
749        return true;
750      }
751    }
752  }
753
754  return false;
755}
756
757bool ARMDAGToDAGISel::SelectT2AddrModeImm8Offset(SDNode *Op, SDValue N,
758                                                 SDValue &OffImm){
759  unsigned Opcode = Op->getOpcode();
760  ISD::MemIndexedMode AM = (Opcode == ISD::LOAD)
761    ? cast<LoadSDNode>(Op)->getAddressingMode()
762    : cast<StoreSDNode>(Op)->getAddressingMode();
763  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N)) {
764    int RHSC = (int)RHS->getZExtValue();
765    if (RHSC >= 0 && RHSC < 0x100) { // 8 bits.
766      OffImm = ((AM == ISD::PRE_INC) || (AM == ISD::POST_INC))
767        ? CurDAG->getTargetConstant(RHSC, MVT::i32)
768        : CurDAG->getTargetConstant(-RHSC, MVT::i32);
769      return true;
770    }
771  }
772
773  return false;
774}
775
776bool ARMDAGToDAGISel::SelectT2AddrModeImm8s4(SDNode *Op, SDValue N,
777                                             SDValue &Base, SDValue &OffImm) {
778  if (N.getOpcode() == ISD::ADD) {
779    if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
780      int RHSC = (int)RHS->getZExtValue();
781      if (((RHSC & 0x3) == 0) &&
782          ((RHSC >= 0 && RHSC < 0x400) || (RHSC < 0 && RHSC > -0x400))) { // 8 bits.
783        Base   = N.getOperand(0);
784        OffImm = CurDAG->getTargetConstant(RHSC, MVT::i32);
785        return true;
786      }
787    }
788  } else if (N.getOpcode() == ISD::SUB) {
789    if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
790      int RHSC = (int)RHS->getZExtValue();
791      if (((RHSC & 0x3) == 0) && (RHSC >= 0 && RHSC < 0x400)) { // 8 bits.
792        Base   = N.getOperand(0);
793        OffImm = CurDAG->getTargetConstant(-RHSC, MVT::i32);
794        return true;
795      }
796    }
797  }
798
799  return false;
800}
801
802bool ARMDAGToDAGISel::SelectT2AddrModeSoReg(SDNode *Op, SDValue N,
803                                            SDValue &Base,
804                                            SDValue &OffReg, SDValue &ShImm) {
805  // (R - imm8) should be handled by t2LDRi8. The rest are handled by t2LDRi12.
806  if (N.getOpcode() != ISD::ADD)
807    return false;
808
809  // Leave (R + imm12) for t2LDRi12, (R - imm8) for t2LDRi8.
810  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
811    int RHSC = (int)RHS->getZExtValue();
812    if (RHSC >= 0 && RHSC < 0x1000) // 12 bits (unsigned)
813      return false;
814    else if (RHSC < 0 && RHSC >= -255) // 8 bits
815      return false;
816  }
817
818  // Look for (R + R) or (R + (R << [1,2,3])).
819  unsigned ShAmt = 0;
820  Base   = N.getOperand(0);
821  OffReg = N.getOperand(1);
822
823  // Swap if it is ((R << c) + R).
824  ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(OffReg);
825  if (ShOpcVal != ARM_AM::lsl) {
826    ShOpcVal = ARM_AM::getShiftOpcForNode(Base);
827    if (ShOpcVal == ARM_AM::lsl)
828      std::swap(Base, OffReg);
829  }
830
831  if (ShOpcVal == ARM_AM::lsl) {
832    // Check to see if the RHS of the shift is a constant, if not, we can't fold
833    // it.
834    if (ConstantSDNode *Sh = dyn_cast<ConstantSDNode>(OffReg.getOperand(1))) {
835      ShAmt = Sh->getZExtValue();
836      if (ShAmt >= 4) {
837        ShAmt = 0;
838        ShOpcVal = ARM_AM::no_shift;
839      } else
840        OffReg = OffReg.getOperand(0);
841    } else {
842      ShOpcVal = ARM_AM::no_shift;
843    }
844  }
845
846  ShImm = CurDAG->getTargetConstant(ShAmt, MVT::i32);
847
848  return true;
849}
850
851//===--------------------------------------------------------------------===//
852
853/// getAL - Returns a ARMCC::AL immediate node.
854static inline SDValue getAL(SelectionDAG *CurDAG) {
855  return CurDAG->getTargetConstant((uint64_t)ARMCC::AL, MVT::i32);
856}
857
858SDNode *ARMDAGToDAGISel::SelectARMIndexedLoad(SDNode *N) {
859  LoadSDNode *LD = cast<LoadSDNode>(N);
860  ISD::MemIndexedMode AM = LD->getAddressingMode();
861  if (AM == ISD::UNINDEXED)
862    return NULL;
863
864  EVT LoadedVT = LD->getMemoryVT();
865  SDValue Offset, AMOpc;
866  bool isPre = (AM == ISD::PRE_INC) || (AM == ISD::PRE_DEC);
867  unsigned Opcode = 0;
868  bool Match = false;
869  if (LoadedVT == MVT::i32 &&
870      SelectAddrMode2Offset(N, LD->getOffset(), Offset, AMOpc)) {
871    Opcode = isPre ? ARM::LDR_PRE : ARM::LDR_POST;
872    Match = true;
873  } else if (LoadedVT == MVT::i16 &&
874             SelectAddrMode3Offset(N, LD->getOffset(), Offset, AMOpc)) {
875    Match = true;
876    Opcode = (LD->getExtensionType() == ISD::SEXTLOAD)
877      ? (isPre ? ARM::LDRSH_PRE : ARM::LDRSH_POST)
878      : (isPre ? ARM::LDRH_PRE : ARM::LDRH_POST);
879  } else if (LoadedVT == MVT::i8 || LoadedVT == MVT::i1) {
880    if (LD->getExtensionType() == ISD::SEXTLOAD) {
881      if (SelectAddrMode3Offset(N, LD->getOffset(), Offset, AMOpc)) {
882        Match = true;
883        Opcode = isPre ? ARM::LDRSB_PRE : ARM::LDRSB_POST;
884      }
885    } else {
886      if (SelectAddrMode2Offset(N, LD->getOffset(), Offset, AMOpc)) {
887        Match = true;
888        Opcode = isPre ? ARM::LDRB_PRE : ARM::LDRB_POST;
889      }
890    }
891  }
892
893  if (Match) {
894    SDValue Chain = LD->getChain();
895    SDValue Base = LD->getBasePtr();
896    SDValue Ops[]= { Base, Offset, AMOpc, getAL(CurDAG),
897                     CurDAG->getRegister(0, MVT::i32), Chain };
898    return CurDAG->getMachineNode(Opcode, N->getDebugLoc(), MVT::i32, MVT::i32,
899                                  MVT::Other, Ops, 6);
900  }
901
902  return NULL;
903}
904
905SDNode *ARMDAGToDAGISel::SelectT2IndexedLoad(SDNode *N) {
906  LoadSDNode *LD = cast<LoadSDNode>(N);
907  ISD::MemIndexedMode AM = LD->getAddressingMode();
908  if (AM == ISD::UNINDEXED)
909    return NULL;
910
911  EVT LoadedVT = LD->getMemoryVT();
912  bool isSExtLd = LD->getExtensionType() == ISD::SEXTLOAD;
913  SDValue Offset;
914  bool isPre = (AM == ISD::PRE_INC) || (AM == ISD::PRE_DEC);
915  unsigned Opcode = 0;
916  bool Match = false;
917  if (SelectT2AddrModeImm8Offset(N, LD->getOffset(), Offset)) {
918    switch (LoadedVT.getSimpleVT().SimpleTy) {
919    case MVT::i32:
920      Opcode = isPre ? ARM::t2LDR_PRE : ARM::t2LDR_POST;
921      break;
922    case MVT::i16:
923      if (isSExtLd)
924        Opcode = isPre ? ARM::t2LDRSH_PRE : ARM::t2LDRSH_POST;
925      else
926        Opcode = isPre ? ARM::t2LDRH_PRE : ARM::t2LDRH_POST;
927      break;
928    case MVT::i8:
929    case MVT::i1:
930      if (isSExtLd)
931        Opcode = isPre ? ARM::t2LDRSB_PRE : ARM::t2LDRSB_POST;
932      else
933        Opcode = isPre ? ARM::t2LDRB_PRE : ARM::t2LDRB_POST;
934      break;
935    default:
936      return NULL;
937    }
938    Match = true;
939  }
940
941  if (Match) {
942    SDValue Chain = LD->getChain();
943    SDValue Base = LD->getBasePtr();
944    SDValue Ops[]= { Base, Offset, getAL(CurDAG),
945                     CurDAG->getRegister(0, MVT::i32), Chain };
946    return CurDAG->getMachineNode(Opcode, N->getDebugLoc(), MVT::i32, MVT::i32,
947                                  MVT::Other, Ops, 5);
948  }
949
950  return NULL;
951}
952
953/// PairDRegs - Form a quad register from a pair of D registers.
954///
955SDNode *ARMDAGToDAGISel::PairDRegs(EVT VT, SDValue V0, SDValue V1) {
956  DebugLoc dl = V0.getNode()->getDebugLoc();
957  SDValue SubReg0 = CurDAG->getTargetConstant(ARM::DSUBREG_0, MVT::i32);
958  SDValue SubReg1 = CurDAG->getTargetConstant(ARM::DSUBREG_1, MVT::i32);
959  if (llvm::ModelWithRegSequence()) {
960    const SDValue Ops[] = { V0, SubReg0, V1, SubReg1 };
961    return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops, 4);
962  }
963  SDValue Undef =
964    SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, dl, VT), 0);
965  SDNode *Pair = CurDAG->getMachineNode(TargetOpcode::INSERT_SUBREG, dl,
966                                        VT, Undef, V0, SubReg0);
967  return CurDAG->getMachineNode(TargetOpcode::INSERT_SUBREG, dl,
968                                VT, SDValue(Pair, 0), V1, SubReg1);
969}
970
971/// PairQRegs - Form 4 consecutive D registers from a pair of Q registers.
972///
973SDNode *ARMDAGToDAGISel::PairQRegs(EVT VT, SDValue V0, SDValue V1) {
974  DebugLoc dl = V0.getNode()->getDebugLoc();
975  SDValue SubReg0 = CurDAG->getTargetConstant(ARM::QSUBREG_0, MVT::i32);
976  SDValue SubReg1 = CurDAG->getTargetConstant(ARM::QSUBREG_1, MVT::i32);
977  const SDValue Ops[] = { V0, SubReg0, V1, SubReg1 };
978  return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops, 4);
979}
980
981/// QuadDRegs - Form 4 consecutive D registers.
982///
983SDNode *ARMDAGToDAGISel::QuadDRegs(EVT VT, SDValue V0, SDValue V1,
984                                   SDValue V2, SDValue V3) {
985  DebugLoc dl = V0.getNode()->getDebugLoc();
986  SDValue SubReg0 = CurDAG->getTargetConstant(ARM::DSUBREG_0, MVT::i32);
987  SDValue SubReg1 = CurDAG->getTargetConstant(ARM::DSUBREG_1, MVT::i32);
988  SDValue SubReg2 = CurDAG->getTargetConstant(ARM::DSUBREG_2, MVT::i32);
989  SDValue SubReg3 = CurDAG->getTargetConstant(ARM::DSUBREG_3, MVT::i32);
990  const SDValue Ops[] = { V0, SubReg0, V1, SubReg1, V2, SubReg2, V3, SubReg3 };
991  return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops, 8);
992}
993
994/// GetNEONSubregVT - Given a type for a 128-bit NEON vector, return the type
995/// for a 64-bit subregister of the vector.
996static EVT GetNEONSubregVT(EVT VT) {
997  switch (VT.getSimpleVT().SimpleTy) {
998  default: llvm_unreachable("unhandled NEON type");
999  case MVT::v16i8: return MVT::v8i8;
1000  case MVT::v8i16: return MVT::v4i16;
1001  case MVT::v4f32: return MVT::v2f32;
1002  case MVT::v4i32: return MVT::v2i32;
1003  case MVT::v2i64: return MVT::v1i64;
1004  }
1005}
1006
1007SDNode *ARMDAGToDAGISel::SelectVLD(SDNode *N, unsigned NumVecs,
1008                                   unsigned *DOpcodes, unsigned *QOpcodes0,
1009                                   unsigned *QOpcodes1) {
1010  assert(NumVecs >= 1 && NumVecs <= 4 && "VLD NumVecs out-of-range");
1011  DebugLoc dl = N->getDebugLoc();
1012
1013  SDValue MemAddr, Align;
1014  if (!SelectAddrMode6(N, N->getOperand(2), MemAddr, Align))
1015    return NULL;
1016
1017  SDValue Chain = N->getOperand(0);
1018  EVT VT = N->getValueType(0);
1019  bool is64BitVector = VT.is64BitVector();
1020
1021  unsigned OpcodeIndex;
1022  switch (VT.getSimpleVT().SimpleTy) {
1023  default: llvm_unreachable("unhandled vld type");
1024    // Double-register operations:
1025  case MVT::v8i8:  OpcodeIndex = 0; break;
1026  case MVT::v4i16: OpcodeIndex = 1; break;
1027  case MVT::v2f32:
1028  case MVT::v2i32: OpcodeIndex = 2; break;
1029  case MVT::v1i64: OpcodeIndex = 3; break;
1030    // Quad-register operations:
1031  case MVT::v16i8: OpcodeIndex = 0; break;
1032  case MVT::v8i16: OpcodeIndex = 1; break;
1033  case MVT::v4f32:
1034  case MVT::v4i32: OpcodeIndex = 2; break;
1035  case MVT::v2i64: OpcodeIndex = 3;
1036    assert(NumVecs == 1 && "v2i64 type only supported for VLD1");
1037    break;
1038  }
1039
1040  SDValue Pred = getAL(CurDAG);
1041  SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
1042  if (is64BitVector) {
1043    unsigned Opc = DOpcodes[OpcodeIndex];
1044    const SDValue Ops[] = { MemAddr, Align, Pred, Reg0, Chain };
1045    std::vector<EVT> ResTys(NumVecs, VT);
1046    ResTys.push_back(MVT::Other);
1047    SDNode *VLd = CurDAG->getMachineNode(Opc, dl, ResTys, Ops, 5);
1048    if (!llvm::ModelWithRegSequence() || NumVecs < 2)
1049      return VLd;
1050
1051    assert(NumVecs <= 4);
1052    SDValue RegSeq;
1053    SDValue V0 = SDValue(VLd, 0);
1054    SDValue V1 = SDValue(VLd, 1);
1055
1056    // Form a REG_SEQUENCE to force register allocation.
1057    if (NumVecs == 2)
1058      RegSeq = SDValue(PairDRegs(MVT::v2i64, V0, V1), 0);
1059    else {
1060      SDValue V2 = SDValue(VLd, 2);
1061      // If it's a vld3, form a quad D-register but discard the last part.
1062      SDValue V3 = (NumVecs == 3)
1063          ? SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF,dl,VT), 0)
1064          : SDValue(VLd, 3);
1065      RegSeq = SDValue(QuadDRegs(MVT::v4i64, V0, V1, V2, V3), 0);
1066    }
1067
1068    SDValue D0 = CurDAG->getTargetExtractSubreg(ARM::DSUBREG_0, dl, VT, RegSeq);
1069    ReplaceUses(SDValue(N, 0), D0);
1070    SDValue D1 = CurDAG->getTargetExtractSubreg(ARM::DSUBREG_1, dl, VT, RegSeq);
1071    ReplaceUses(SDValue(N, 1), D1);
1072
1073    if (NumVecs > 2) {
1074      SDValue D2 = CurDAG->getTargetExtractSubreg(ARM::DSUBREG_2, dl, VT, RegSeq);
1075      ReplaceUses(SDValue(N, 2), D2);
1076    }
1077    if (NumVecs > 3) {
1078      SDValue D3 = CurDAG->getTargetExtractSubreg(ARM::DSUBREG_3, dl, VT, RegSeq);
1079      ReplaceUses(SDValue(N, 3), D3);
1080    }
1081    ReplaceUses(SDValue(N, NumVecs), SDValue(VLd, NumVecs));
1082    return NULL;
1083  }
1084
1085  EVT RegVT = GetNEONSubregVT(VT);
1086  if (NumVecs <= 2) {
1087    // Quad registers are directly supported for VLD1 and VLD2,
1088    // loading pairs of D regs.
1089    unsigned Opc = QOpcodes0[OpcodeIndex];
1090    const SDValue Ops[] = { MemAddr, Align, Pred, Reg0, Chain };
1091    std::vector<EVT> ResTys(2 * NumVecs, RegVT);
1092    ResTys.push_back(MVT::Other);
1093    SDNode *VLd = CurDAG->getMachineNode(Opc, dl, ResTys, Ops, 5);
1094    Chain = SDValue(VLd, 2 * NumVecs);
1095
1096    // Combine the even and odd subregs to produce the result.
1097    if (llvm::ModelWithRegSequence()) {
1098      if (NumVecs == 1) {
1099        SDNode *Q = PairDRegs(VT, SDValue(VLd, 0), SDValue(VLd, 1));
1100        ReplaceUses(SDValue(N, 0), SDValue(Q, 0));
1101      } else {
1102        SDValue QQ = SDValue(QuadDRegs(MVT::v4i64,
1103                                       SDValue(VLd, 0), SDValue(VLd, 1),
1104                                       SDValue(VLd, 2), SDValue(VLd, 3)), 0);
1105        SDValue Q0 = CurDAG->getTargetExtractSubreg(ARM::QSUBREG_0, dl, VT, QQ);
1106        SDValue Q1 = CurDAG->getTargetExtractSubreg(ARM::QSUBREG_1, dl, VT, QQ);
1107        ReplaceUses(SDValue(N, 0), Q0);
1108        ReplaceUses(SDValue(N, 1), Q1);
1109      }
1110    } else {
1111      for (unsigned Vec = 0; Vec < NumVecs; ++Vec) {
1112        SDNode *Q = PairDRegs(VT, SDValue(VLd, 2*Vec), SDValue(VLd, 2*Vec+1));
1113        ReplaceUses(SDValue(N, Vec), SDValue(Q, 0));
1114      }
1115    }
1116  } else {
1117    // Otherwise, quad registers are loaded with two separate instructions,
1118    // where one loads the even registers and the other loads the odd registers.
1119
1120    std::vector<EVT> ResTys(NumVecs, RegVT);
1121    ResTys.push_back(MemAddr.getValueType());
1122    ResTys.push_back(MVT::Other);
1123
1124    // Load the even subregs.
1125    unsigned Opc = QOpcodes0[OpcodeIndex];
1126    const SDValue OpsA[] = { MemAddr, Align, Reg0, Pred, Reg0, Chain };
1127    SDNode *VLdA = CurDAG->getMachineNode(Opc, dl, ResTys, OpsA, 6);
1128    Chain = SDValue(VLdA, NumVecs+1);
1129
1130    // Load the odd subregs.
1131    Opc = QOpcodes1[OpcodeIndex];
1132    const SDValue OpsB[] = { SDValue(VLdA, NumVecs),
1133                             Align, Reg0, Pred, Reg0, Chain };
1134    SDNode *VLdB = CurDAG->getMachineNode(Opc, dl, ResTys, OpsB, 6);
1135    Chain = SDValue(VLdB, NumVecs+1);
1136
1137    // Combine the even and odd subregs to produce the result.
1138    for (unsigned Vec = 0; Vec < NumVecs; ++Vec) {
1139      SDNode *Q = PairDRegs(VT, SDValue(VLdA, Vec), SDValue(VLdB, Vec));
1140      ReplaceUses(SDValue(N, Vec), SDValue(Q, 0));
1141    }
1142  }
1143  ReplaceUses(SDValue(N, NumVecs), Chain);
1144  return NULL;
1145}
1146
1147SDNode *ARMDAGToDAGISel::SelectVST(SDNode *N, unsigned NumVecs,
1148                                   unsigned *DOpcodes, unsigned *QOpcodes0,
1149                                   unsigned *QOpcodes1) {
1150  assert(NumVecs >=1 && NumVecs <= 4 && "VST NumVecs out-of-range");
1151  DebugLoc dl = N->getDebugLoc();
1152
1153  SDValue MemAddr, Align;
1154  if (!SelectAddrMode6(N, N->getOperand(2), MemAddr, Align))
1155    return NULL;
1156
1157  SDValue Chain = N->getOperand(0);
1158  EVT VT = N->getOperand(3).getValueType();
1159  bool is64BitVector = VT.is64BitVector();
1160
1161  unsigned OpcodeIndex;
1162  switch (VT.getSimpleVT().SimpleTy) {
1163  default: llvm_unreachable("unhandled vst type");
1164    // Double-register operations:
1165  case MVT::v8i8:  OpcodeIndex = 0; break;
1166  case MVT::v4i16: OpcodeIndex = 1; break;
1167  case MVT::v2f32:
1168  case MVT::v2i32: OpcodeIndex = 2; break;
1169  case MVT::v1i64: OpcodeIndex = 3; break;
1170    // Quad-register operations:
1171  case MVT::v16i8: OpcodeIndex = 0; break;
1172  case MVT::v8i16: OpcodeIndex = 1; break;
1173  case MVT::v4f32:
1174  case MVT::v4i32: OpcodeIndex = 2; break;
1175  case MVT::v2i64: OpcodeIndex = 3;
1176    assert(NumVecs == 1 && "v2i64 type only supported for VST1");
1177    break;
1178  }
1179
1180  SDValue Pred = getAL(CurDAG);
1181  SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
1182
1183  SmallVector<SDValue, 10> Ops;
1184  Ops.push_back(MemAddr);
1185  Ops.push_back(Align);
1186
1187  if (is64BitVector) {
1188    if (llvm::ModelWithRegSequence() && NumVecs >= 2) {
1189      assert(NumVecs <= 4);
1190      SDValue RegSeq;
1191      SDValue V0 = N->getOperand(0+3);
1192      SDValue V1 = N->getOperand(1+3);
1193
1194      // Form a REG_SEQUENCE to force register allocation.
1195      if (NumVecs == 2)
1196        RegSeq = SDValue(PairDRegs(MVT::v2i64, V0, V1), 0);
1197      else {
1198        SDValue V2 = N->getOperand(2+3);
1199        // If it's a vld3, form a quad D-register and leave the last part as
1200        // an undef.
1201        SDValue V3 = (NumVecs == 3)
1202          ? SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF,dl,VT), 0)
1203          : N->getOperand(3+3);
1204        RegSeq = SDValue(QuadDRegs(MVT::v4i64, V0, V1, V2, V3), 0);
1205      }
1206
1207      // Now extract the D registers back out.
1208      Ops.push_back(CurDAG->getTargetExtractSubreg(ARM::DSUBREG_0, dl, VT,
1209                                                   RegSeq));
1210      Ops.push_back(CurDAG->getTargetExtractSubreg(ARM::DSUBREG_1, dl, VT,
1211                                                   RegSeq));
1212      if (NumVecs > 2)
1213        Ops.push_back(CurDAG->getTargetExtractSubreg(ARM::DSUBREG_2, dl, VT,
1214                                                     RegSeq));
1215      if (NumVecs > 3)
1216        Ops.push_back(CurDAG->getTargetExtractSubreg(ARM::DSUBREG_3, dl, VT,
1217                                                     RegSeq));
1218    } else {
1219      for (unsigned Vec = 0; Vec < NumVecs; ++Vec)
1220        Ops.push_back(N->getOperand(Vec+3));
1221    }
1222    Ops.push_back(Pred);
1223    Ops.push_back(Reg0); // predicate register
1224    Ops.push_back(Chain);
1225    unsigned Opc = DOpcodes[OpcodeIndex];
1226    return CurDAG->getMachineNode(Opc, dl, MVT::Other, Ops.data(), NumVecs+5);
1227  }
1228
1229  EVT RegVT = GetNEONSubregVT(VT);
1230  if (NumVecs <= 2) {
1231    // Quad registers are directly supported for VST1 and VST2,
1232    // storing pairs of D regs.
1233    unsigned Opc = QOpcodes0[OpcodeIndex];
1234    if (llvm::ModelWithRegSequence() && NumVecs == 2) {
1235      // First extract the pair of Q registers.
1236      SDValue Q0 = N->getOperand(3);
1237      SDValue Q1 = N->getOperand(4);
1238
1239      // Form a QQ register.
1240      SDValue QQ = SDValue(PairQRegs(MVT::v4i64, Q0, Q1), 0);
1241
1242      // Now extract the D registers back out.
1243      Ops.push_back(CurDAG->getTargetExtractSubreg(ARM::DSUBREG_0, dl, RegVT,
1244                                                   QQ));
1245      Ops.push_back(CurDAG->getTargetExtractSubreg(ARM::DSUBREG_1, dl, RegVT,
1246                                                   QQ));
1247      Ops.push_back(CurDAG->getTargetExtractSubreg(ARM::DSUBREG_2, dl, RegVT,
1248                                                   QQ));
1249      Ops.push_back(CurDAG->getTargetExtractSubreg(ARM::DSUBREG_3, dl, RegVT,
1250                                                   QQ));
1251      Ops.push_back(Pred);
1252      Ops.push_back(Reg0); // predicate register
1253      Ops.push_back(Chain);
1254      return CurDAG->getMachineNode(Opc, dl, MVT::Other, Ops.data(), 5 + 4);
1255    } else {
1256      for (unsigned Vec = 0; Vec < NumVecs; ++Vec) {
1257        Ops.push_back(CurDAG->getTargetExtractSubreg(ARM::DSUBREG_0, dl, RegVT,
1258                                                     N->getOperand(Vec+3)));
1259        Ops.push_back(CurDAG->getTargetExtractSubreg(ARM::DSUBREG_1, dl, RegVT,
1260                                                     N->getOperand(Vec+3)));
1261      }
1262      Ops.push_back(Pred);
1263      Ops.push_back(Reg0); // predicate register
1264      Ops.push_back(Chain);
1265      return CurDAG->getMachineNode(Opc, dl, MVT::Other, Ops.data(),
1266                                    5 + 2 * NumVecs);
1267    }
1268  }
1269
1270  // Otherwise, quad registers are stored with two separate instructions,
1271  // where one stores the even registers and the other stores the odd registers.
1272
1273  Ops.push_back(Reg0); // post-access address offset
1274
1275  // Store the even subregs.
1276  for (unsigned Vec = 0; Vec < NumVecs; ++Vec)
1277    Ops.push_back(CurDAG->getTargetExtractSubreg(ARM::DSUBREG_0, dl, RegVT,
1278                                                 N->getOperand(Vec+3)));
1279  Ops.push_back(Pred);
1280  Ops.push_back(Reg0); // predicate register
1281  Ops.push_back(Chain);
1282  unsigned Opc = QOpcodes0[OpcodeIndex];
1283  SDNode *VStA = CurDAG->getMachineNode(Opc, dl, MemAddr.getValueType(),
1284                                        MVT::Other, Ops.data(), NumVecs+6);
1285  Chain = SDValue(VStA, 1);
1286
1287  // Store the odd subregs.
1288  Ops[0] = SDValue(VStA, 0); // MemAddr
1289  for (unsigned Vec = 0; Vec < NumVecs; ++Vec)
1290    Ops[Vec+3] = CurDAG->getTargetExtractSubreg(ARM::DSUBREG_1, dl, RegVT,
1291                                                N->getOperand(Vec+3));
1292  Ops[NumVecs+5] = Chain;
1293  Opc = QOpcodes1[OpcodeIndex];
1294  SDNode *VStB = CurDAG->getMachineNode(Opc, dl, MemAddr.getValueType(),
1295                                        MVT::Other, Ops.data(), NumVecs+6);
1296  Chain = SDValue(VStB, 1);
1297  ReplaceUses(SDValue(N, 0), Chain);
1298  return NULL;
1299}
1300
1301SDNode *ARMDAGToDAGISel::SelectVLDSTLane(SDNode *N, bool IsLoad,
1302                                         unsigned NumVecs, unsigned *DOpcodes,
1303                                         unsigned *QOpcodes0,
1304                                         unsigned *QOpcodes1) {
1305  assert(NumVecs >=2 && NumVecs <= 4 && "VLDSTLane NumVecs out-of-range");
1306  DebugLoc dl = N->getDebugLoc();
1307
1308  SDValue MemAddr, Align;
1309  if (!SelectAddrMode6(N, N->getOperand(2), MemAddr, Align))
1310    return NULL;
1311
1312  SDValue Chain = N->getOperand(0);
1313  unsigned Lane =
1314    cast<ConstantSDNode>(N->getOperand(NumVecs+3))->getZExtValue();
1315  EVT VT = IsLoad ? N->getValueType(0) : N->getOperand(3).getValueType();
1316  bool is64BitVector = VT.is64BitVector();
1317
1318  // Quad registers are handled by load/store of subregs. Find the subreg info.
1319  unsigned NumElts = 0;
1320  int SubregIdx = 0;
1321  EVT RegVT = VT;
1322  if (!is64BitVector) {
1323    RegVT = GetNEONSubregVT(VT);
1324    NumElts = RegVT.getVectorNumElements();
1325    SubregIdx = (Lane < NumElts) ? ARM::DSUBREG_0 : ARM::DSUBREG_1;
1326  }
1327
1328  unsigned OpcodeIndex;
1329  switch (VT.getSimpleVT().SimpleTy) {
1330  default: llvm_unreachable("unhandled vld/vst lane type");
1331    // Double-register operations:
1332  case MVT::v8i8:  OpcodeIndex = 0; break;
1333  case MVT::v4i16: OpcodeIndex = 1; break;
1334  case MVT::v2f32:
1335  case MVT::v2i32: OpcodeIndex = 2; break;
1336    // Quad-register operations:
1337  case MVT::v8i16: OpcodeIndex = 0; break;
1338  case MVT::v4f32:
1339  case MVT::v4i32: OpcodeIndex = 1; break;
1340  }
1341
1342  SDValue Pred = getAL(CurDAG);
1343  SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
1344
1345  SmallVector<SDValue, 10> Ops;
1346  Ops.push_back(MemAddr);
1347  Ops.push_back(Align);
1348
1349  unsigned Opc = 0;
1350  if (is64BitVector) {
1351    Opc = DOpcodes[OpcodeIndex];
1352    for (unsigned Vec = 0; Vec < NumVecs; ++Vec)
1353      Ops.push_back(N->getOperand(Vec+3));
1354  } else {
1355    // Check if this is loading the even or odd subreg of a Q register.
1356    if (Lane < NumElts) {
1357      Opc = QOpcodes0[OpcodeIndex];
1358    } else {
1359      Lane -= NumElts;
1360      Opc = QOpcodes1[OpcodeIndex];
1361    }
1362    // Extract the subregs of the input vector.
1363    for (unsigned Vec = 0; Vec < NumVecs; ++Vec)
1364      Ops.push_back(CurDAG->getTargetExtractSubreg(SubregIdx, dl, RegVT,
1365                                                   N->getOperand(Vec+3)));
1366  }
1367  Ops.push_back(getI32Imm(Lane));
1368  Ops.push_back(Pred);
1369  Ops.push_back(Reg0);
1370  Ops.push_back(Chain);
1371
1372  if (!IsLoad)
1373    return CurDAG->getMachineNode(Opc, dl, MVT::Other, Ops.data(), NumVecs+6);
1374
1375  std::vector<EVT> ResTys(NumVecs, RegVT);
1376  ResTys.push_back(MVT::Other);
1377  SDNode *VLdLn =
1378    CurDAG->getMachineNode(Opc, dl, ResTys, Ops.data(), NumVecs+6);
1379  // For a 64-bit vector load to D registers, nothing more needs to be done.
1380  if (is64BitVector)
1381    return VLdLn;
1382
1383  // For 128-bit vectors, take the 64-bit results of the load and insert them
1384  // as subregs into the result.
1385  for (unsigned Vec = 0; Vec < NumVecs; ++Vec) {
1386    SDValue QuadVec = CurDAG->getTargetInsertSubreg(SubregIdx, dl, VT,
1387                                                    N->getOperand(Vec+3),
1388                                                    SDValue(VLdLn, Vec));
1389    ReplaceUses(SDValue(N, Vec), QuadVec);
1390  }
1391
1392  Chain = SDValue(VLdLn, NumVecs);
1393  ReplaceUses(SDValue(N, NumVecs), Chain);
1394  return NULL;
1395}
1396
1397SDNode *ARMDAGToDAGISel::SelectV6T2BitfieldExtractOp(SDNode *N,
1398                                                     bool isSigned) {
1399  if (!Subtarget->hasV6T2Ops())
1400    return NULL;
1401
1402  unsigned Opc = isSigned ? (Subtarget->isThumb() ? ARM::t2SBFX : ARM::SBFX)
1403    : (Subtarget->isThumb() ? ARM::t2UBFX : ARM::UBFX);
1404
1405
1406  // For unsigned extracts, check for a shift right and mask
1407  unsigned And_imm = 0;
1408  if (N->getOpcode() == ISD::AND) {
1409    if (isOpcWithIntImmediate(N, ISD::AND, And_imm)) {
1410
1411      // The immediate is a mask of the low bits iff imm & (imm+1) == 0
1412      if (And_imm & (And_imm + 1))
1413        return NULL;
1414
1415      unsigned Srl_imm = 0;
1416      if (isOpcWithIntImmediate(N->getOperand(0).getNode(), ISD::SRL,
1417                                Srl_imm)) {
1418        assert(Srl_imm > 0 && Srl_imm < 32 && "bad amount in shift node!");
1419
1420        unsigned Width = CountTrailingOnes_32(And_imm);
1421        unsigned LSB = Srl_imm;
1422        SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
1423        SDValue Ops[] = { N->getOperand(0).getOperand(0),
1424                          CurDAG->getTargetConstant(LSB, MVT::i32),
1425                          CurDAG->getTargetConstant(Width, MVT::i32),
1426          getAL(CurDAG), Reg0 };
1427        return CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops, 5);
1428      }
1429    }
1430    return NULL;
1431  }
1432
1433  // Otherwise, we're looking for a shift of a shift
1434  unsigned Shl_imm = 0;
1435  if (isOpcWithIntImmediate(N->getOperand(0).getNode(), ISD::SHL, Shl_imm)) {
1436    assert(Shl_imm > 0 && Shl_imm < 32 && "bad amount in shift node!");
1437    unsigned Srl_imm = 0;
1438    if (isInt32Immediate(N->getOperand(1), Srl_imm)) {
1439      assert(Srl_imm > 0 && Srl_imm < 32 && "bad amount in shift node!");
1440      unsigned Width = 32 - Srl_imm;
1441      int LSB = Srl_imm - Shl_imm;
1442      if (LSB < 0)
1443        return NULL;
1444      SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
1445      SDValue Ops[] = { N->getOperand(0).getOperand(0),
1446                        CurDAG->getTargetConstant(LSB, MVT::i32),
1447                        CurDAG->getTargetConstant(Width, MVT::i32),
1448                        getAL(CurDAG), Reg0 };
1449      return CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops, 5);
1450    }
1451  }
1452  return NULL;
1453}
1454
1455SDNode *ARMDAGToDAGISel::
1456SelectT2CMOVShiftOp(SDNode *N, SDValue FalseVal, SDValue TrueVal,
1457                    ARMCC::CondCodes CCVal, SDValue CCR, SDValue InFlag) {
1458  SDValue CPTmp0;
1459  SDValue CPTmp1;
1460  if (SelectT2ShifterOperandReg(N, TrueVal, CPTmp0, CPTmp1)) {
1461    unsigned SOVal = cast<ConstantSDNode>(CPTmp1)->getZExtValue();
1462    unsigned SOShOp = ARM_AM::getSORegShOp(SOVal);
1463    unsigned Opc = 0;
1464    switch (SOShOp) {
1465    case ARM_AM::lsl: Opc = ARM::t2MOVCClsl; break;
1466    case ARM_AM::lsr: Opc = ARM::t2MOVCClsr; break;
1467    case ARM_AM::asr: Opc = ARM::t2MOVCCasr; break;
1468    case ARM_AM::ror: Opc = ARM::t2MOVCCror; break;
1469    default:
1470      llvm_unreachable("Unknown so_reg opcode!");
1471      break;
1472    }
1473    SDValue SOShImm =
1474      CurDAG->getTargetConstant(ARM_AM::getSORegOffset(SOVal), MVT::i32);
1475    SDValue CC = CurDAG->getTargetConstant(CCVal, MVT::i32);
1476    SDValue Ops[] = { FalseVal, CPTmp0, SOShImm, CC, CCR, InFlag };
1477    return CurDAG->SelectNodeTo(N, Opc, MVT::i32,Ops, 6);
1478  }
1479  return 0;
1480}
1481
1482SDNode *ARMDAGToDAGISel::
1483SelectARMCMOVShiftOp(SDNode *N, SDValue FalseVal, SDValue TrueVal,
1484                     ARMCC::CondCodes CCVal, SDValue CCR, SDValue InFlag) {
1485  SDValue CPTmp0;
1486  SDValue CPTmp1;
1487  SDValue CPTmp2;
1488  if (SelectShifterOperandReg(N, TrueVal, CPTmp0, CPTmp1, CPTmp2)) {
1489    SDValue CC = CurDAG->getTargetConstant(CCVal, MVT::i32);
1490    SDValue Ops[] = { FalseVal, CPTmp0, CPTmp1, CPTmp2, CC, CCR, InFlag };
1491    return CurDAG->SelectNodeTo(N, ARM::MOVCCs, MVT::i32, Ops, 7);
1492  }
1493  return 0;
1494}
1495
1496SDNode *ARMDAGToDAGISel::
1497SelectT2CMOVSoImmOp(SDNode *N, SDValue FalseVal, SDValue TrueVal,
1498                    ARMCC::CondCodes CCVal, SDValue CCR, SDValue InFlag) {
1499  ConstantSDNode *T = dyn_cast<ConstantSDNode>(TrueVal);
1500  if (!T)
1501    return 0;
1502
1503  if (Predicate_t2_so_imm(TrueVal.getNode())) {
1504    SDValue True = CurDAG->getTargetConstant(T->getZExtValue(), MVT::i32);
1505    SDValue CC = CurDAG->getTargetConstant(CCVal, MVT::i32);
1506    SDValue Ops[] = { FalseVal, True, CC, CCR, InFlag };
1507    return CurDAG->SelectNodeTo(N,
1508                                ARM::t2MOVCCi, MVT::i32, Ops, 5);
1509  }
1510  return 0;
1511}
1512
1513SDNode *ARMDAGToDAGISel::
1514SelectARMCMOVSoImmOp(SDNode *N, SDValue FalseVal, SDValue TrueVal,
1515                     ARMCC::CondCodes CCVal, SDValue CCR, SDValue InFlag) {
1516  ConstantSDNode *T = dyn_cast<ConstantSDNode>(TrueVal);
1517  if (!T)
1518    return 0;
1519
1520  if (Predicate_so_imm(TrueVal.getNode())) {
1521    SDValue True = CurDAG->getTargetConstant(T->getZExtValue(), MVT::i32);
1522    SDValue CC = CurDAG->getTargetConstant(CCVal, MVT::i32);
1523    SDValue Ops[] = { FalseVal, True, CC, CCR, InFlag };
1524    return CurDAG->SelectNodeTo(N,
1525                                ARM::MOVCCi, MVT::i32, Ops, 5);
1526  }
1527  return 0;
1528}
1529
1530SDNode *ARMDAGToDAGISel::SelectCMOVOp(SDNode *N) {
1531  EVT VT = N->getValueType(0);
1532  SDValue FalseVal = N->getOperand(0);
1533  SDValue TrueVal  = N->getOperand(1);
1534  SDValue CC = N->getOperand(2);
1535  SDValue CCR = N->getOperand(3);
1536  SDValue InFlag = N->getOperand(4);
1537  assert(CC.getOpcode() == ISD::Constant);
1538  assert(CCR.getOpcode() == ISD::Register);
1539  ARMCC::CondCodes CCVal =
1540    (ARMCC::CondCodes)cast<ConstantSDNode>(CC)->getZExtValue();
1541
1542  if (!Subtarget->isThumb1Only() && VT == MVT::i32) {
1543    // Pattern: (ARMcmov:i32 GPR:i32:$false, so_reg:i32:$true, (imm:i32):$cc)
1544    // Emits: (MOVCCs:i32 GPR:i32:$false, so_reg:i32:$true, (imm:i32):$cc)
1545    // Pattern complexity = 18  cost = 1  size = 0
1546    SDValue CPTmp0;
1547    SDValue CPTmp1;
1548    SDValue CPTmp2;
1549    if (Subtarget->isThumb()) {
1550      SDNode *Res = SelectT2CMOVShiftOp(N, FalseVal, TrueVal,
1551                                        CCVal, CCR, InFlag);
1552      if (!Res)
1553        Res = SelectT2CMOVShiftOp(N, TrueVal, FalseVal,
1554                               ARMCC::getOppositeCondition(CCVal), CCR, InFlag);
1555      if (Res)
1556        return Res;
1557    } else {
1558      SDNode *Res = SelectARMCMOVShiftOp(N, FalseVal, TrueVal,
1559                                         CCVal, CCR, InFlag);
1560      if (!Res)
1561        Res = SelectARMCMOVShiftOp(N, TrueVal, FalseVal,
1562                               ARMCC::getOppositeCondition(CCVal), CCR, InFlag);
1563      if (Res)
1564        return Res;
1565    }
1566
1567    // Pattern: (ARMcmov:i32 GPR:i32:$false,
1568    //             (imm:i32)<<P:Predicate_so_imm>>:$true,
1569    //             (imm:i32):$cc)
1570    // Emits: (MOVCCi:i32 GPR:i32:$false,
1571    //           (so_imm:i32 (imm:i32):$true), (imm:i32):$cc)
1572    // Pattern complexity = 10  cost = 1  size = 0
1573    if (Subtarget->isThumb()) {
1574      SDNode *Res = SelectT2CMOVSoImmOp(N, FalseVal, TrueVal,
1575                                        CCVal, CCR, InFlag);
1576      if (!Res)
1577        Res = SelectT2CMOVSoImmOp(N, TrueVal, FalseVal,
1578                               ARMCC::getOppositeCondition(CCVal), CCR, InFlag);
1579      if (Res)
1580        return Res;
1581    } else {
1582      SDNode *Res = SelectARMCMOVSoImmOp(N, FalseVal, TrueVal,
1583                                         CCVal, CCR, InFlag);
1584      if (!Res)
1585        Res = SelectARMCMOVSoImmOp(N, TrueVal, FalseVal,
1586                               ARMCC::getOppositeCondition(CCVal), CCR, InFlag);
1587      if (Res)
1588        return Res;
1589    }
1590  }
1591
1592  // Pattern: (ARMcmov:i32 GPR:i32:$false, GPR:i32:$true, (imm:i32):$cc)
1593  // Emits: (MOVCCr:i32 GPR:i32:$false, GPR:i32:$true, (imm:i32):$cc)
1594  // Pattern complexity = 6  cost = 1  size = 0
1595  //
1596  // Pattern: (ARMcmov:i32 GPR:i32:$false, GPR:i32:$true, (imm:i32):$cc)
1597  // Emits: (tMOVCCr:i32 GPR:i32:$false, GPR:i32:$true, (imm:i32):$cc)
1598  // Pattern complexity = 6  cost = 11  size = 0
1599  //
1600  // Also FCPYScc and FCPYDcc.
1601  SDValue Tmp2 = CurDAG->getTargetConstant(CCVal, MVT::i32);
1602  SDValue Ops[] = { FalseVal, TrueVal, Tmp2, CCR, InFlag };
1603  unsigned Opc = 0;
1604  switch (VT.getSimpleVT().SimpleTy) {
1605  default: assert(false && "Illegal conditional move type!");
1606    break;
1607  case MVT::i32:
1608    Opc = Subtarget->isThumb()
1609      ? (Subtarget->hasThumb2() ? ARM::t2MOVCCr : ARM::tMOVCCr_pseudo)
1610      : ARM::MOVCCr;
1611    break;
1612  case MVT::f32:
1613    Opc = ARM::VMOVScc;
1614    break;
1615  case MVT::f64:
1616    Opc = ARM::VMOVDcc;
1617    break;
1618  }
1619  return CurDAG->SelectNodeTo(N, Opc, VT, Ops, 5);
1620}
1621
1622SDNode *ARMDAGToDAGISel::SelectConcatVector(SDNode *N) {
1623  // The only time a CONCAT_VECTORS operation can have legal types is when
1624  // two 64-bit vectors are concatenated to a 128-bit vector.
1625  EVT VT = N->getValueType(0);
1626  if (!VT.is128BitVector() || N->getNumOperands() != 2)
1627    llvm_unreachable("unexpected CONCAT_VECTORS");
1628  DebugLoc dl = N->getDebugLoc();
1629  SDValue V0 = N->getOperand(0);
1630  SDValue V1 = N->getOperand(1);
1631  SDValue SubReg0 = CurDAG->getTargetConstant(ARM::DSUBREG_0, MVT::i32);
1632  SDValue SubReg1 = CurDAG->getTargetConstant(ARM::DSUBREG_1, MVT::i32);
1633  const SDValue Ops[] = { V0, SubReg0, V1, SubReg1 };
1634  return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops, 4);
1635}
1636
1637SDNode *ARMDAGToDAGISel::Select(SDNode *N) {
1638  DebugLoc dl = N->getDebugLoc();
1639
1640  if (N->isMachineOpcode())
1641    return NULL;   // Already selected.
1642
1643  switch (N->getOpcode()) {
1644  default: break;
1645  case ISD::Constant: {
1646    unsigned Val = cast<ConstantSDNode>(N)->getZExtValue();
1647    bool UseCP = true;
1648    if (Subtarget->hasThumb2())
1649      // Thumb2-aware targets have the MOVT instruction, so all immediates can
1650      // be done with MOV + MOVT, at worst.
1651      UseCP = 0;
1652    else {
1653      if (Subtarget->isThumb()) {
1654        UseCP = (Val > 255 &&                          // MOV
1655                 ~Val > 255 &&                         // MOV + MVN
1656                 !ARM_AM::isThumbImmShiftedVal(Val));  // MOV + LSL
1657      } else
1658        UseCP = (ARM_AM::getSOImmVal(Val) == -1 &&     // MOV
1659                 ARM_AM::getSOImmVal(~Val) == -1 &&    // MVN
1660                 !ARM_AM::isSOImmTwoPartVal(Val));     // two instrs.
1661    }
1662
1663    if (UseCP) {
1664      SDValue CPIdx =
1665        CurDAG->getTargetConstantPool(ConstantInt::get(
1666                                  Type::getInt32Ty(*CurDAG->getContext()), Val),
1667                                      TLI.getPointerTy());
1668
1669      SDNode *ResNode;
1670      if (Subtarget->isThumb1Only()) {
1671        SDValue Pred = getAL(CurDAG);
1672        SDValue PredReg = CurDAG->getRegister(0, MVT::i32);
1673        SDValue Ops[] = { CPIdx, Pred, PredReg, CurDAG->getEntryNode() };
1674        ResNode = CurDAG->getMachineNode(ARM::tLDRcp, dl, MVT::i32, MVT::Other,
1675                                         Ops, 4);
1676      } else {
1677        SDValue Ops[] = {
1678          CPIdx,
1679          CurDAG->getRegister(0, MVT::i32),
1680          CurDAG->getTargetConstant(0, MVT::i32),
1681          getAL(CurDAG),
1682          CurDAG->getRegister(0, MVT::i32),
1683          CurDAG->getEntryNode()
1684        };
1685        ResNode=CurDAG->getMachineNode(ARM::LDRcp, dl, MVT::i32, MVT::Other,
1686                                       Ops, 6);
1687      }
1688      ReplaceUses(SDValue(N, 0), SDValue(ResNode, 0));
1689      return NULL;
1690    }
1691
1692    // Other cases are autogenerated.
1693    break;
1694  }
1695  case ISD::FrameIndex: {
1696    // Selects to ADDri FI, 0 which in turn will become ADDri SP, imm.
1697    int FI = cast<FrameIndexSDNode>(N)->getIndex();
1698    SDValue TFI = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
1699    if (Subtarget->isThumb1Only()) {
1700      return CurDAG->SelectNodeTo(N, ARM::tADDrSPi, MVT::i32, TFI,
1701                                  CurDAG->getTargetConstant(0, MVT::i32));
1702    } else {
1703      unsigned Opc = ((Subtarget->isThumb() && Subtarget->hasThumb2()) ?
1704                      ARM::t2ADDri : ARM::ADDri);
1705      SDValue Ops[] = { TFI, CurDAG->getTargetConstant(0, MVT::i32),
1706                        getAL(CurDAG), CurDAG->getRegister(0, MVT::i32),
1707                        CurDAG->getRegister(0, MVT::i32) };
1708      return CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops, 5);
1709    }
1710  }
1711  case ISD::SRL:
1712    if (SDNode *I = SelectV6T2BitfieldExtractOp(N, false))
1713      return I;
1714    break;
1715  case ISD::SRA:
1716    if (SDNode *I = SelectV6T2BitfieldExtractOp(N, true))
1717      return I;
1718    break;
1719  case ISD::MUL:
1720    if (Subtarget->isThumb1Only())
1721      break;
1722    if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(N->getOperand(1))) {
1723      unsigned RHSV = C->getZExtValue();
1724      if (!RHSV) break;
1725      if (isPowerOf2_32(RHSV-1)) {  // 2^n+1?
1726        unsigned ShImm = Log2_32(RHSV-1);
1727        if (ShImm >= 32)
1728          break;
1729        SDValue V = N->getOperand(0);
1730        ShImm = ARM_AM::getSORegOpc(ARM_AM::lsl, ShImm);
1731        SDValue ShImmOp = CurDAG->getTargetConstant(ShImm, MVT::i32);
1732        SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
1733        if (Subtarget->isThumb()) {
1734          SDValue Ops[] = { V, V, ShImmOp, getAL(CurDAG), Reg0, Reg0 };
1735          return CurDAG->SelectNodeTo(N, ARM::t2ADDrs, MVT::i32, Ops, 6);
1736        } else {
1737          SDValue Ops[] = { V, V, Reg0, ShImmOp, getAL(CurDAG), Reg0, Reg0 };
1738          return CurDAG->SelectNodeTo(N, ARM::ADDrs, MVT::i32, Ops, 7);
1739        }
1740      }
1741      if (isPowerOf2_32(RHSV+1)) {  // 2^n-1?
1742        unsigned ShImm = Log2_32(RHSV+1);
1743        if (ShImm >= 32)
1744          break;
1745        SDValue V = N->getOperand(0);
1746        ShImm = ARM_AM::getSORegOpc(ARM_AM::lsl, ShImm);
1747        SDValue ShImmOp = CurDAG->getTargetConstant(ShImm, MVT::i32);
1748        SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
1749        if (Subtarget->isThumb()) {
1750          SDValue Ops[] = { V, V, ShImmOp, getAL(CurDAG), Reg0 };
1751          return CurDAG->SelectNodeTo(N, ARM::t2RSBrs, MVT::i32, Ops, 5);
1752        } else {
1753          SDValue Ops[] = { V, V, Reg0, ShImmOp, getAL(CurDAG), Reg0, Reg0 };
1754          return CurDAG->SelectNodeTo(N, ARM::RSBrs, MVT::i32, Ops, 7);
1755        }
1756      }
1757    }
1758    break;
1759  case ISD::AND: {
1760    // Check for unsigned bitfield extract
1761    if (SDNode *I = SelectV6T2BitfieldExtractOp(N, false))
1762      return I;
1763
1764    // (and (or x, c2), c1) and top 16-bits of c1 and c2 match, lower 16-bits
1765    // of c1 are 0xffff, and lower 16-bit of c2 are 0. That is, the top 16-bits
1766    // are entirely contributed by c2 and lower 16-bits are entirely contributed
1767    // by x. That's equal to (or (and x, 0xffff), (and c1, 0xffff0000)).
1768    // Select it to: "movt x, ((c1 & 0xffff) >> 16)
1769    EVT VT = N->getValueType(0);
1770    if (VT != MVT::i32)
1771      break;
1772    unsigned Opc = (Subtarget->isThumb() && Subtarget->hasThumb2())
1773      ? ARM::t2MOVTi16
1774      : (Subtarget->hasV6T2Ops() ? ARM::MOVTi16 : 0);
1775    if (!Opc)
1776      break;
1777    SDValue N0 = N->getOperand(0), N1 = N->getOperand(1);
1778    ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
1779    if (!N1C)
1780      break;
1781    if (N0.getOpcode() == ISD::OR && N0.getNode()->hasOneUse()) {
1782      SDValue N2 = N0.getOperand(1);
1783      ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(N2);
1784      if (!N2C)
1785        break;
1786      unsigned N1CVal = N1C->getZExtValue();
1787      unsigned N2CVal = N2C->getZExtValue();
1788      if ((N1CVal & 0xffff0000U) == (N2CVal & 0xffff0000U) &&
1789          (N1CVal & 0xffffU) == 0xffffU &&
1790          (N2CVal & 0xffffU) == 0x0U) {
1791        SDValue Imm16 = CurDAG->getTargetConstant((N2CVal & 0xFFFF0000U) >> 16,
1792                                                  MVT::i32);
1793        SDValue Ops[] = { N0.getOperand(0), Imm16,
1794                          getAL(CurDAG), CurDAG->getRegister(0, MVT::i32) };
1795        return CurDAG->getMachineNode(Opc, dl, VT, Ops, 4);
1796      }
1797    }
1798    break;
1799  }
1800  case ARMISD::VMOVRRD:
1801    return CurDAG->getMachineNode(ARM::VMOVRRD, dl, MVT::i32, MVT::i32,
1802                                  N->getOperand(0), getAL(CurDAG),
1803                                  CurDAG->getRegister(0, MVT::i32));
1804  case ISD::UMUL_LOHI: {
1805    if (Subtarget->isThumb1Only())
1806      break;
1807    if (Subtarget->isThumb()) {
1808      SDValue Ops[] = { N->getOperand(0), N->getOperand(1),
1809                        getAL(CurDAG), CurDAG->getRegister(0, MVT::i32),
1810                        CurDAG->getRegister(0, MVT::i32) };
1811      return CurDAG->getMachineNode(ARM::t2UMULL, dl, MVT::i32, MVT::i32, Ops,4);
1812    } else {
1813      SDValue Ops[] = { N->getOperand(0), N->getOperand(1),
1814                        getAL(CurDAG), CurDAG->getRegister(0, MVT::i32),
1815                        CurDAG->getRegister(0, MVT::i32) };
1816      return CurDAG->getMachineNode(ARM::UMULL, dl, MVT::i32, MVT::i32, Ops, 5);
1817    }
1818  }
1819  case ISD::SMUL_LOHI: {
1820    if (Subtarget->isThumb1Only())
1821      break;
1822    if (Subtarget->isThumb()) {
1823      SDValue Ops[] = { N->getOperand(0), N->getOperand(1),
1824                        getAL(CurDAG), CurDAG->getRegister(0, MVT::i32) };
1825      return CurDAG->getMachineNode(ARM::t2SMULL, dl, MVT::i32, MVT::i32, Ops,4);
1826    } else {
1827      SDValue Ops[] = { N->getOperand(0), N->getOperand(1),
1828                        getAL(CurDAG), CurDAG->getRegister(0, MVT::i32),
1829                        CurDAG->getRegister(0, MVT::i32) };
1830      return CurDAG->getMachineNode(ARM::SMULL, dl, MVT::i32, MVT::i32, Ops, 5);
1831    }
1832  }
1833  case ISD::LOAD: {
1834    SDNode *ResNode = 0;
1835    if (Subtarget->isThumb() && Subtarget->hasThumb2())
1836      ResNode = SelectT2IndexedLoad(N);
1837    else
1838      ResNode = SelectARMIndexedLoad(N);
1839    if (ResNode)
1840      return ResNode;
1841
1842    // VLDMQ must be custom-selected for "v2f64 load" to set the AM5Opc value.
1843    if (Subtarget->hasVFP2() &&
1844        N->getValueType(0).getSimpleVT().SimpleTy == MVT::v2f64) {
1845      SDValue Chain = N->getOperand(0);
1846      SDValue AM5Opc =
1847        CurDAG->getTargetConstant(ARM_AM::getAM5Opc(ARM_AM::ia, 4), MVT::i32);
1848      SDValue Pred = getAL(CurDAG);
1849      SDValue PredReg = CurDAG->getRegister(0, MVT::i32);
1850      SDValue Ops[] = { N->getOperand(1), AM5Opc, Pred, PredReg, Chain };
1851      return CurDAG->getMachineNode(ARM::VLDMQ, dl, MVT::v2f64, MVT::Other,
1852                                    Ops, 5);
1853    }
1854    // Other cases are autogenerated.
1855    break;
1856  }
1857  case ISD::STORE: {
1858    // VSTMQ must be custom-selected for "v2f64 store" to set the AM5Opc value.
1859    if (Subtarget->hasVFP2() &&
1860        N->getOperand(1).getValueType().getSimpleVT().SimpleTy == MVT::v2f64) {
1861      SDValue Chain = N->getOperand(0);
1862      SDValue AM5Opc =
1863        CurDAG->getTargetConstant(ARM_AM::getAM5Opc(ARM_AM::ia, 4), MVT::i32);
1864      SDValue Pred = getAL(CurDAG);
1865      SDValue PredReg = CurDAG->getRegister(0, MVT::i32);
1866      SDValue Ops[] = { N->getOperand(1), N->getOperand(2),
1867                        AM5Opc, Pred, PredReg, Chain };
1868      return CurDAG->getMachineNode(ARM::VSTMQ, dl, MVT::Other, Ops, 6);
1869    }
1870    // Other cases are autogenerated.
1871    break;
1872  }
1873  case ARMISD::BRCOND: {
1874    // Pattern: (ARMbrcond:void (bb:Other):$dst, (imm:i32):$cc)
1875    // Emits: (Bcc:void (bb:Other):$dst, (imm:i32):$cc)
1876    // Pattern complexity = 6  cost = 1  size = 0
1877
1878    // Pattern: (ARMbrcond:void (bb:Other):$dst, (imm:i32):$cc)
1879    // Emits: (tBcc:void (bb:Other):$dst, (imm:i32):$cc)
1880    // Pattern complexity = 6  cost = 1  size = 0
1881
1882    // Pattern: (ARMbrcond:void (bb:Other):$dst, (imm:i32):$cc)
1883    // Emits: (t2Bcc:void (bb:Other):$dst, (imm:i32):$cc)
1884    // Pattern complexity = 6  cost = 1  size = 0
1885
1886    unsigned Opc = Subtarget->isThumb() ?
1887      ((Subtarget->hasThumb2()) ? ARM::t2Bcc : ARM::tBcc) : ARM::Bcc;
1888    SDValue Chain = N->getOperand(0);
1889    SDValue N1 = N->getOperand(1);
1890    SDValue N2 = N->getOperand(2);
1891    SDValue N3 = N->getOperand(3);
1892    SDValue InFlag = N->getOperand(4);
1893    assert(N1.getOpcode() == ISD::BasicBlock);
1894    assert(N2.getOpcode() == ISD::Constant);
1895    assert(N3.getOpcode() == ISD::Register);
1896
1897    SDValue Tmp2 = CurDAG->getTargetConstant(((unsigned)
1898                               cast<ConstantSDNode>(N2)->getZExtValue()),
1899                               MVT::i32);
1900    SDValue Ops[] = { N1, Tmp2, N3, Chain, InFlag };
1901    SDNode *ResNode = CurDAG->getMachineNode(Opc, dl, MVT::Other,
1902                                             MVT::Flag, Ops, 5);
1903    Chain = SDValue(ResNode, 0);
1904    if (N->getNumValues() == 2) {
1905      InFlag = SDValue(ResNode, 1);
1906      ReplaceUses(SDValue(N, 1), InFlag);
1907    }
1908    ReplaceUses(SDValue(N, 0),
1909                SDValue(Chain.getNode(), Chain.getResNo()));
1910    return NULL;
1911  }
1912  case ARMISD::CMOV:
1913    return SelectCMOVOp(N);
1914  case ARMISD::CNEG: {
1915    EVT VT = N->getValueType(0);
1916    SDValue N0 = N->getOperand(0);
1917    SDValue N1 = N->getOperand(1);
1918    SDValue N2 = N->getOperand(2);
1919    SDValue N3 = N->getOperand(3);
1920    SDValue InFlag = N->getOperand(4);
1921    assert(N2.getOpcode() == ISD::Constant);
1922    assert(N3.getOpcode() == ISD::Register);
1923
1924    SDValue Tmp2 = CurDAG->getTargetConstant(((unsigned)
1925                               cast<ConstantSDNode>(N2)->getZExtValue()),
1926                               MVT::i32);
1927    SDValue Ops[] = { N0, N1, Tmp2, N3, InFlag };
1928    unsigned Opc = 0;
1929    switch (VT.getSimpleVT().SimpleTy) {
1930    default: assert(false && "Illegal conditional move type!");
1931      break;
1932    case MVT::f32:
1933      Opc = ARM::VNEGScc;
1934      break;
1935    case MVT::f64:
1936      Opc = ARM::VNEGDcc;
1937      break;
1938    }
1939    return CurDAG->SelectNodeTo(N, Opc, VT, Ops, 5);
1940  }
1941
1942  case ARMISD::VZIP: {
1943    unsigned Opc = 0;
1944    EVT VT = N->getValueType(0);
1945    switch (VT.getSimpleVT().SimpleTy) {
1946    default: return NULL;
1947    case MVT::v8i8:  Opc = ARM::VZIPd8; break;
1948    case MVT::v4i16: Opc = ARM::VZIPd16; break;
1949    case MVT::v2f32:
1950    case MVT::v2i32: Opc = ARM::VZIPd32; break;
1951    case MVT::v16i8: Opc = ARM::VZIPq8; break;
1952    case MVT::v8i16: Opc = ARM::VZIPq16; break;
1953    case MVT::v4f32:
1954    case MVT::v4i32: Opc = ARM::VZIPq32; break;
1955    }
1956    SDValue Pred = getAL(CurDAG);
1957    SDValue PredReg = CurDAG->getRegister(0, MVT::i32);
1958    SDValue Ops[] = { N->getOperand(0), N->getOperand(1), Pred, PredReg };
1959    return CurDAG->getMachineNode(Opc, dl, VT, VT, Ops, 4);
1960  }
1961  case ARMISD::VUZP: {
1962    unsigned Opc = 0;
1963    EVT VT = N->getValueType(0);
1964    switch (VT.getSimpleVT().SimpleTy) {
1965    default: return NULL;
1966    case MVT::v8i8:  Opc = ARM::VUZPd8; break;
1967    case MVT::v4i16: Opc = ARM::VUZPd16; break;
1968    case MVT::v2f32:
1969    case MVT::v2i32: Opc = ARM::VUZPd32; break;
1970    case MVT::v16i8: Opc = ARM::VUZPq8; break;
1971    case MVT::v8i16: Opc = ARM::VUZPq16; break;
1972    case MVT::v4f32:
1973    case MVT::v4i32: Opc = ARM::VUZPq32; break;
1974    }
1975    SDValue Pred = getAL(CurDAG);
1976    SDValue PredReg = CurDAG->getRegister(0, MVT::i32);
1977    SDValue Ops[] = { N->getOperand(0), N->getOperand(1), Pred, PredReg };
1978    return CurDAG->getMachineNode(Opc, dl, VT, VT, Ops, 4);
1979  }
1980  case ARMISD::VTRN: {
1981    unsigned Opc = 0;
1982    EVT VT = N->getValueType(0);
1983    switch (VT.getSimpleVT().SimpleTy) {
1984    default: return NULL;
1985    case MVT::v8i8:  Opc = ARM::VTRNd8; break;
1986    case MVT::v4i16: Opc = ARM::VTRNd16; break;
1987    case MVT::v2f32:
1988    case MVT::v2i32: Opc = ARM::VTRNd32; break;
1989    case MVT::v16i8: Opc = ARM::VTRNq8; break;
1990    case MVT::v8i16: Opc = ARM::VTRNq16; break;
1991    case MVT::v4f32:
1992    case MVT::v4i32: Opc = ARM::VTRNq32; break;
1993    }
1994    SDValue Pred = getAL(CurDAG);
1995    SDValue PredReg = CurDAG->getRegister(0, MVT::i32);
1996    SDValue Ops[] = { N->getOperand(0), N->getOperand(1), Pred, PredReg };
1997    return CurDAG->getMachineNode(Opc, dl, VT, VT, Ops, 4);
1998  }
1999
2000  case ISD::INTRINSIC_VOID:
2001  case ISD::INTRINSIC_W_CHAIN: {
2002    unsigned IntNo = cast<ConstantSDNode>(N->getOperand(1))->getZExtValue();
2003    switch (IntNo) {
2004    default:
2005      break;
2006
2007    case Intrinsic::arm_neon_vld1: {
2008      unsigned DOpcodes[] = { ARM::VLD1d8, ARM::VLD1d16,
2009                              ARM::VLD1d32, ARM::VLD1d64 };
2010      unsigned QOpcodes[] = { ARM::VLD1q8, ARM::VLD1q16,
2011                              ARM::VLD1q32, ARM::VLD1q64 };
2012      return SelectVLD(N, 1, DOpcodes, QOpcodes, 0);
2013    }
2014
2015    case Intrinsic::arm_neon_vld2: {
2016      unsigned DOpcodes[] = { ARM::VLD2d8, ARM::VLD2d16,
2017                              ARM::VLD2d32, ARM::VLD1q64 };
2018      unsigned QOpcodes[] = { ARM::VLD2q8, ARM::VLD2q16, ARM::VLD2q32 };
2019      return SelectVLD(N, 2, DOpcodes, QOpcodes, 0);
2020    }
2021
2022    case Intrinsic::arm_neon_vld3: {
2023      unsigned DOpcodes[] = { ARM::VLD3d8, ARM::VLD3d16,
2024                              ARM::VLD3d32, ARM::VLD1d64T };
2025      unsigned QOpcodes0[] = { ARM::VLD3q8_UPD,
2026                               ARM::VLD3q16_UPD,
2027                               ARM::VLD3q32_UPD };
2028      unsigned QOpcodes1[] = { ARM::VLD3q8odd_UPD,
2029                               ARM::VLD3q16odd_UPD,
2030                               ARM::VLD3q32odd_UPD };
2031      return SelectVLD(N, 3, DOpcodes, QOpcodes0, QOpcodes1);
2032    }
2033
2034    case Intrinsic::arm_neon_vld4: {
2035      unsigned DOpcodes[] = { ARM::VLD4d8, ARM::VLD4d16,
2036                              ARM::VLD4d32, ARM::VLD1d64Q };
2037      unsigned QOpcodes0[] = { ARM::VLD4q8_UPD,
2038                               ARM::VLD4q16_UPD,
2039                               ARM::VLD4q32_UPD };
2040      unsigned QOpcodes1[] = { ARM::VLD4q8odd_UPD,
2041                               ARM::VLD4q16odd_UPD,
2042                               ARM::VLD4q32odd_UPD };
2043      return SelectVLD(N, 4, DOpcodes, QOpcodes0, QOpcodes1);
2044    }
2045
2046    case Intrinsic::arm_neon_vld2lane: {
2047      unsigned DOpcodes[] = { ARM::VLD2LNd8, ARM::VLD2LNd16, ARM::VLD2LNd32 };
2048      unsigned QOpcodes0[] = { ARM::VLD2LNq16, ARM::VLD2LNq32 };
2049      unsigned QOpcodes1[] = { ARM::VLD2LNq16odd, ARM::VLD2LNq32odd };
2050      return SelectVLDSTLane(N, true, 2, DOpcodes, QOpcodes0, QOpcodes1);
2051    }
2052
2053    case Intrinsic::arm_neon_vld3lane: {
2054      unsigned DOpcodes[] = { ARM::VLD3LNd8, ARM::VLD3LNd16, ARM::VLD3LNd32 };
2055      unsigned QOpcodes0[] = { ARM::VLD3LNq16, ARM::VLD3LNq32 };
2056      unsigned QOpcodes1[] = { ARM::VLD3LNq16odd, ARM::VLD3LNq32odd };
2057      return SelectVLDSTLane(N, true, 3, DOpcodes, QOpcodes0, QOpcodes1);
2058    }
2059
2060    case Intrinsic::arm_neon_vld4lane: {
2061      unsigned DOpcodes[] = { ARM::VLD4LNd8, ARM::VLD4LNd16, ARM::VLD4LNd32 };
2062      unsigned QOpcodes0[] = { ARM::VLD4LNq16, ARM::VLD4LNq32 };
2063      unsigned QOpcodes1[] = { ARM::VLD4LNq16odd, ARM::VLD4LNq32odd };
2064      return SelectVLDSTLane(N, true, 4, DOpcodes, QOpcodes0, QOpcodes1);
2065    }
2066
2067    case Intrinsic::arm_neon_vst1: {
2068      unsigned DOpcodes[] = { ARM::VST1d8, ARM::VST1d16,
2069                              ARM::VST1d32, ARM::VST1d64 };
2070      unsigned QOpcodes[] = { ARM::VST1q8, ARM::VST1q16,
2071                              ARM::VST1q32, ARM::VST1q64 };
2072      return SelectVST(N, 1, DOpcodes, QOpcodes, 0);
2073    }
2074
2075    case Intrinsic::arm_neon_vst2: {
2076      unsigned DOpcodes[] = { ARM::VST2d8, ARM::VST2d16,
2077                              ARM::VST2d32, ARM::VST1q64 };
2078      unsigned QOpcodes[] = { ARM::VST2q8, ARM::VST2q16, ARM::VST2q32 };
2079      return SelectVST(N, 2, DOpcodes, QOpcodes, 0);
2080    }
2081
2082    case Intrinsic::arm_neon_vst3: {
2083      unsigned DOpcodes[] = { ARM::VST3d8, ARM::VST3d16,
2084                              ARM::VST3d32, ARM::VST1d64T };
2085      unsigned QOpcodes0[] = { ARM::VST3q8_UPD,
2086                               ARM::VST3q16_UPD,
2087                               ARM::VST3q32_UPD };
2088      unsigned QOpcodes1[] = { ARM::VST3q8odd_UPD,
2089                               ARM::VST3q16odd_UPD,
2090                               ARM::VST3q32odd_UPD };
2091      return SelectVST(N, 3, DOpcodes, QOpcodes0, QOpcodes1);
2092    }
2093
2094    case Intrinsic::arm_neon_vst4: {
2095      unsigned DOpcodes[] = { ARM::VST4d8, ARM::VST4d16,
2096                              ARM::VST4d32, ARM::VST1d64Q };
2097      unsigned QOpcodes0[] = { ARM::VST4q8_UPD,
2098                               ARM::VST4q16_UPD,
2099                               ARM::VST4q32_UPD };
2100      unsigned QOpcodes1[] = { ARM::VST4q8odd_UPD,
2101                               ARM::VST4q16odd_UPD,
2102                               ARM::VST4q32odd_UPD };
2103      return SelectVST(N, 4, DOpcodes, QOpcodes0, QOpcodes1);
2104    }
2105
2106    case Intrinsic::arm_neon_vst2lane: {
2107      unsigned DOpcodes[] = { ARM::VST2LNd8, ARM::VST2LNd16, ARM::VST2LNd32 };
2108      unsigned QOpcodes0[] = { ARM::VST2LNq16, ARM::VST2LNq32 };
2109      unsigned QOpcodes1[] = { ARM::VST2LNq16odd, ARM::VST2LNq32odd };
2110      return SelectVLDSTLane(N, false, 2, DOpcodes, QOpcodes0, QOpcodes1);
2111    }
2112
2113    case Intrinsic::arm_neon_vst3lane: {
2114      unsigned DOpcodes[] = { ARM::VST3LNd8, ARM::VST3LNd16, ARM::VST3LNd32 };
2115      unsigned QOpcodes0[] = { ARM::VST3LNq16, ARM::VST3LNq32 };
2116      unsigned QOpcodes1[] = { ARM::VST3LNq16odd, ARM::VST3LNq32odd };
2117      return SelectVLDSTLane(N, false, 3, DOpcodes, QOpcodes0, QOpcodes1);
2118    }
2119
2120    case Intrinsic::arm_neon_vst4lane: {
2121      unsigned DOpcodes[] = { ARM::VST4LNd8, ARM::VST4LNd16, ARM::VST4LNd32 };
2122      unsigned QOpcodes0[] = { ARM::VST4LNq16, ARM::VST4LNq32 };
2123      unsigned QOpcodes1[] = { ARM::VST4LNq16odd, ARM::VST4LNq32odd };
2124      return SelectVLDSTLane(N, false, 4, DOpcodes, QOpcodes0, QOpcodes1);
2125    }
2126    }
2127    break;
2128  }
2129
2130  case ISD::CONCAT_VECTORS:
2131    return SelectConcatVector(N);
2132  }
2133
2134  return SelectCode(N);
2135}
2136
2137bool ARMDAGToDAGISel::
2138SelectInlineAsmMemoryOperand(const SDValue &Op, char ConstraintCode,
2139                             std::vector<SDValue> &OutOps) {
2140  assert(ConstraintCode == 'm' && "unexpected asm memory constraint");
2141  // Require the address to be in a register.  That is safe for all ARM
2142  // variants and it is hard to do anything much smarter without knowing
2143  // how the operand is used.
2144  OutOps.push_back(Op);
2145  return false;
2146}
2147
2148/// createARMISelDag - This pass converts a legalized DAG into a
2149/// ARM-specific DAG, ready for instruction scheduling.
2150///
2151FunctionPass *llvm::createARMISelDag(ARMBaseTargetMachine &TM,
2152                                     CodeGenOpt::Level OptLevel) {
2153  return new ARMDAGToDAGISel(TM, OptLevel);
2154}
2155
2156/// ModelWithRegSequence - Return true if isel should use REG_SEQUENCE to model
2157/// operations involving sub-registers.
2158bool llvm::ModelWithRegSequence() {
2159  return UseRegSeq;
2160}
2161