ARMISelDAGToDAG.cpp revision 8466fa1842ad4f2d6fadcf5c23c15319ae96b972 
1//===-- ARMISelDAGToDAG.cpp - A dag to dag inst selector for ARM ----------===//
2//
3//                     The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10// This file defines an instruction selector for the ARM target.
11//
12//===----------------------------------------------------------------------===//
13
14#define DEBUG_TYPE "arm-isel"
15#include "ARM.h"
16#include "ARMAddressingModes.h"
17#include "ARMTargetMachine.h"
18#include "llvm/CallingConv.h"
19#include "llvm/Constants.h"
20#include "llvm/DerivedTypes.h"
21#include "llvm/Function.h"
22#include "llvm/Intrinsics.h"
23#include "llvm/LLVMContext.h"
24#include "llvm/CodeGen/MachineFrameInfo.h"
25#include "llvm/CodeGen/MachineFunction.h"
26#include "llvm/CodeGen/MachineInstrBuilder.h"
27#include "llvm/CodeGen/SelectionDAG.h"
28#include "llvm/CodeGen/SelectionDAGISel.h"
29#include "llvm/Target/TargetLowering.h"
30#include "llvm/Target/TargetOptions.h"
31#include "llvm/Support/CommandLine.h"
32#include "llvm/Support/Compiler.h"
33#include "llvm/Support/Debug.h"
34#include "llvm/Support/ErrorHandling.h"
35#include "llvm/Support/raw_ostream.h"
36
37using namespace llvm;
38
39static cl::opt<bool>
40DisableShifterOp("disable-shifter-op", cl::Hidden,
41  cl::desc("Disable isel of shifter-op"),
42  cl::init(false));
43
44//===--------------------------------------------------------------------===//
45/// ARMDAGToDAGISel - ARM specific code to select ARM machine
46/// instructions for SelectionDAG operations.
47///
48namespace {
49class ARMDAGToDAGISel : public SelectionDAGISel {
50  ARMBaseTargetMachine &TM;
51
52  /// Subtarget - Keep a pointer to the ARMSubtarget around so that we can
53  /// make the right decision when generating code for different targets.
54  const ARMSubtarget *Subtarget;
55
56public:
57  explicit ARMDAGToDAGISel(ARMBaseTargetMachine &tm,
58                           CodeGenOpt::Level OptLevel)
59    : SelectionDAGISel(tm, OptLevel), TM(tm),
60    Subtarget(&TM.getSubtarget<ARMSubtarget>()) {
61  }
62
63  virtual const char *getPassName() const {
64    return "ARM Instruction Selection";
65  }
66
67  /// getI32Imm - Return a target constant of type i32 with the specified
68  /// value.
69  inline SDValue getI32Imm(unsigned Imm) {
70    return CurDAG->getTargetConstant(Imm, MVT::i32);
71  }
72
73  SDNode *Select(SDNode *N);
74
75  bool SelectShifterOperandReg(SDNode *Op, SDValue N, SDValue &A,
76                               SDValue &B, SDValue &C);
77  bool SelectAddrMode2(SDNode *Op, SDValue N, SDValue &Base,
78                       SDValue &Offset, SDValue &Opc);
79  bool SelectAddrMode2Offset(SDNode *Op, SDValue N,
80                             SDValue &Offset, SDValue &Opc);
81  bool SelectAddrMode3(SDNode *Op, SDValue N, SDValue &Base,
82                       SDValue &Offset, SDValue &Opc);
83  bool SelectAddrMode3Offset(SDNode *Op, SDValue N,
84                             SDValue &Offset, SDValue &Opc);
85  bool SelectAddrMode4(SDNode *Op, SDValue N, SDValue &Addr,
86                       SDValue &Mode);
87  bool SelectAddrMode5(SDNode *Op, SDValue N, SDValue &Base,
88                       SDValue &Offset);
89  bool SelectAddrMode6(SDNode *Op, SDValue N, SDValue &Addr, SDValue &Align);
90
91  bool SelectAddrModePC(SDNode *Op, SDValue N, SDValue &Offset,
92                        SDValue &Label);
93
94  bool SelectThumbAddrModeRR(SDNode *Op, SDValue N, SDValue &Base,
95                             SDValue &Offset);
96  bool SelectThumbAddrModeRI5(SDNode *Op, SDValue N, unsigned Scale,
97                              SDValue &Base, SDValue &OffImm,
98                              SDValue &Offset);
99  bool SelectThumbAddrModeS1(SDNode *Op, SDValue N, SDValue &Base,
100                             SDValue &OffImm, SDValue &Offset);
101  bool SelectThumbAddrModeS2(SDNode *Op, SDValue N, SDValue &Base,
102                             SDValue &OffImm, SDValue &Offset);
103  bool SelectThumbAddrModeS4(SDNode *Op, SDValue N, SDValue &Base,
104                             SDValue &OffImm, SDValue &Offset);
105  bool SelectThumbAddrModeSP(SDNode *Op, SDValue N, SDValue &Base,
106                             SDValue &OffImm);
107
108  bool SelectT2ShifterOperandReg(SDNode *Op, SDValue N,
109                                 SDValue &BaseReg, SDValue &Opc);
110  bool SelectT2AddrModeImm12(SDNode *Op, SDValue N, SDValue &Base,
111                             SDValue &OffImm);
112  bool SelectT2AddrModeImm8(SDNode *Op, SDValue N, SDValue &Base,
113                            SDValue &OffImm);
114  bool SelectT2AddrModeImm8Offset(SDNode *Op, SDValue N,
115                                 SDValue &OffImm);
116  bool SelectT2AddrModeSoReg(SDNode *Op, SDValue N, SDValue &Base,
117                             SDValue &OffReg, SDValue &ShImm);
118
119  inline bool Pred_so_imm(SDNode *inN) const {
120    ConstantSDNode *N = cast<ConstantSDNode>(inN);
121    return ARM_AM::getSOImmVal(N->getZExtValue()) != -1;
122  }
123
124  inline bool Pred_t2_so_imm(SDNode *inN) const {
125    ConstantSDNode *N = cast<ConstantSDNode>(inN);
126    return ARM_AM::getT2SOImmVal(N->getZExtValue()) != -1;
127  }
128
129  // Include the pieces autogenerated from the target description.
130#include "ARMGenDAGISel.inc"
131
132private:
133  /// SelectARMIndexedLoad - Indexed (pre/post inc/dec) load matching code for
134  /// ARM.
135  SDNode *SelectARMIndexedLoad(SDNode *N);
136  SDNode *SelectT2IndexedLoad(SDNode *N);
137
138  /// SelectVLD - Select NEON load intrinsics.  NumVecs should be
139  /// 1, 2, 3 or 4.  The opcode arrays specify the instructions used for
140  /// loads of D registers and even subregs and odd subregs of Q registers.
141  /// For NumVecs <= 2, QOpcodes1 is not used.
142  SDNode *SelectVLD(SDNode *N, unsigned NumVecs, unsigned *DOpcodes,
143                    unsigned *QOpcodes0, unsigned *QOpcodes1);
144
145  /// SelectVST - Select NEON store intrinsics.  NumVecs should
146  /// be 1, 2, 3 or 4.  The opcode arrays specify the instructions used for
147  /// stores of D registers and even subregs and odd subregs of Q registers.
148  /// For NumVecs <= 2, QOpcodes1 is not used.
149  SDNode *SelectVST(SDNode *N, unsigned NumVecs, unsigned *DOpcodes,
150                    unsigned *QOpcodes0, unsigned *QOpcodes1);
151
152  /// SelectVLDSTLane - Select NEON load/store lane intrinsics.  NumVecs should
153  /// be 2, 3 or 4.  The opcode arrays specify the instructions used for
154  /// load/store of D registers and Q registers.
155  SDNode *SelectVLDSTLane(SDNode *N, bool IsLoad, unsigned NumVecs,
156                          unsigned *DOpcodes, unsigned *QOpcodes);
157
158  /// SelectVTBL - Select NEON VTBL and VTBX intrinsics.  NumVecs should be 2,
159  /// 3 or 4.  These are custom-selected so that a REG_SEQUENCE can be
160  /// generated to force the table registers to be consecutive.
161  SDNode *SelectVTBL(SDNode *N, bool IsExt, unsigned NumVecs, unsigned Opc);
162
163  /// SelectV6T2BitfieldExtractOp - Select SBFX/UBFX instructions for ARM.
164  SDNode *SelectV6T2BitfieldExtractOp(SDNode *N, bool isSigned);
165
166  /// SelectCMOVOp - Select CMOV instructions for ARM.
167  SDNode *SelectCMOVOp(SDNode *N);
168  SDNode *SelectT2CMOVShiftOp(SDNode *N, SDValue FalseVal, SDValue TrueVal,
169                              ARMCC::CondCodes CCVal, SDValue CCR,
170                              SDValue InFlag);
171  SDNode *SelectARMCMOVShiftOp(SDNode *N, SDValue FalseVal, SDValue TrueVal,
172                               ARMCC::CondCodes CCVal, SDValue CCR,
173                               SDValue InFlag);
174  SDNode *SelectT2CMOVSoImmOp(SDNode *N, SDValue FalseVal, SDValue TrueVal,
175                              ARMCC::CondCodes CCVal, SDValue CCR,
176                              SDValue InFlag);
177  SDNode *SelectARMCMOVSoImmOp(SDNode *N, SDValue FalseVal, SDValue TrueVal,
178                               ARMCC::CondCodes CCVal, SDValue CCR,
179                               SDValue InFlag);
180
181  SDNode *SelectConcatVector(SDNode *N);
182
183  /// SelectInlineAsmMemoryOperand - Implement addressing mode selection for
184  /// inline asm expressions.
185  virtual bool SelectInlineAsmMemoryOperand(const SDValue &Op,
186                                            char ConstraintCode,
187                                            std::vector<SDValue> &OutOps);
188
189  // Form pairs of consecutive S, D, or Q registers.
190  SDNode *PairSRegs(EVT VT, SDValue V0, SDValue V1);
191  SDNode *PairDRegs(EVT VT, SDValue V0, SDValue V1);
192  SDNode *PairQRegs(EVT VT, SDValue V0, SDValue V1);
193
194  // Form sequences of 4 consecutive S, D, or Q registers.
195  SDNode *QuadSRegs(EVT VT, SDValue V0, SDValue V1, SDValue V2, SDValue V3);
196  SDNode *QuadDRegs(EVT VT, SDValue V0, SDValue V1, SDValue V2, SDValue V3);
197  SDNode *QuadQRegs(EVT VT, SDValue V0, SDValue V1, SDValue V2, SDValue V3);
198};
199}
200
201/// isInt32Immediate - This method tests to see if the node is a 32-bit constant
202/// operand. If so Imm will receive the 32-bit value.
203static bool isInt32Immediate(SDNode *N, unsigned &Imm) {
204  if (N->getOpcode() == ISD::Constant && N->getValueType(0) == MVT::i32) {
205    Imm = cast<ConstantSDNode>(N)->getZExtValue();
206    return true;
207  }
208  return false;
209}
210
211// isInt32Immediate - This method tests to see if a constant operand.
212// If so Imm will receive the 32 bit value.
213static bool isInt32Immediate(SDValue N, unsigned &Imm) {
214  return isInt32Immediate(N.getNode(), Imm);
215}
216
217// isOpcWithIntImmediate - This method tests to see if the node is a specific
218// opcode and that it has a immediate integer right operand.
219// If so Imm will receive the 32 bit value.
220static bool isOpcWithIntImmediate(SDNode *N, unsigned Opc, unsigned& Imm) {
221  return N->getOpcode() == Opc &&
222         isInt32Immediate(N->getOperand(1).getNode(), Imm);
223}
224
225
226bool ARMDAGToDAGISel::SelectShifterOperandReg(SDNode *Op,
227                                              SDValue N,
228                                              SDValue &BaseReg,
229                                              SDValue &ShReg,
230                                              SDValue &Opc) {
231  if (DisableShifterOp)
232    return false;
233
234  ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(N);
235
236  // Don't match base register only case. That is matched to a separate
237  // lower complexity pattern with explicit register operand.
238  if (ShOpcVal == ARM_AM::no_shift) return false;
239
240  BaseReg = N.getOperand(0);
241  unsigned ShImmVal = 0;
242  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
243    ShReg = CurDAG->getRegister(0, MVT::i32);
244    ShImmVal = RHS->getZExtValue() & 31;
245  } else {
246    ShReg = N.getOperand(1);
247  }
248  Opc = CurDAG->getTargetConstant(ARM_AM::getSORegOpc(ShOpcVal, ShImmVal),
249                                  MVT::i32);
250  return true;
251}
252
253bool ARMDAGToDAGISel::SelectAddrMode2(SDNode *Op, SDValue N,
254                                      SDValue &Base, SDValue &Offset,
255                                      SDValue &Opc) {
256  if (N.getOpcode() == ISD::MUL) {
257    if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
258      // X * [3,5,9] -> X + X * [2,4,8] etc.
259      int RHSC = (int)RHS->getZExtValue();
260      if (RHSC & 1) {
261        RHSC = RHSC & ~1;
262        ARM_AM::AddrOpc AddSub = ARM_AM::add;
263        if (RHSC < 0) {
264          AddSub = ARM_AM::sub;
265          RHSC = - RHSC;
266        }
267        if (isPowerOf2_32(RHSC)) {
268          unsigned ShAmt = Log2_32(RHSC);
269          Base = Offset = N.getOperand(0);
270          Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, ShAmt,
271                                                            ARM_AM::lsl),
272                                          MVT::i32);
273          return true;
274        }
275      }
276    }
277  }
278
279  if (N.getOpcode() != ISD::ADD && N.getOpcode() != ISD::SUB) {
280    Base = N;
281    if (N.getOpcode() == ISD::FrameIndex) {
282      int FI = cast<FrameIndexSDNode>(N)->getIndex();
283      Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
284    } else if (N.getOpcode() == ARMISD::Wrapper &&
285               !(Subtarget->useMovt() &&
286                 N.getOperand(0).getOpcode() == ISD::TargetGlobalAddress)) {
287      Base = N.getOperand(0);
288    }
289    Offset = CurDAG->getRegister(0, MVT::i32);
290    Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(ARM_AM::add, 0,
291                                                      ARM_AM::no_shift),
292                                    MVT::i32);
293    return true;
294  }
295
296  // Match simple R +/- imm12 operands.
297  if (N.getOpcode() == ISD::ADD)
298    if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
299      int RHSC = (int)RHS->getZExtValue();
300      if ((RHSC >= 0 && RHSC < 0x1000) ||
301          (RHSC < 0 && RHSC > -0x1000)) { // 12 bits.
302        Base = N.getOperand(0);
303        if (Base.getOpcode() == ISD::FrameIndex) {
304          int FI = cast<FrameIndexSDNode>(Base)->getIndex();
305          Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
306        }
307        Offset = CurDAG->getRegister(0, MVT::i32);
308
309        ARM_AM::AddrOpc AddSub = ARM_AM::add;
310        if (RHSC < 0) {
311          AddSub = ARM_AM::sub;
312          RHSC = - RHSC;
313        }
314        Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, RHSC,
315                                                          ARM_AM::no_shift),
316                                        MVT::i32);
317        return true;
318      }
319    }
320
321  // Otherwise this is R +/- [possibly shifted] R.
322  ARM_AM::AddrOpc AddSub = N.getOpcode() == ISD::ADD ? ARM_AM::add:ARM_AM::sub;
323  ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(N.getOperand(1));
324  unsigned ShAmt = 0;
325
326  Base   = N.getOperand(0);
327  Offset = N.getOperand(1);
328
329  if (ShOpcVal != ARM_AM::no_shift) {
330    // Check to see if the RHS of the shift is a constant, if not, we can't fold
331    // it.
332    if (ConstantSDNode *Sh =
333           dyn_cast<ConstantSDNode>(N.getOperand(1).getOperand(1))) {
334      ShAmt = Sh->getZExtValue();
335      Offset = N.getOperand(1).getOperand(0);
336    } else {
337      ShOpcVal = ARM_AM::no_shift;
338    }
339  }
340
341  // Try matching (R shl C) + (R).
342  if (N.getOpcode() == ISD::ADD && ShOpcVal == ARM_AM::no_shift) {
343    ShOpcVal = ARM_AM::getShiftOpcForNode(N.getOperand(0));
344    if (ShOpcVal != ARM_AM::no_shift) {
345      // Check to see if the RHS of the shift is a constant, if not, we can't
346      // fold it.
347      if (ConstantSDNode *Sh =
348          dyn_cast<ConstantSDNode>(N.getOperand(0).getOperand(1))) {
349        ShAmt = Sh->getZExtValue();
350        Offset = N.getOperand(0).getOperand(0);
351        Base = N.getOperand(1);
352      } else {
353        ShOpcVal = ARM_AM::no_shift;
354      }
355    }
356  }
357
358  Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, ShAmt, ShOpcVal),
359                                  MVT::i32);
360  return true;
361}
362
363bool ARMDAGToDAGISel::SelectAddrMode2Offset(SDNode *Op, SDValue N,
364                                            SDValue &Offset, SDValue &Opc) {
365  unsigned Opcode = Op->getOpcode();
366  ISD::MemIndexedMode AM = (Opcode == ISD::LOAD)
367    ? cast<LoadSDNode>(Op)->getAddressingMode()
368    : cast<StoreSDNode>(Op)->getAddressingMode();
369  ARM_AM::AddrOpc AddSub = (AM == ISD::PRE_INC || AM == ISD::POST_INC)
370    ? ARM_AM::add : ARM_AM::sub;
371  if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(N)) {
372    int Val = (int)C->getZExtValue();
373    if (Val >= 0 && Val < 0x1000) { // 12 bits.
374      Offset = CurDAG->getRegister(0, MVT::i32);
375      Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, Val,
376                                                        ARM_AM::no_shift),
377                                      MVT::i32);
378      return true;
379    }
380  }
381
382  Offset = N;
383  ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(N);
384  unsigned ShAmt = 0;
385  if (ShOpcVal != ARM_AM::no_shift) {
386    // Check to see if the RHS of the shift is a constant, if not, we can't fold
387    // it.
388    if (ConstantSDNode *Sh = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
389      ShAmt = Sh->getZExtValue();
390      Offset = N.getOperand(0);
391    } else {
392      ShOpcVal = ARM_AM::no_shift;
393    }
394  }
395
396  Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, ShAmt, ShOpcVal),
397                                  MVT::i32);
398  return true;
399}
400
401
402bool ARMDAGToDAGISel::SelectAddrMode3(SDNode *Op, SDValue N,
403                                      SDValue &Base, SDValue &Offset,
404                                      SDValue &Opc) {
405  if (N.getOpcode() == ISD::SUB) {
406    // X - C  is canonicalize to X + -C, no need to handle it here.
407    Base = N.getOperand(0);
408    Offset = N.getOperand(1);
409    Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(ARM_AM::sub, 0),MVT::i32);
410    return true;
411  }
412
413  if (N.getOpcode() != ISD::ADD) {
414    Base = N;
415    if (N.getOpcode() == ISD::FrameIndex) {
416      int FI = cast<FrameIndexSDNode>(N)->getIndex();
417      Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
418    }
419    Offset = CurDAG->getRegister(0, MVT::i32);
420    Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(ARM_AM::add, 0),MVT::i32);
421    return true;
422  }
423
424  // If the RHS is +/- imm8, fold into addr mode.
425  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
426    int RHSC = (int)RHS->getZExtValue();
427    if ((RHSC >= 0 && RHSC < 256) ||
428        (RHSC < 0 && RHSC > -256)) { // note -256 itself isn't allowed.
429      Base = N.getOperand(0);
430      if (Base.getOpcode() == ISD::FrameIndex) {
431        int FI = cast<FrameIndexSDNode>(Base)->getIndex();
432        Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
433      }
434      Offset = CurDAG->getRegister(0, MVT::i32);
435
436      ARM_AM::AddrOpc AddSub = ARM_AM::add;
437      if (RHSC < 0) {
438        AddSub = ARM_AM::sub;
439        RHSC = - RHSC;
440      }
441      Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(AddSub, RHSC),MVT::i32);
442      return true;
443    }
444  }
445
446  Base = N.getOperand(0);
447  Offset = N.getOperand(1);
448  Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(ARM_AM::add, 0), MVT::i32);
449  return true;
450}
451
452bool ARMDAGToDAGISel::SelectAddrMode3Offset(SDNode *Op, SDValue N,
453                                            SDValue &Offset, SDValue &Opc) {
454  unsigned Opcode = Op->getOpcode();
455  ISD::MemIndexedMode AM = (Opcode == ISD::LOAD)
456    ? cast<LoadSDNode>(Op)->getAddressingMode()
457    : cast<StoreSDNode>(Op)->getAddressingMode();
458  ARM_AM::AddrOpc AddSub = (AM == ISD::PRE_INC || AM == ISD::POST_INC)
459    ? ARM_AM::add : ARM_AM::sub;
460  if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(N)) {
461    int Val = (int)C->getZExtValue();
462    if (Val >= 0 && Val < 256) {
463      Offset = CurDAG->getRegister(0, MVT::i32);
464      Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(AddSub, Val), MVT::i32);
465      return true;
466    }
467  }
468
469  Offset = N;
470  Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(AddSub, 0), MVT::i32);
471  return true;
472}
473
474bool ARMDAGToDAGISel::SelectAddrMode4(SDNode *Op, SDValue N,
475                                      SDValue &Addr, SDValue &Mode) {
476  Addr = N;
477  Mode = CurDAG->getTargetConstant(ARM_AM::getAM4ModeImm(ARM_AM::ia), MVT::i32);
478  return true;
479}
480
481bool ARMDAGToDAGISel::SelectAddrMode5(SDNode *Op, SDValue N,
482                                      SDValue &Base, SDValue &Offset) {
483  if (N.getOpcode() != ISD::ADD) {
484    Base = N;
485    if (N.getOpcode() == ISD::FrameIndex) {
486      int FI = cast<FrameIndexSDNode>(N)->getIndex();
487      Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
488    } else if (N.getOpcode() == ARMISD::Wrapper &&
489               !(Subtarget->useMovt() &&
490                 N.getOperand(0).getOpcode() == ISD::TargetGlobalAddress)) {
491      Base = N.getOperand(0);
492    }
493    Offset = CurDAG->getTargetConstant(ARM_AM::getAM5Opc(ARM_AM::add, 0),
494                                       MVT::i32);
495    return true;
496  }
497
498  // If the RHS is +/- imm8, fold into addr mode.
499  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
500    int RHSC = (int)RHS->getZExtValue();
501    if ((RHSC & 3) == 0) {  // The constant is implicitly multiplied by 4.
502      RHSC >>= 2;
503      if ((RHSC >= 0 && RHSC < 256) ||
504          (RHSC < 0 && RHSC > -256)) { // note -256 itself isn't allowed.
505        Base = N.getOperand(0);
506        if (Base.getOpcode() == ISD::FrameIndex) {
507          int FI = cast<FrameIndexSDNode>(Base)->getIndex();
508          Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
509        }
510
511        ARM_AM::AddrOpc AddSub = ARM_AM::add;
512        if (RHSC < 0) {
513          AddSub = ARM_AM::sub;
514          RHSC = - RHSC;
515        }
516        Offset = CurDAG->getTargetConstant(ARM_AM::getAM5Opc(AddSub, RHSC),
517                                           MVT::i32);
518        return true;
519      }
520    }
521  }
522
523  Base = N;
524  Offset = CurDAG->getTargetConstant(ARM_AM::getAM5Opc(ARM_AM::add, 0),
525                                     MVT::i32);
526  return true;
527}
528
529bool ARMDAGToDAGISel::SelectAddrMode6(SDNode *Op, SDValue N,
530                                      SDValue &Addr, SDValue &Align) {
531  Addr = N;
532  // Default to no alignment.
533  Align = CurDAG->getTargetConstant(0, MVT::i32);
534  return true;
535}
536
537bool ARMDAGToDAGISel::SelectAddrModePC(SDNode *Op, SDValue N,
538                                       SDValue &Offset, SDValue &Label) {
539  if (N.getOpcode() == ARMISD::PIC_ADD && N.hasOneUse()) {
540    Offset = N.getOperand(0);
541    SDValue N1 = N.getOperand(1);
542    Label  = CurDAG->getTargetConstant(cast<ConstantSDNode>(N1)->getZExtValue(),
543                                       MVT::i32);
544    return true;
545  }
546  return false;
547}
548
549bool ARMDAGToDAGISel::SelectThumbAddrModeRR(SDNode *Op, SDValue N,
550                                            SDValue &Base, SDValue &Offset){
551  // FIXME dl should come from the parent load or store, not the address
552  if (N.getOpcode() != ISD::ADD) {
553    ConstantSDNode *NC = dyn_cast<ConstantSDNode>(N);
554    if (!NC || !NC->isNullValue())
555      return false;
556
557    Base = Offset = N;
558    return true;
559  }
560
561  Base = N.getOperand(0);
562  Offset = N.getOperand(1);
563  return true;
564}
565
566bool
567ARMDAGToDAGISel::SelectThumbAddrModeRI5(SDNode *Op, SDValue N,
568                                        unsigned Scale, SDValue &Base,
569                                        SDValue &OffImm, SDValue &Offset) {
570  if (Scale == 4) {
571    SDValue TmpBase, TmpOffImm;
572    if (SelectThumbAddrModeSP(Op, N, TmpBase, TmpOffImm))
573      return false;  // We want to select tLDRspi / tSTRspi instead.
574    if (N.getOpcode() == ARMISD::Wrapper &&
575        N.getOperand(0).getOpcode() == ISD::TargetConstantPool)
576      return false;  // We want to select tLDRpci instead.
577  }
578
579  if (N.getOpcode() != ISD::ADD) {
580    if (N.getOpcode() == ARMISD::Wrapper &&
581        !(Subtarget->useMovt() &&
582          N.getOperand(0).getOpcode() == ISD::TargetGlobalAddress)) {
583      Base = N.getOperand(0);
584    } else
585      Base = N;
586
587    Offset = CurDAG->getRegister(0, MVT::i32);
588    OffImm = CurDAG->getTargetConstant(0, MVT::i32);
589    return true;
590  }
591
592  // Thumb does not have [sp, r] address mode.
593  RegisterSDNode *LHSR = dyn_cast<RegisterSDNode>(N.getOperand(0));
594  RegisterSDNode *RHSR = dyn_cast<RegisterSDNode>(N.getOperand(1));
595  if ((LHSR && LHSR->getReg() == ARM::SP) ||
596      (RHSR && RHSR->getReg() == ARM::SP)) {
597    Base = N;
598    Offset = CurDAG->getRegister(0, MVT::i32);
599    OffImm = CurDAG->getTargetConstant(0, MVT::i32);
600    return true;
601  }
602
603  // If the RHS is + imm5 * scale, fold into addr mode.
604  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
605    int RHSC = (int)RHS->getZExtValue();
606    if ((RHSC & (Scale-1)) == 0) {  // The constant is implicitly multiplied.
607      RHSC /= Scale;
608      if (RHSC >= 0 && RHSC < 32) {
609        Base = N.getOperand(0);
610        Offset = CurDAG->getRegister(0, MVT::i32);
611        OffImm = CurDAG->getTargetConstant(RHSC, MVT::i32);
612        return true;
613      }
614    }
615  }
616
617  Base = N.getOperand(0);
618  Offset = N.getOperand(1);
619  OffImm = CurDAG->getTargetConstant(0, MVT::i32);
620  return true;
621}
622
623bool ARMDAGToDAGISel::SelectThumbAddrModeS1(SDNode *Op, SDValue N,
624                                            SDValue &Base, SDValue &OffImm,
625                                            SDValue &Offset) {
626  return SelectThumbAddrModeRI5(Op, N, 1, Base, OffImm, Offset);
627}
628
629bool ARMDAGToDAGISel::SelectThumbAddrModeS2(SDNode *Op, SDValue N,
630                                            SDValue &Base, SDValue &OffImm,
631                                            SDValue &Offset) {
632  return SelectThumbAddrModeRI5(Op, N, 2, Base, OffImm, Offset);
633}
634
635bool ARMDAGToDAGISel::SelectThumbAddrModeS4(SDNode *Op, SDValue N,
636                                            SDValue &Base, SDValue &OffImm,
637                                            SDValue &Offset) {
638  return SelectThumbAddrModeRI5(Op, N, 4, Base, OffImm, Offset);
639}
640
641bool ARMDAGToDAGISel::SelectThumbAddrModeSP(SDNode *Op, SDValue N,
642                                           SDValue &Base, SDValue &OffImm) {
643  if (N.getOpcode() == ISD::FrameIndex) {
644    int FI = cast<FrameIndexSDNode>(N)->getIndex();
645    Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
646    OffImm = CurDAG->getTargetConstant(0, MVT::i32);
647    return true;
648  }
649
650  if (N.getOpcode() != ISD::ADD)
651    return false;
652
653  RegisterSDNode *LHSR = dyn_cast<RegisterSDNode>(N.getOperand(0));
654  if (N.getOperand(0).getOpcode() == ISD::FrameIndex ||
655      (LHSR && LHSR->getReg() == ARM::SP)) {
656    // If the RHS is + imm8 * scale, fold into addr mode.
657    if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
658      int RHSC = (int)RHS->getZExtValue();
659      if ((RHSC & 3) == 0) {  // The constant is implicitly multiplied.
660        RHSC >>= 2;
661        if (RHSC >= 0 && RHSC < 256) {
662          Base = N.getOperand(0);
663          if (Base.getOpcode() == ISD::FrameIndex) {
664            int FI = cast<FrameIndexSDNode>(Base)->getIndex();
665            Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
666          }
667          OffImm = CurDAG->getTargetConstant(RHSC, MVT::i32);
668          return true;
669        }
670      }
671    }
672  }
673
674  return false;
675}
676
677bool ARMDAGToDAGISel::SelectT2ShifterOperandReg(SDNode *Op, SDValue N,
678                                                SDValue &BaseReg,
679                                                SDValue &Opc) {
680  if (DisableShifterOp)
681    return false;
682
683  ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(N);
684
685  // Don't match base register only case. That is matched to a separate
686  // lower complexity pattern with explicit register operand.
687  if (ShOpcVal == ARM_AM::no_shift) return false;
688
689  BaseReg = N.getOperand(0);
690  unsigned ShImmVal = 0;
691  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
692    ShImmVal = RHS->getZExtValue() & 31;
693    Opc = getI32Imm(ARM_AM::getSORegOpc(ShOpcVal, ShImmVal));
694    return true;
695  }
696
697  return false;
698}
699
700bool ARMDAGToDAGISel::SelectT2AddrModeImm12(SDNode *Op, SDValue N,
701                                            SDValue &Base, SDValue &OffImm) {
702  // Match simple R + imm12 operands.
703
704  // Base only.
705  if (N.getOpcode() != ISD::ADD && N.getOpcode() != ISD::SUB) {
706    if (N.getOpcode() == ISD::FrameIndex) {
707      // Match frame index...
708      int FI = cast<FrameIndexSDNode>(N)->getIndex();
709      Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
710      OffImm  = CurDAG->getTargetConstant(0, MVT::i32);
711      return true;
712    } else if (N.getOpcode() == ARMISD::Wrapper &&
713               !(Subtarget->useMovt() &&
714                 N.getOperand(0).getOpcode() == ISD::TargetGlobalAddress)) {
715      Base = N.getOperand(0);
716      if (Base.getOpcode() == ISD::TargetConstantPool)
717        return false;  // We want to select t2LDRpci instead.
718    } else
719      Base = N;
720    OffImm  = CurDAG->getTargetConstant(0, MVT::i32);
721    return true;
722  }
723
724  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
725    if (SelectT2AddrModeImm8(Op, N, Base, OffImm))
726      // Let t2LDRi8 handle (R - imm8).
727      return false;
728
729    int RHSC = (int)RHS->getZExtValue();
730    if (N.getOpcode() == ISD::SUB)
731      RHSC = -RHSC;
732
733    if (RHSC >= 0 && RHSC < 0x1000) { // 12 bits (unsigned)
734      Base   = N.getOperand(0);
735      if (Base.getOpcode() == ISD::FrameIndex) {
736        int FI = cast<FrameIndexSDNode>(Base)->getIndex();
737        Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
738      }
739      OffImm = CurDAG->getTargetConstant(RHSC, MVT::i32);
740      return true;
741    }
742  }
743
744  // Base only.
745  Base = N;
746  OffImm  = CurDAG->getTargetConstant(0, MVT::i32);
747  return true;
748}
749
750bool ARMDAGToDAGISel::SelectT2AddrModeImm8(SDNode *Op, SDValue N,
751                                           SDValue &Base, SDValue &OffImm) {
752  // Match simple R - imm8 operands.
753  if (N.getOpcode() == ISD::ADD || N.getOpcode() == ISD::SUB) {
754    if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
755      int RHSC = (int)RHS->getSExtValue();
756      if (N.getOpcode() == ISD::SUB)
757        RHSC = -RHSC;
758
759      if ((RHSC >= -255) && (RHSC < 0)) { // 8 bits (always negative)
760        Base = N.getOperand(0);
761        if (Base.getOpcode() == ISD::FrameIndex) {
762          int FI = cast<FrameIndexSDNode>(Base)->getIndex();
763          Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
764        }
765        OffImm = CurDAG->getTargetConstant(RHSC, MVT::i32);
766        return true;
767      }
768    }
769  }
770
771  return false;
772}
773
774bool ARMDAGToDAGISel::SelectT2AddrModeImm8Offset(SDNode *Op, SDValue N,
775                                                 SDValue &OffImm){
776  unsigned Opcode = Op->getOpcode();
777  ISD::MemIndexedMode AM = (Opcode == ISD::LOAD)
778    ? cast<LoadSDNode>(Op)->getAddressingMode()
779    : cast<StoreSDNode>(Op)->getAddressingMode();
780  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N)) {
781    int RHSC = (int)RHS->getZExtValue();
782    if (RHSC >= 0 && RHSC < 0x100) { // 8 bits.
783      OffImm = ((AM == ISD::PRE_INC) || (AM == ISD::POST_INC))
784        ? CurDAG->getTargetConstant(RHSC, MVT::i32)
785        : CurDAG->getTargetConstant(-RHSC, MVT::i32);
786      return true;
787    }
788  }
789
790  return false;
791}
792
793bool ARMDAGToDAGISel::SelectT2AddrModeSoReg(SDNode *Op, SDValue N,
794                                            SDValue &Base,
795                                            SDValue &OffReg, SDValue &ShImm) {
796  // (R - imm8) should be handled by t2LDRi8. The rest are handled by t2LDRi12.
797  if (N.getOpcode() != ISD::ADD)
798    return false;
799
800  // Leave (R + imm12) for t2LDRi12, (R - imm8) for t2LDRi8.
801  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
802    int RHSC = (int)RHS->getZExtValue();
803    if (RHSC >= 0 && RHSC < 0x1000) // 12 bits (unsigned)
804      return false;
805    else if (RHSC < 0 && RHSC >= -255) // 8 bits
806      return false;
807  }
808
809  // Look for (R + R) or (R + (R << [1,2,3])).
810  unsigned ShAmt = 0;
811  Base   = N.getOperand(0);
812  OffReg = N.getOperand(1);
813
814  // Swap if it is ((R << c) + R).
815  ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(OffReg);
816  if (ShOpcVal != ARM_AM::lsl) {
817    ShOpcVal = ARM_AM::getShiftOpcForNode(Base);
818    if (ShOpcVal == ARM_AM::lsl)
819      std::swap(Base, OffReg);
820  }
821
822  if (ShOpcVal == ARM_AM::lsl) {
823    // Check to see if the RHS of the shift is a constant, if not, we can't fold
824    // it.
825    if (ConstantSDNode *Sh = dyn_cast<ConstantSDNode>(OffReg.getOperand(1))) {
826      ShAmt = Sh->getZExtValue();
827      if (ShAmt >= 4) {
828        ShAmt = 0;
829        ShOpcVal = ARM_AM::no_shift;
830      } else
831        OffReg = OffReg.getOperand(0);
832    } else {
833      ShOpcVal = ARM_AM::no_shift;
834    }
835  }
836
837  ShImm = CurDAG->getTargetConstant(ShAmt, MVT::i32);
838
839  return true;
840}
841
842//===--------------------------------------------------------------------===//
843
844/// getAL - Returns a ARMCC::AL immediate node.
845static inline SDValue getAL(SelectionDAG *CurDAG) {
846  return CurDAG->getTargetConstant((uint64_t)ARMCC::AL, MVT::i32);
847}
848
849SDNode *ARMDAGToDAGISel::SelectARMIndexedLoad(SDNode *N) {
850  LoadSDNode *LD = cast<LoadSDNode>(N);
851  ISD::MemIndexedMode AM = LD->getAddressingMode();
852  if (AM == ISD::UNINDEXED)
853    return NULL;
854
855  EVT LoadedVT = LD->getMemoryVT();
856  SDValue Offset, AMOpc;
857  bool isPre = (AM == ISD::PRE_INC) || (AM == ISD::PRE_DEC);
858  unsigned Opcode = 0;
859  bool Match = false;
860  if (LoadedVT == MVT::i32 &&
861      SelectAddrMode2Offset(N, LD->getOffset(), Offset, AMOpc)) {
862    Opcode = isPre ? ARM::LDR_PRE : ARM::LDR_POST;
863    Match = true;
864  } else if (LoadedVT == MVT::i16 &&
865             SelectAddrMode3Offset(N, LD->getOffset(), Offset, AMOpc)) {
866    Match = true;
867    Opcode = (LD->getExtensionType() == ISD::SEXTLOAD)
868      ? (isPre ? ARM::LDRSH_PRE : ARM::LDRSH_POST)
869      : (isPre ? ARM::LDRH_PRE : ARM::LDRH_POST);
870  } else if (LoadedVT == MVT::i8 || LoadedVT == MVT::i1) {
871    if (LD->getExtensionType() == ISD::SEXTLOAD) {
872      if (SelectAddrMode3Offset(N, LD->getOffset(), Offset, AMOpc)) {
873        Match = true;
874        Opcode = isPre ? ARM::LDRSB_PRE : ARM::LDRSB_POST;
875      }
876    } else {
877      if (SelectAddrMode2Offset(N, LD->getOffset(), Offset, AMOpc)) {
878        Match = true;
879        Opcode = isPre ? ARM::LDRB_PRE : ARM::LDRB_POST;
880      }
881    }
882  }
883
884  if (Match) {
885    SDValue Chain = LD->getChain();
886    SDValue Base = LD->getBasePtr();
887    SDValue Ops[]= { Base, Offset, AMOpc, getAL(CurDAG),
888                     CurDAG->getRegister(0, MVT::i32), Chain };
889    return CurDAG->getMachineNode(Opcode, N->getDebugLoc(), MVT::i32, MVT::i32,
890                                  MVT::Other, Ops, 6);
891  }
892
893  return NULL;
894}
895
896SDNode *ARMDAGToDAGISel::SelectT2IndexedLoad(SDNode *N) {
897  LoadSDNode *LD = cast<LoadSDNode>(N);
898  ISD::MemIndexedMode AM = LD->getAddressingMode();
899  if (AM == ISD::UNINDEXED)
900    return NULL;
901
902  EVT LoadedVT = LD->getMemoryVT();
903  bool isSExtLd = LD->getExtensionType() == ISD::SEXTLOAD;
904  SDValue Offset;
905  bool isPre = (AM == ISD::PRE_INC) || (AM == ISD::PRE_DEC);
906  unsigned Opcode = 0;
907  bool Match = false;
908  if (SelectT2AddrModeImm8Offset(N, LD->getOffset(), Offset)) {
909    switch (LoadedVT.getSimpleVT().SimpleTy) {
910    case MVT::i32:
911      Opcode = isPre ? ARM::t2LDR_PRE : ARM::t2LDR_POST;
912      break;
913    case MVT::i16:
914      if (isSExtLd)
915        Opcode = isPre ? ARM::t2LDRSH_PRE : ARM::t2LDRSH_POST;
916      else
917        Opcode = isPre ? ARM::t2LDRH_PRE : ARM::t2LDRH_POST;
918      break;
919    case MVT::i8:
920    case MVT::i1:
921      if (isSExtLd)
922        Opcode = isPre ? ARM::t2LDRSB_PRE : ARM::t2LDRSB_POST;
923      else
924        Opcode = isPre ? ARM::t2LDRB_PRE : ARM::t2LDRB_POST;
925      break;
926    default:
927      return NULL;
928    }
929    Match = true;
930  }
931
932  if (Match) {
933    SDValue Chain = LD->getChain();
934    SDValue Base = LD->getBasePtr();
935    SDValue Ops[]= { Base, Offset, getAL(CurDAG),
936                     CurDAG->getRegister(0, MVT::i32), Chain };
937    return CurDAG->getMachineNode(Opcode, N->getDebugLoc(), MVT::i32, MVT::i32,
938                                  MVT::Other, Ops, 5);
939  }
940
941  return NULL;
942}
943
944/// PairSRegs - Form a D register from a pair of S registers.
945///
946SDNode *ARMDAGToDAGISel::PairSRegs(EVT VT, SDValue V0, SDValue V1) {
947  DebugLoc dl = V0.getNode()->getDebugLoc();
948  SDValue SubReg0 = CurDAG->getTargetConstant(ARM::ssub_0, MVT::i32);
949  SDValue SubReg1 = CurDAG->getTargetConstant(ARM::ssub_1, MVT::i32);
950  const SDValue Ops[] = { V0, SubReg0, V1, SubReg1 };
951  return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops, 4);
952}
953
954/// PairDRegs - Form a quad register from a pair of D registers.
955///
956SDNode *ARMDAGToDAGISel::PairDRegs(EVT VT, SDValue V0, SDValue V1) {
957  DebugLoc dl = V0.getNode()->getDebugLoc();
958  SDValue SubReg0 = CurDAG->getTargetConstant(ARM::dsub_0, MVT::i32);
959  SDValue SubReg1 = CurDAG->getTargetConstant(ARM::dsub_1, MVT::i32);
960  const SDValue Ops[] = { V0, SubReg0, V1, SubReg1 };
961  return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops, 4);
962}
963
964/// PairQRegs - Form 4 consecutive D registers from a pair of Q registers.
965///
966SDNode *ARMDAGToDAGISel::PairQRegs(EVT VT, SDValue V0, SDValue V1) {
967  DebugLoc dl = V0.getNode()->getDebugLoc();
968  SDValue SubReg0 = CurDAG->getTargetConstant(ARM::qsub_0, MVT::i32);
969  SDValue SubReg1 = CurDAG->getTargetConstant(ARM::qsub_1, MVT::i32);
970  const SDValue Ops[] = { V0, SubReg0, V1, SubReg1 };
971  return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops, 4);
972}
973
974/// QuadSRegs - Form 4 consecutive S registers.
975///
976SDNode *ARMDAGToDAGISel::QuadSRegs(EVT VT, SDValue V0, SDValue V1,
977                                   SDValue V2, SDValue V3) {
978  DebugLoc dl = V0.getNode()->getDebugLoc();
979  SDValue SubReg0 = CurDAG->getTargetConstant(ARM::ssub_0, MVT::i32);
980  SDValue SubReg1 = CurDAG->getTargetConstant(ARM::ssub_1, MVT::i32);
981  SDValue SubReg2 = CurDAG->getTargetConstant(ARM::ssub_2, MVT::i32);
982  SDValue SubReg3 = CurDAG->getTargetConstant(ARM::ssub_3, MVT::i32);
983  const SDValue Ops[] = { V0, SubReg0, V1, SubReg1, V2, SubReg2, V3, SubReg3 };
984  return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops, 8);
985}
986
987/// QuadDRegs - Form 4 consecutive D registers.
988///
989SDNode *ARMDAGToDAGISel::QuadDRegs(EVT VT, SDValue V0, SDValue V1,
990                                   SDValue V2, SDValue V3) {
991  DebugLoc dl = V0.getNode()->getDebugLoc();
992  SDValue SubReg0 = CurDAG->getTargetConstant(ARM::dsub_0, MVT::i32);
993  SDValue SubReg1 = CurDAG->getTargetConstant(ARM::dsub_1, MVT::i32);
994  SDValue SubReg2 = CurDAG->getTargetConstant(ARM::dsub_2, MVT::i32);
995  SDValue SubReg3 = CurDAG->getTargetConstant(ARM::dsub_3, MVT::i32);
996  const SDValue Ops[] = { V0, SubReg0, V1, SubReg1, V2, SubReg2, V3, SubReg3 };
997  return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops, 8);
998}
999
1000/// QuadQRegs - Form 4 consecutive Q registers.
1001///
1002SDNode *ARMDAGToDAGISel::QuadQRegs(EVT VT, SDValue V0, SDValue V1,
1003                                   SDValue V2, SDValue V3) {
1004  DebugLoc dl = V0.getNode()->getDebugLoc();
1005  SDValue SubReg0 = CurDAG->getTargetConstant(ARM::qsub_0, MVT::i32);
1006  SDValue SubReg1 = CurDAG->getTargetConstant(ARM::qsub_1, MVT::i32);
1007  SDValue SubReg2 = CurDAG->getTargetConstant(ARM::qsub_2, MVT::i32);
1008  SDValue SubReg3 = CurDAG->getTargetConstant(ARM::qsub_3, MVT::i32);
1009  const SDValue Ops[] = { V0, SubReg0, V1, SubReg1, V2, SubReg2, V3, SubReg3 };
1010  return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops, 8);
1011}
1012
1013SDNode *ARMDAGToDAGISel::SelectVLD(SDNode *N, unsigned NumVecs,
1014                                   unsigned *DOpcodes, unsigned *QOpcodes0,
1015                                   unsigned *QOpcodes1) {
1016  assert(NumVecs >= 1 && NumVecs <= 4 && "VLD NumVecs out-of-range");
1017  DebugLoc dl = N->getDebugLoc();
1018
1019  SDValue MemAddr, Align;
1020  if (!SelectAddrMode6(N, N->getOperand(2), MemAddr, Align))
1021    return NULL;
1022
1023  SDValue Chain = N->getOperand(0);
1024  EVT VT = N->getValueType(0);
1025  bool is64BitVector = VT.is64BitVector();
1026
1027  unsigned OpcodeIndex;
1028  switch (VT.getSimpleVT().SimpleTy) {
1029  default: llvm_unreachable("unhandled vld type");
1030    // Double-register operations:
1031  case MVT::v8i8:  OpcodeIndex = 0; break;
1032  case MVT::v4i16: OpcodeIndex = 1; break;
1033  case MVT::v2f32:
1034  case MVT::v2i32: OpcodeIndex = 2; break;
1035  case MVT::v1i64: OpcodeIndex = 3; break;
1036    // Quad-register operations:
1037  case MVT::v16i8: OpcodeIndex = 0; break;
1038  case MVT::v8i16: OpcodeIndex = 1; break;
1039  case MVT::v4f32:
1040  case MVT::v4i32: OpcodeIndex = 2; break;
1041  case MVT::v2i64: OpcodeIndex = 3;
1042    assert(NumVecs == 1 && "v2i64 type only supported for VLD1");
1043    break;
1044  }
1045
1046  EVT ResTy;
1047  if (NumVecs == 1)
1048    ResTy = VT;
1049  else {
1050    unsigned ResTyElts = (NumVecs == 3) ? 4 : NumVecs;
1051    if (!is64BitVector)
1052      ResTyElts *= 2;
1053    ResTy = EVT::getVectorVT(*CurDAG->getContext(), MVT::i64, ResTyElts);
1054  }
1055
1056  SDValue Pred = getAL(CurDAG);
1057  SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
1058  SDValue SuperReg;
1059  if (is64BitVector) {
1060    unsigned Opc = DOpcodes[OpcodeIndex];
1061    const SDValue Ops[] = { MemAddr, Align, Pred, Reg0, Chain };
1062    SDNode *VLd = CurDAG->getMachineNode(Opc, dl, ResTy, MVT::Other, Ops, 5);
1063    if (NumVecs == 1)
1064      return VLd;
1065
1066    SuperReg = SDValue(VLd, 0);
1067    assert(ARM::dsub_7 == ARM::dsub_0+7 && "Unexpected subreg numbering");
1068    for (unsigned Vec = 0; Vec < NumVecs; ++Vec) {
1069      SDValue D = CurDAG->getTargetExtractSubreg(ARM::dsub_0+Vec,
1070                                                 dl, VT, SuperReg);
1071      ReplaceUses(SDValue(N, Vec), D);
1072    }
1073    ReplaceUses(SDValue(N, NumVecs), SDValue(VLd, 1));
1074    return NULL;
1075  }
1076
1077  if (NumVecs <= 2) {
1078    // Quad registers are directly supported for VLD1 and VLD2,
1079    // loading pairs of D regs.
1080    unsigned Opc = QOpcodes0[OpcodeIndex];
1081    const SDValue Ops[] = { MemAddr, Align, Pred, Reg0, Chain };
1082    SDNode *VLd = CurDAG->getMachineNode(Opc, dl, ResTy, MVT::Other, Ops, 5);
1083    if (NumVecs == 1)
1084      return VLd;
1085
1086    SuperReg = SDValue(VLd, 0);
1087    Chain = SDValue(VLd, 1);
1088
1089  } else {
1090    // Otherwise, quad registers are loaded with two separate instructions,
1091    // where one loads the even registers and the other loads the odd registers.
1092    EVT AddrTy = MemAddr.getValueType();
1093
1094    // Load the even subregs.
1095    unsigned Opc = QOpcodes0[OpcodeIndex];
1096    SDValue ImplDef =
1097      SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, dl, ResTy), 0);
1098    const SDValue OpsA[] = { MemAddr, Align, Reg0, ImplDef, Pred, Reg0, Chain };
1099    SDNode *VLdA =
1100      CurDAG->getMachineNode(Opc, dl, ResTy, AddrTy, MVT::Other, OpsA, 7);
1101    Chain = SDValue(VLdA, 2);
1102
1103    // Load the odd subregs.
1104    Opc = QOpcodes1[OpcodeIndex];
1105    const SDValue OpsB[] = { SDValue(VLdA, 1), Align, Reg0, SDValue(VLdA, 0),
1106                             Pred, Reg0, Chain };
1107    SDNode *VLdB =
1108      CurDAG->getMachineNode(Opc, dl, ResTy, AddrTy, MVT::Other, OpsB, 7);
1109    SuperReg = SDValue(VLdB, 0);
1110    Chain = SDValue(VLdB, 2);
1111  }
1112
1113  // Extract out the Q registers.
1114  assert(ARM::qsub_3 == ARM::qsub_0+3 && "Unexpected subreg numbering");
1115  for (unsigned Vec = 0; Vec < NumVecs; ++Vec) {
1116    SDValue Q = CurDAG->getTargetExtractSubreg(ARM::qsub_0+Vec,
1117                                               dl, VT, SuperReg);
1118    ReplaceUses(SDValue(N, Vec), Q);
1119  }
1120  ReplaceUses(SDValue(N, NumVecs), Chain);
1121  return NULL;
1122}
1123
1124SDNode *ARMDAGToDAGISel::SelectVST(SDNode *N, unsigned NumVecs,
1125                                   unsigned *DOpcodes, unsigned *QOpcodes0,
1126                                   unsigned *QOpcodes1) {
1127  assert(NumVecs >= 1 && NumVecs <= 4 && "VST NumVecs out-of-range");
1128  DebugLoc dl = N->getDebugLoc();
1129
1130  SDValue MemAddr, Align;
1131  if (!SelectAddrMode6(N, N->getOperand(2), MemAddr, Align))
1132    return NULL;
1133
1134  SDValue Chain = N->getOperand(0);
1135  EVT VT = N->getOperand(3).getValueType();
1136  bool is64BitVector = VT.is64BitVector();
1137
1138  unsigned OpcodeIndex;
1139  switch (VT.getSimpleVT().SimpleTy) {
1140  default: llvm_unreachable("unhandled vst type");
1141    // Double-register operations:
1142  case MVT::v8i8:  OpcodeIndex = 0; break;
1143  case MVT::v4i16: OpcodeIndex = 1; break;
1144  case MVT::v2f32:
1145  case MVT::v2i32: OpcodeIndex = 2; break;
1146  case MVT::v1i64: OpcodeIndex = 3; break;
1147    // Quad-register operations:
1148  case MVT::v16i8: OpcodeIndex = 0; break;
1149  case MVT::v8i16: OpcodeIndex = 1; break;
1150  case MVT::v4f32:
1151  case MVT::v4i32: OpcodeIndex = 2; break;
1152  case MVT::v2i64: OpcodeIndex = 3;
1153    assert(NumVecs == 1 && "v2i64 type only supported for VST1");
1154    break;
1155  }
1156
1157  SDValue Pred = getAL(CurDAG);
1158  SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
1159
1160  SmallVector<SDValue, 7> Ops;
1161  Ops.push_back(MemAddr);
1162  Ops.push_back(Align);
1163
1164  if (is64BitVector) {
1165    if (NumVecs == 1) {
1166      Ops.push_back(N->getOperand(3));
1167    } else {
1168      SDValue RegSeq;
1169      SDValue V0 = N->getOperand(0+3);
1170      SDValue V1 = N->getOperand(1+3);
1171
1172      // Form a REG_SEQUENCE to force register allocation.
1173      if (NumVecs == 2)
1174        RegSeq = SDValue(PairDRegs(MVT::v2i64, V0, V1), 0);
1175      else {
1176        SDValue V2 = N->getOperand(2+3);
1177        // If it's a vld3, form a quad D-register and leave the last part as
1178        // an undef.
1179        SDValue V3 = (NumVecs == 3)
1180          ? SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF,dl,VT), 0)
1181          : N->getOperand(3+3);
1182        RegSeq = SDValue(QuadDRegs(MVT::v4i64, V0, V1, V2, V3), 0);
1183      }
1184      Ops.push_back(RegSeq);
1185    }
1186    Ops.push_back(Pred);
1187    Ops.push_back(Reg0); // predicate register
1188    Ops.push_back(Chain);
1189    unsigned Opc = DOpcodes[OpcodeIndex];
1190    return CurDAG->getMachineNode(Opc, dl, MVT::Other, Ops.data(), 6);
1191  }
1192
1193  if (NumVecs <= 2) {
1194    // Quad registers are directly supported for VST1 and VST2.
1195    unsigned Opc = QOpcodes0[OpcodeIndex];
1196    if (NumVecs == 1) {
1197      Ops.push_back(N->getOperand(3));
1198    } else {
1199      // Form a QQ register.
1200      SDValue Q0 = N->getOperand(3);
1201      SDValue Q1 = N->getOperand(4);
1202      Ops.push_back(SDValue(PairQRegs(MVT::v4i64, Q0, Q1), 0));
1203    }
1204    Ops.push_back(Pred);
1205    Ops.push_back(Reg0); // predicate register
1206    Ops.push_back(Chain);
1207    return CurDAG->getMachineNode(Opc, dl, MVT::Other, Ops.data(), 6);
1208  }
1209
1210  // Otherwise, quad registers are stored with two separate instructions,
1211  // where one stores the even registers and the other stores the odd registers.
1212
1213  // Form the QQQQ REG_SEQUENCE.
1214  SDValue V0 = N->getOperand(0+3);
1215  SDValue V1 = N->getOperand(1+3);
1216  SDValue V2 = N->getOperand(2+3);
1217  SDValue V3 = (NumVecs == 3)
1218    ? SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, dl, VT), 0)
1219    : N->getOperand(3+3);
1220  SDValue RegSeq = SDValue(QuadQRegs(MVT::v8i64, V0, V1, V2, V3), 0);
1221
1222  // Store the even D registers.
1223  Ops.push_back(Reg0); // post-access address offset
1224  Ops.push_back(RegSeq);
1225  Ops.push_back(Pred);
1226  Ops.push_back(Reg0); // predicate register
1227  Ops.push_back(Chain);
1228  unsigned Opc = QOpcodes0[OpcodeIndex];
1229  SDNode *VStA = CurDAG->getMachineNode(Opc, dl, MemAddr.getValueType(),
1230                                        MVT::Other, Ops.data(), 7);
1231  Chain = SDValue(VStA, 1);
1232
1233  // Store the odd D registers.
1234  Ops[0] = SDValue(VStA, 0); // MemAddr
1235  Ops[6] = Chain;
1236  Opc = QOpcodes1[OpcodeIndex];
1237  SDNode *VStB = CurDAG->getMachineNode(Opc, dl, MemAddr.getValueType(),
1238                                        MVT::Other, Ops.data(), 7);
1239  Chain = SDValue(VStB, 1);
1240  ReplaceUses(SDValue(N, 0), Chain);
1241  return NULL;
1242}
1243
1244SDNode *ARMDAGToDAGISel::SelectVLDSTLane(SDNode *N, bool IsLoad,
1245                                         unsigned NumVecs, unsigned *DOpcodes,
1246                                         unsigned *QOpcodes) {
1247  assert(NumVecs >=2 && NumVecs <= 4 && "VLDSTLane NumVecs out-of-range");
1248  DebugLoc dl = N->getDebugLoc();
1249
1250  SDValue MemAddr, Align;
1251  if (!SelectAddrMode6(N, N->getOperand(2), MemAddr, Align))
1252    return NULL;
1253
1254  SDValue Chain = N->getOperand(0);
1255  unsigned Lane =
1256    cast<ConstantSDNode>(N->getOperand(NumVecs+3))->getZExtValue();
1257  EVT VT = IsLoad ? N->getValueType(0) : N->getOperand(3).getValueType();
1258  bool is64BitVector = VT.is64BitVector();
1259
1260  unsigned OpcodeIndex;
1261  switch (VT.getSimpleVT().SimpleTy) {
1262  default: llvm_unreachable("unhandled vld/vst lane type");
1263    // Double-register operations:
1264  case MVT::v8i8:  OpcodeIndex = 0; break;
1265  case MVT::v4i16: OpcodeIndex = 1; break;
1266  case MVT::v2f32:
1267  case MVT::v2i32: OpcodeIndex = 2; break;
1268    // Quad-register operations:
1269  case MVT::v8i16: OpcodeIndex = 0; break;
1270  case MVT::v4f32:
1271  case MVT::v4i32: OpcodeIndex = 1; break;
1272  }
1273
1274  SDValue Pred = getAL(CurDAG);
1275  SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
1276
1277  SmallVector<SDValue, 7> Ops;
1278  Ops.push_back(MemAddr);
1279  Ops.push_back(Align);
1280
1281  unsigned Opc = (is64BitVector ? DOpcodes[OpcodeIndex] :
1282                  Opc = QOpcodes[OpcodeIndex]);
1283
1284  SDValue SuperReg;
1285  SDValue V0 = N->getOperand(0+3);
1286  SDValue V1 = N->getOperand(1+3);
1287  if (NumVecs == 2) {
1288    if (is64BitVector)
1289      SuperReg = SDValue(PairDRegs(MVT::v2i64, V0, V1), 0);
1290    else
1291      SuperReg = SDValue(PairQRegs(MVT::v4i64, V0, V1), 0);
1292  } else {
1293    SDValue V2 = N->getOperand(2+3);
1294    SDValue V3 = (NumVecs == 3)
1295      ? SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF,dl,VT), 0)
1296      : N->getOperand(3+3);
1297    if (is64BitVector)
1298      SuperReg = SDValue(QuadDRegs(MVT::v4i64, V0, V1, V2, V3), 0);
1299    else
1300      SuperReg = SDValue(QuadQRegs(MVT::v8i64, V0, V1, V2, V3), 0);
1301  }
1302  Ops.push_back(SuperReg);
1303  Ops.push_back(getI32Imm(Lane));
1304  Ops.push_back(Pred);
1305  Ops.push_back(Reg0);
1306  Ops.push_back(Chain);
1307
1308  if (!IsLoad)
1309    return CurDAG->getMachineNode(Opc, dl, MVT::Other, Ops.data(), 7);
1310
1311  EVT ResTy;
1312  unsigned ResTyElts = (NumVecs == 3) ? 4 : NumVecs;
1313  if (!is64BitVector)
1314    ResTyElts *= 2;
1315  ResTy = EVT::getVectorVT(*CurDAG->getContext(), MVT::i64, ResTyElts);
1316
1317  SDNode *VLdLn = CurDAG->getMachineNode(Opc, dl, ResTy, MVT::Other,
1318                                         Ops.data(), 7);
1319  SuperReg = SDValue(VLdLn, 0);
1320  Chain = SDValue(VLdLn, 1);
1321
1322  // Extract the subregisters.
1323  assert(ARM::dsub_7 == ARM::dsub_0+7 && "Unexpected subreg numbering");
1324  assert(ARM::qsub_3 == ARM::qsub_0+3 && "Unexpected subreg numbering");
1325  unsigned SubIdx = is64BitVector ? ARM::dsub_0 : ARM::qsub_0;
1326  for (unsigned Vec = 0; Vec < NumVecs; ++Vec)
1327    ReplaceUses(SDValue(N, Vec),
1328                CurDAG->getTargetExtractSubreg(SubIdx+Vec, dl, VT, SuperReg));
1329  ReplaceUses(SDValue(N, NumVecs), Chain);
1330  return NULL;
1331}
1332
1333SDNode *ARMDAGToDAGISel::SelectVTBL(SDNode *N, bool IsExt, unsigned NumVecs,
1334                                    unsigned Opc) {
1335  assert(NumVecs >= 2 && NumVecs <= 4 && "VTBL NumVecs out-of-range");
1336  DebugLoc dl = N->getDebugLoc();
1337  EVT VT = N->getValueType(0);
1338  unsigned FirstTblReg = IsExt ? 2 : 1;
1339
1340  // Form a REG_SEQUENCE to force register allocation.
1341  SDValue RegSeq;
1342  SDValue V0 = N->getOperand(FirstTblReg + 0);
1343  SDValue V1 = N->getOperand(FirstTblReg + 1);
1344  if (NumVecs == 2)
1345    RegSeq = SDValue(PairDRegs(MVT::v16i8, V0, V1), 0);
1346  else {
1347    SDValue V2 = N->getOperand(FirstTblReg + 2);
1348    // If it's a vtbl3, form a quad D-register and leave the last part as
1349    // an undef.
1350    SDValue V3 = (NumVecs == 3)
1351      ? SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, dl, VT), 0)
1352      : N->getOperand(FirstTblReg + 3);
1353    RegSeq = SDValue(QuadDRegs(MVT::v4i64, V0, V1, V2, V3), 0);
1354  }
1355
1356  // Now extract the D registers back out.
1357  SmallVector<SDValue, 6> Ops;
1358  if (IsExt)
1359    Ops.push_back(N->getOperand(1));
1360  Ops.push_back(CurDAG->getTargetExtractSubreg(ARM::dsub_0, dl, VT, RegSeq));
1361  Ops.push_back(CurDAG->getTargetExtractSubreg(ARM::dsub_1, dl, VT, RegSeq));
1362  if (NumVecs > 2)
1363    Ops.push_back(CurDAG->getTargetExtractSubreg(ARM::dsub_2, dl, VT, RegSeq));
1364  if (NumVecs > 3)
1365    Ops.push_back(CurDAG->getTargetExtractSubreg(ARM::dsub_3, dl, VT, RegSeq));
1366
1367  Ops.push_back(N->getOperand(FirstTblReg + NumVecs));
1368  Ops.push_back(getAL(CurDAG)); // predicate
1369  Ops.push_back(CurDAG->getRegister(0, MVT::i32)); // predicate register
1370  return CurDAG->getMachineNode(Opc, dl, VT, Ops.data(), Ops.size());
1371}
1372
1373SDNode *ARMDAGToDAGISel::SelectV6T2BitfieldExtractOp(SDNode *N,
1374                                                     bool isSigned) {
1375  if (!Subtarget->hasV6T2Ops())
1376    return NULL;
1377
1378  unsigned Opc = isSigned ? (Subtarget->isThumb() ? ARM::t2SBFX : ARM::SBFX)
1379    : (Subtarget->isThumb() ? ARM::t2UBFX : ARM::UBFX);
1380
1381
1382  // For unsigned extracts, check for a shift right and mask
1383  unsigned And_imm = 0;
1384  if (N->getOpcode() == ISD::AND) {
1385    if (isOpcWithIntImmediate(N, ISD::AND, And_imm)) {
1386
1387      // The immediate is a mask of the low bits iff imm & (imm+1) == 0
1388      if (And_imm & (And_imm + 1))
1389        return NULL;
1390
1391      unsigned Srl_imm = 0;
1392      if (isOpcWithIntImmediate(N->getOperand(0).getNode(), ISD::SRL,
1393                                Srl_imm)) {
1394        assert(Srl_imm > 0 && Srl_imm < 32 && "bad amount in shift node!");
1395
1396        unsigned Width = CountTrailingOnes_32(And_imm);
1397        unsigned LSB = Srl_imm;
1398        SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
1399        SDValue Ops[] = { N->getOperand(0).getOperand(0),
1400                          CurDAG->getTargetConstant(LSB, MVT::i32),
1401                          CurDAG->getTargetConstant(Width, MVT::i32),
1402          getAL(CurDAG), Reg0 };
1403        return CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops, 5);
1404      }
1405    }
1406    return NULL;
1407  }
1408
1409  // Otherwise, we're looking for a shift of a shift
1410  unsigned Shl_imm = 0;
1411  if (isOpcWithIntImmediate(N->getOperand(0).getNode(), ISD::SHL, Shl_imm)) {
1412    assert(Shl_imm > 0 && Shl_imm < 32 && "bad amount in shift node!");
1413    unsigned Srl_imm = 0;
1414    if (isInt32Immediate(N->getOperand(1), Srl_imm)) {
1415      assert(Srl_imm > 0 && Srl_imm < 32 && "bad amount in shift node!");
1416      unsigned Width = 32 - Srl_imm;
1417      int LSB = Srl_imm - Shl_imm;
1418      if (LSB < 0)
1419        return NULL;
1420      SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
1421      SDValue Ops[] = { N->getOperand(0).getOperand(0),
1422                        CurDAG->getTargetConstant(LSB, MVT::i32),
1423                        CurDAG->getTargetConstant(Width, MVT::i32),
1424                        getAL(CurDAG), Reg0 };
1425      return CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops, 5);
1426    }
1427  }
1428  return NULL;
1429}
1430
1431SDNode *ARMDAGToDAGISel::
1432SelectT2CMOVShiftOp(SDNode *N, SDValue FalseVal, SDValue TrueVal,
1433                    ARMCC::CondCodes CCVal, SDValue CCR, SDValue InFlag) {
1434  SDValue CPTmp0;
1435  SDValue CPTmp1;
1436  if (SelectT2ShifterOperandReg(N, TrueVal, CPTmp0, CPTmp1)) {
1437    unsigned SOVal = cast<ConstantSDNode>(CPTmp1)->getZExtValue();
1438    unsigned SOShOp = ARM_AM::getSORegShOp(SOVal);
1439    unsigned Opc = 0;
1440    switch (SOShOp) {
1441    case ARM_AM::lsl: Opc = ARM::t2MOVCClsl; break;
1442    case ARM_AM::lsr: Opc = ARM::t2MOVCClsr; break;
1443    case ARM_AM::asr: Opc = ARM::t2MOVCCasr; break;
1444    case ARM_AM::ror: Opc = ARM::t2MOVCCror; break;
1445    default:
1446      llvm_unreachable("Unknown so_reg opcode!");
1447      break;
1448    }
1449    SDValue SOShImm =
1450      CurDAG->getTargetConstant(ARM_AM::getSORegOffset(SOVal), MVT::i32);
1451    SDValue CC = CurDAG->getTargetConstant(CCVal, MVT::i32);
1452    SDValue Ops[] = { FalseVal, CPTmp0, SOShImm, CC, CCR, InFlag };
1453    return CurDAG->SelectNodeTo(N, Opc, MVT::i32,Ops, 6);
1454  }
1455  return 0;
1456}
1457
1458SDNode *ARMDAGToDAGISel::
1459SelectARMCMOVShiftOp(SDNode *N, SDValue FalseVal, SDValue TrueVal,
1460                     ARMCC::CondCodes CCVal, SDValue CCR, SDValue InFlag) {
1461  SDValue CPTmp0;
1462  SDValue CPTmp1;
1463  SDValue CPTmp2;
1464  if (SelectShifterOperandReg(N, TrueVal, CPTmp0, CPTmp1, CPTmp2)) {
1465    SDValue CC = CurDAG->getTargetConstant(CCVal, MVT::i32);
1466    SDValue Ops[] = { FalseVal, CPTmp0, CPTmp1, CPTmp2, CC, CCR, InFlag };
1467    return CurDAG->SelectNodeTo(N, ARM::MOVCCs, MVT::i32, Ops, 7);
1468  }
1469  return 0;
1470}
1471
1472SDNode *ARMDAGToDAGISel::
1473SelectT2CMOVSoImmOp(SDNode *N, SDValue FalseVal, SDValue TrueVal,
1474                    ARMCC::CondCodes CCVal, SDValue CCR, SDValue InFlag) {
1475  ConstantSDNode *T = dyn_cast<ConstantSDNode>(TrueVal);
1476  if (!T)
1477    return 0;
1478
1479  if (Pred_t2_so_imm(TrueVal.getNode())) {
1480    SDValue True = CurDAG->getTargetConstant(T->getZExtValue(), MVT::i32);
1481    SDValue CC = CurDAG->getTargetConstant(CCVal, MVT::i32);
1482    SDValue Ops[] = { FalseVal, True, CC, CCR, InFlag };
1483    return CurDAG->SelectNodeTo(N,
1484                                ARM::t2MOVCCi, MVT::i32, Ops, 5);
1485  }
1486  return 0;
1487}
1488
1489SDNode *ARMDAGToDAGISel::
1490SelectARMCMOVSoImmOp(SDNode *N, SDValue FalseVal, SDValue TrueVal,
1491                     ARMCC::CondCodes CCVal, SDValue CCR, SDValue InFlag) {
1492  ConstantSDNode *T = dyn_cast<ConstantSDNode>(TrueVal);
1493  if (!T)
1494    return 0;
1495
1496  if (Pred_so_imm(TrueVal.getNode())) {
1497    SDValue True = CurDAG->getTargetConstant(T->getZExtValue(), MVT::i32);
1498    SDValue CC = CurDAG->getTargetConstant(CCVal, MVT::i32);
1499    SDValue Ops[] = { FalseVal, True, CC, CCR, InFlag };
1500    return CurDAG->SelectNodeTo(N,
1501                                ARM::MOVCCi, MVT::i32, Ops, 5);
1502  }
1503  return 0;
1504}
1505
1506SDNode *ARMDAGToDAGISel::SelectCMOVOp(SDNode *N) {
1507  EVT VT = N->getValueType(0);
1508  SDValue FalseVal = N->getOperand(0);
1509  SDValue TrueVal  = N->getOperand(1);
1510  SDValue CC = N->getOperand(2);
1511  SDValue CCR = N->getOperand(3);
1512  SDValue InFlag = N->getOperand(4);
1513  assert(CC.getOpcode() == ISD::Constant);
1514  assert(CCR.getOpcode() == ISD::Register);
1515  ARMCC::CondCodes CCVal =
1516    (ARMCC::CondCodes)cast<ConstantSDNode>(CC)->getZExtValue();
1517
1518  if (!Subtarget->isThumb1Only() && VT == MVT::i32) {
1519    // Pattern: (ARMcmov:i32 GPR:i32:$false, so_reg:i32:$true, (imm:i32):$cc)
1520    // Emits: (MOVCCs:i32 GPR:i32:$false, so_reg:i32:$true, (imm:i32):$cc)
1521    // Pattern complexity = 18  cost = 1  size = 0
1522    SDValue CPTmp0;
1523    SDValue CPTmp1;
1524    SDValue CPTmp2;
1525    if (Subtarget->isThumb()) {
1526      SDNode *Res = SelectT2CMOVShiftOp(N, FalseVal, TrueVal,
1527                                        CCVal, CCR, InFlag);
1528      if (!Res)
1529        Res = SelectT2CMOVShiftOp(N, TrueVal, FalseVal,
1530                               ARMCC::getOppositeCondition(CCVal), CCR, InFlag);
1531      if (Res)
1532        return Res;
1533    } else {
1534      SDNode *Res = SelectARMCMOVShiftOp(N, FalseVal, TrueVal,
1535                                         CCVal, CCR, InFlag);
1536      if (!Res)
1537        Res = SelectARMCMOVShiftOp(N, TrueVal, FalseVal,
1538                               ARMCC::getOppositeCondition(CCVal), CCR, InFlag);
1539      if (Res)
1540        return Res;
1541    }
1542
1543    // Pattern: (ARMcmov:i32 GPR:i32:$false,
1544    //             (imm:i32)<<P:Pred_so_imm>>:$true,
1545    //             (imm:i32):$cc)
1546    // Emits: (MOVCCi:i32 GPR:i32:$false,
1547    //           (so_imm:i32 (imm:i32):$true), (imm:i32):$cc)
1548    // Pattern complexity = 10  cost = 1  size = 0
1549    if (Subtarget->isThumb()) {
1550      SDNode *Res = SelectT2CMOVSoImmOp(N, FalseVal, TrueVal,
1551                                        CCVal, CCR, InFlag);
1552      if (!Res)
1553        Res = SelectT2CMOVSoImmOp(N, TrueVal, FalseVal,
1554                               ARMCC::getOppositeCondition(CCVal), CCR, InFlag);
1555      if (Res)
1556        return Res;
1557    } else {
1558      SDNode *Res = SelectARMCMOVSoImmOp(N, FalseVal, TrueVal,
1559                                         CCVal, CCR, InFlag);
1560      if (!Res)
1561        Res = SelectARMCMOVSoImmOp(N, TrueVal, FalseVal,
1562                               ARMCC::getOppositeCondition(CCVal), CCR, InFlag);
1563      if (Res)
1564        return Res;
1565    }
1566  }
1567
1568  // Pattern: (ARMcmov:i32 GPR:i32:$false, GPR:i32:$true, (imm:i32):$cc)
1569  // Emits: (MOVCCr:i32 GPR:i32:$false, GPR:i32:$true, (imm:i32):$cc)
1570  // Pattern complexity = 6  cost = 1  size = 0
1571  //
1572  // Pattern: (ARMcmov:i32 GPR:i32:$false, GPR:i32:$true, (imm:i32):$cc)
1573  // Emits: (tMOVCCr:i32 GPR:i32:$false, GPR:i32:$true, (imm:i32):$cc)
1574  // Pattern complexity = 6  cost = 11  size = 0
1575  //
1576  // Also FCPYScc and FCPYDcc.
1577  SDValue Tmp2 = CurDAG->getTargetConstant(CCVal, MVT::i32);
1578  SDValue Ops[] = { FalseVal, TrueVal, Tmp2, CCR, InFlag };
1579  unsigned Opc = 0;
1580  switch (VT.getSimpleVT().SimpleTy) {
1581  default: assert(false && "Illegal conditional move type!");
1582    break;
1583  case MVT::i32:
1584    Opc = Subtarget->isThumb()
1585      ? (Subtarget->hasThumb2() ? ARM::t2MOVCCr : ARM::tMOVCCr_pseudo)
1586      : ARM::MOVCCr;
1587    break;
1588  case MVT::f32:
1589    Opc = ARM::VMOVScc;
1590    break;
1591  case MVT::f64:
1592    Opc = ARM::VMOVDcc;
1593    break;
1594  }
1595  return CurDAG->SelectNodeTo(N, Opc, VT, Ops, 5);
1596}
1597
1598SDNode *ARMDAGToDAGISel::SelectConcatVector(SDNode *N) {
1599  // The only time a CONCAT_VECTORS operation can have legal types is when
1600  // two 64-bit vectors are concatenated to a 128-bit vector.
1601  EVT VT = N->getValueType(0);
1602  if (!VT.is128BitVector() || N->getNumOperands() != 2)
1603    llvm_unreachable("unexpected CONCAT_VECTORS");
1604  DebugLoc dl = N->getDebugLoc();
1605  SDValue V0 = N->getOperand(0);
1606  SDValue V1 = N->getOperand(1);
1607  SDValue SubReg0 = CurDAG->getTargetConstant(ARM::dsub_0, MVT::i32);
1608  SDValue SubReg1 = CurDAG->getTargetConstant(ARM::dsub_1, MVT::i32);
1609  const SDValue Ops[] = { V0, SubReg0, V1, SubReg1 };
1610  return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops, 4);
1611}
1612
1613SDNode *ARMDAGToDAGISel::Select(SDNode *N) {
1614  DebugLoc dl = N->getDebugLoc();
1615
1616  if (N->isMachineOpcode())
1617    return NULL;   // Already selected.
1618
1619  switch (N->getOpcode()) {
1620  default: break;
1621  case ISD::Constant: {
1622    unsigned Val = cast<ConstantSDNode>(N)->getZExtValue();
1623    bool UseCP = true;
1624    if (Subtarget->hasThumb2())
1625      // Thumb2-aware targets have the MOVT instruction, so all immediates can
1626      // be done with MOV + MOVT, at worst.
1627      UseCP = 0;
1628    else {
1629      if (Subtarget->isThumb()) {
1630        UseCP = (Val > 255 &&                          // MOV
1631                 ~Val > 255 &&                         // MOV + MVN
1632                 !ARM_AM::isThumbImmShiftedVal(Val));  // MOV + LSL
1633      } else
1634        UseCP = (ARM_AM::getSOImmVal(Val) == -1 &&     // MOV
1635                 ARM_AM::getSOImmVal(~Val) == -1 &&    // MVN
1636                 !ARM_AM::isSOImmTwoPartVal(Val));     // two instrs.
1637    }
1638
1639    if (UseCP) {
1640      SDValue CPIdx =
1641        CurDAG->getTargetConstantPool(ConstantInt::get(
1642                                  Type::getInt32Ty(*CurDAG->getContext()), Val),
1643                                      TLI.getPointerTy());
1644
1645      SDNode *ResNode;
1646      if (Subtarget->isThumb1Only()) {
1647        SDValue Pred = getAL(CurDAG);
1648        SDValue PredReg = CurDAG->getRegister(0, MVT::i32);
1649        SDValue Ops[] = { CPIdx, Pred, PredReg, CurDAG->getEntryNode() };
1650        ResNode = CurDAG->getMachineNode(ARM::tLDRcp, dl, MVT::i32, MVT::Other,
1651                                         Ops, 4);
1652      } else {
1653        SDValue Ops[] = {
1654          CPIdx,
1655          CurDAG->getRegister(0, MVT::i32),
1656          CurDAG->getTargetConstant(0, MVT::i32),
1657          getAL(CurDAG),
1658          CurDAG->getRegister(0, MVT::i32),
1659          CurDAG->getEntryNode()
1660        };
1661        ResNode=CurDAG->getMachineNode(ARM::LDRcp, dl, MVT::i32, MVT::Other,
1662                                       Ops, 6);
1663      }
1664      ReplaceUses(SDValue(N, 0), SDValue(ResNode, 0));
1665      return NULL;
1666    }
1667
1668    // Other cases are autogenerated.
1669    break;
1670  }
1671  case ISD::FrameIndex: {
1672    // Selects to ADDri FI, 0 which in turn will become ADDri SP, imm.
1673    int FI = cast<FrameIndexSDNode>(N)->getIndex();
1674    SDValue TFI = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
1675    if (Subtarget->isThumb1Only()) {
1676      return CurDAG->SelectNodeTo(N, ARM::tADDrSPi, MVT::i32, TFI,
1677                                  CurDAG->getTargetConstant(0, MVT::i32));
1678    } else {
1679      unsigned Opc = ((Subtarget->isThumb() && Subtarget->hasThumb2()) ?
1680                      ARM::t2ADDri : ARM::ADDri);
1681      SDValue Ops[] = { TFI, CurDAG->getTargetConstant(0, MVT::i32),
1682                        getAL(CurDAG), CurDAG->getRegister(0, MVT::i32),
1683                        CurDAG->getRegister(0, MVT::i32) };
1684      return CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops, 5);
1685    }
1686  }
1687  case ISD::SRL:
1688    if (SDNode *I = SelectV6T2BitfieldExtractOp(N, false))
1689      return I;
1690    break;
1691  case ISD::SRA:
1692    if (SDNode *I = SelectV6T2BitfieldExtractOp(N, true))
1693      return I;
1694    break;
1695  case ISD::MUL:
1696    if (Subtarget->isThumb1Only())
1697      break;
1698    if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(N->getOperand(1))) {
1699      unsigned RHSV = C->getZExtValue();
1700      if (!RHSV) break;
1701      if (isPowerOf2_32(RHSV-1)) {  // 2^n+1?
1702        unsigned ShImm = Log2_32(RHSV-1);
1703        if (ShImm >= 32)
1704          break;
1705        SDValue V = N->getOperand(0);
1706        ShImm = ARM_AM::getSORegOpc(ARM_AM::lsl, ShImm);
1707        SDValue ShImmOp = CurDAG->getTargetConstant(ShImm, MVT::i32);
1708        SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
1709        if (Subtarget->isThumb()) {
1710          SDValue Ops[] = { V, V, ShImmOp, getAL(CurDAG), Reg0, Reg0 };
1711          return CurDAG->SelectNodeTo(N, ARM::t2ADDrs, MVT::i32, Ops, 6);
1712        } else {
1713          SDValue Ops[] = { V, V, Reg0, ShImmOp, getAL(CurDAG), Reg0, Reg0 };
1714          return CurDAG->SelectNodeTo(N, ARM::ADDrs, MVT::i32, Ops, 7);
1715        }
1716      }
1717      if (isPowerOf2_32(RHSV+1)) {  // 2^n-1?
1718        unsigned ShImm = Log2_32(RHSV+1);
1719        if (ShImm >= 32)
1720          break;
1721        SDValue V = N->getOperand(0);
1722        ShImm = ARM_AM::getSORegOpc(ARM_AM::lsl, ShImm);
1723        SDValue ShImmOp = CurDAG->getTargetConstant(ShImm, MVT::i32);
1724        SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
1725        if (Subtarget->isThumb()) {
1726          SDValue Ops[] = { V, V, ShImmOp, getAL(CurDAG), Reg0, Reg0 };
1727          return CurDAG->SelectNodeTo(N, ARM::t2RSBrs, MVT::i32, Ops, 6);
1728        } else {
1729          SDValue Ops[] = { V, V, Reg0, ShImmOp, getAL(CurDAG), Reg0, Reg0 };
1730          return CurDAG->SelectNodeTo(N, ARM::RSBrs, MVT::i32, Ops, 7);
1731        }
1732      }
1733    }
1734    break;
1735  case ISD::AND: {
1736    // Check for unsigned bitfield extract
1737    if (SDNode *I = SelectV6T2BitfieldExtractOp(N, false))
1738      return I;
1739
1740    // (and (or x, c2), c1) and top 16-bits of c1 and c2 match, lower 16-bits
1741    // of c1 are 0xffff, and lower 16-bit of c2 are 0. That is, the top 16-bits
1742    // are entirely contributed by c2 and lower 16-bits are entirely contributed
1743    // by x. That's equal to (or (and x, 0xffff), (and c1, 0xffff0000)).
1744    // Select it to: "movt x, ((c1 & 0xffff) >> 16)
1745    EVT VT = N->getValueType(0);
1746    if (VT != MVT::i32)
1747      break;
1748    unsigned Opc = (Subtarget->isThumb() && Subtarget->hasThumb2())
1749      ? ARM::t2MOVTi16
1750      : (Subtarget->hasV6T2Ops() ? ARM::MOVTi16 : 0);
1751    if (!Opc)
1752      break;
1753    SDValue N0 = N->getOperand(0), N1 = N->getOperand(1);
1754    ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
1755    if (!N1C)
1756      break;
1757    if (N0.getOpcode() == ISD::OR && N0.getNode()->hasOneUse()) {
1758      SDValue N2 = N0.getOperand(1);
1759      ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(N2);
1760      if (!N2C)
1761        break;
1762      unsigned N1CVal = N1C->getZExtValue();
1763      unsigned N2CVal = N2C->getZExtValue();
1764      if ((N1CVal & 0xffff0000U) == (N2CVal & 0xffff0000U) &&
1765          (N1CVal & 0xffffU) == 0xffffU &&
1766          (N2CVal & 0xffffU) == 0x0U) {
1767        SDValue Imm16 = CurDAG->getTargetConstant((N2CVal & 0xFFFF0000U) >> 16,
1768                                                  MVT::i32);
1769        SDValue Ops[] = { N0.getOperand(0), Imm16,
1770                          getAL(CurDAG), CurDAG->getRegister(0, MVT::i32) };
1771        return CurDAG->getMachineNode(Opc, dl, VT, Ops, 4);
1772      }
1773    }
1774    break;
1775  }
1776  case ARMISD::VMOVRRD:
1777    return CurDAG->getMachineNode(ARM::VMOVRRD, dl, MVT::i32, MVT::i32,
1778                                  N->getOperand(0), getAL(CurDAG),
1779                                  CurDAG->getRegister(0, MVT::i32));
1780  case ISD::UMUL_LOHI: {
1781    if (Subtarget->isThumb1Only())
1782      break;
1783    if (Subtarget->isThumb()) {
1784      SDValue Ops[] = { N->getOperand(0), N->getOperand(1),
1785                        getAL(CurDAG), CurDAG->getRegister(0, MVT::i32),
1786                        CurDAG->getRegister(0, MVT::i32) };
1787      return CurDAG->getMachineNode(ARM::t2UMULL, dl, MVT::i32, MVT::i32,Ops,4);
1788    } else {
1789      SDValue Ops[] = { N->getOperand(0), N->getOperand(1),
1790                        getAL(CurDAG), CurDAG->getRegister(0, MVT::i32),
1791                        CurDAG->getRegister(0, MVT::i32) };
1792      return CurDAG->getMachineNode(ARM::UMULL, dl, MVT::i32, MVT::i32, Ops, 5);
1793    }
1794  }
1795  case ISD::SMUL_LOHI: {
1796    if (Subtarget->isThumb1Only())
1797      break;
1798    if (Subtarget->isThumb()) {
1799      SDValue Ops[] = { N->getOperand(0), N->getOperand(1),
1800                        getAL(CurDAG), CurDAG->getRegister(0, MVT::i32) };
1801      return CurDAG->getMachineNode(ARM::t2SMULL, dl, MVT::i32, MVT::i32,Ops,4);
1802    } else {
1803      SDValue Ops[] = { N->getOperand(0), N->getOperand(1),
1804                        getAL(CurDAG), CurDAG->getRegister(0, MVT::i32),
1805                        CurDAG->getRegister(0, MVT::i32) };
1806      return CurDAG->getMachineNode(ARM::SMULL, dl, MVT::i32, MVT::i32, Ops, 5);
1807    }
1808  }
1809  case ISD::LOAD: {
1810    SDNode *ResNode = 0;
1811    if (Subtarget->isThumb() && Subtarget->hasThumb2())
1812      ResNode = SelectT2IndexedLoad(N);
1813    else
1814      ResNode = SelectARMIndexedLoad(N);
1815    if (ResNode)
1816      return ResNode;
1817    // Other cases are autogenerated.
1818    break;
1819  }
1820  case ARMISD::BRCOND: {
1821    // Pattern: (ARMbrcond:void (bb:Other):$dst, (imm:i32):$cc)
1822    // Emits: (Bcc:void (bb:Other):$dst, (imm:i32):$cc)
1823    // Pattern complexity = 6  cost = 1  size = 0
1824
1825    // Pattern: (ARMbrcond:void (bb:Other):$dst, (imm:i32):$cc)
1826    // Emits: (tBcc:void (bb:Other):$dst, (imm:i32):$cc)
1827    // Pattern complexity = 6  cost = 1  size = 0
1828
1829    // Pattern: (ARMbrcond:void (bb:Other):$dst, (imm:i32):$cc)
1830    // Emits: (t2Bcc:void (bb:Other):$dst, (imm:i32):$cc)
1831    // Pattern complexity = 6  cost = 1  size = 0
1832
1833    unsigned Opc = Subtarget->isThumb() ?
1834      ((Subtarget->hasThumb2()) ? ARM::t2Bcc : ARM::tBcc) : ARM::Bcc;
1835    SDValue Chain = N->getOperand(0);
1836    SDValue N1 = N->getOperand(1);
1837    SDValue N2 = N->getOperand(2);
1838    SDValue N3 = N->getOperand(3);
1839    SDValue InFlag = N->getOperand(4);
1840    assert(N1.getOpcode() == ISD::BasicBlock);
1841    assert(N2.getOpcode() == ISD::Constant);
1842    assert(N3.getOpcode() == ISD::Register);
1843
1844    SDValue Tmp2 = CurDAG->getTargetConstant(((unsigned)
1845                               cast<ConstantSDNode>(N2)->getZExtValue()),
1846                               MVT::i32);
1847    SDValue Ops[] = { N1, Tmp2, N3, Chain, InFlag };
1848    SDNode *ResNode = CurDAG->getMachineNode(Opc, dl, MVT::Other,
1849                                             MVT::Flag, Ops, 5);
1850    Chain = SDValue(ResNode, 0);
1851    if (N->getNumValues() == 2) {
1852      InFlag = SDValue(ResNode, 1);
1853      ReplaceUses(SDValue(N, 1), InFlag);
1854    }
1855    ReplaceUses(SDValue(N, 0),
1856                SDValue(Chain.getNode(), Chain.getResNo()));
1857    return NULL;
1858  }
1859  case ARMISD::CMOV:
1860    return SelectCMOVOp(N);
1861  case ARMISD::CNEG: {
1862    EVT VT = N->getValueType(0);
1863    SDValue N0 = N->getOperand(0);
1864    SDValue N1 = N->getOperand(1);
1865    SDValue N2 = N->getOperand(2);
1866    SDValue N3 = N->getOperand(3);
1867    SDValue InFlag = N->getOperand(4);
1868    assert(N2.getOpcode() == ISD::Constant);
1869    assert(N3.getOpcode() == ISD::Register);
1870
1871    SDValue Tmp2 = CurDAG->getTargetConstant(((unsigned)
1872                               cast<ConstantSDNode>(N2)->getZExtValue()),
1873                               MVT::i32);
1874    SDValue Ops[] = { N0, N1, Tmp2, N3, InFlag };
1875    unsigned Opc = 0;
1876    switch (VT.getSimpleVT().SimpleTy) {
1877    default: assert(false && "Illegal conditional move type!");
1878      break;
1879    case MVT::f32:
1880      Opc = ARM::VNEGScc;
1881      break;
1882    case MVT::f64:
1883      Opc = ARM::VNEGDcc;
1884      break;
1885    }
1886    return CurDAG->SelectNodeTo(N, Opc, VT, Ops, 5);
1887  }
1888
1889  case ARMISD::VZIP: {
1890    unsigned Opc = 0;
1891    EVT VT = N->getValueType(0);
1892    switch (VT.getSimpleVT().SimpleTy) {
1893    default: return NULL;
1894    case MVT::v8i8:  Opc = ARM::VZIPd8; break;
1895    case MVT::v4i16: Opc = ARM::VZIPd16; break;
1896    case MVT::v2f32:
1897    case MVT::v2i32: Opc = ARM::VZIPd32; break;
1898    case MVT::v16i8: Opc = ARM::VZIPq8; break;
1899    case MVT::v8i16: Opc = ARM::VZIPq16; break;
1900    case MVT::v4f32:
1901    case MVT::v4i32: Opc = ARM::VZIPq32; break;
1902    }
1903    SDValue Pred = getAL(CurDAG);
1904    SDValue PredReg = CurDAG->getRegister(0, MVT::i32);
1905    SDValue Ops[] = { N->getOperand(0), N->getOperand(1), Pred, PredReg };
1906    return CurDAG->getMachineNode(Opc, dl, VT, VT, Ops, 4);
1907  }
1908  case ARMISD::VUZP: {
1909    unsigned Opc = 0;
1910    EVT VT = N->getValueType(0);
1911    switch (VT.getSimpleVT().SimpleTy) {
1912    default: return NULL;
1913    case MVT::v8i8:  Opc = ARM::VUZPd8; break;
1914    case MVT::v4i16: Opc = ARM::VUZPd16; break;
1915    case MVT::v2f32:
1916    case MVT::v2i32: Opc = ARM::VUZPd32; break;
1917    case MVT::v16i8: Opc = ARM::VUZPq8; break;
1918    case MVT::v8i16: Opc = ARM::VUZPq16; break;
1919    case MVT::v4f32:
1920    case MVT::v4i32: Opc = ARM::VUZPq32; break;
1921    }
1922    SDValue Pred = getAL(CurDAG);
1923    SDValue PredReg = CurDAG->getRegister(0, MVT::i32);
1924    SDValue Ops[] = { N->getOperand(0), N->getOperand(1), Pred, PredReg };
1925    return CurDAG->getMachineNode(Opc, dl, VT, VT, Ops, 4);
1926  }
1927  case ARMISD::VTRN: {
1928    unsigned Opc = 0;
1929    EVT VT = N->getValueType(0);
1930    switch (VT.getSimpleVT().SimpleTy) {
1931    default: return NULL;
1932    case MVT::v8i8:  Opc = ARM::VTRNd8; break;
1933    case MVT::v4i16: Opc = ARM::VTRNd16; break;
1934    case MVT::v2f32:
1935    case MVT::v2i32: Opc = ARM::VTRNd32; break;
1936    case MVT::v16i8: Opc = ARM::VTRNq8; break;
1937    case MVT::v8i16: Opc = ARM::VTRNq16; break;
1938    case MVT::v4f32:
1939    case MVT::v4i32: Opc = ARM::VTRNq32; break;
1940    }
1941    SDValue Pred = getAL(CurDAG);
1942    SDValue PredReg = CurDAG->getRegister(0, MVT::i32);
1943    SDValue Ops[] = { N->getOperand(0), N->getOperand(1), Pred, PredReg };
1944    return CurDAG->getMachineNode(Opc, dl, VT, VT, Ops, 4);
1945  }
1946  case ARMISD::BUILD_VECTOR: {
1947    EVT VecVT = N->getValueType(0);
1948    EVT EltVT = VecVT.getVectorElementType();
1949    unsigned NumElts = VecVT.getVectorNumElements();
1950    if (EltVT.getSimpleVT() == MVT::f64) {
1951      assert(NumElts == 2 && "unexpected type for BUILD_VECTOR");
1952      return PairDRegs(VecVT, N->getOperand(0), N->getOperand(1));
1953    }
1954    assert(EltVT.getSimpleVT() == MVT::f32 &&
1955           "unexpected type for BUILD_VECTOR");
1956    if (NumElts == 2)
1957      return PairSRegs(VecVT, N->getOperand(0), N->getOperand(1));
1958    assert(NumElts == 4 && "unexpected type for BUILD_VECTOR");
1959    return QuadSRegs(VecVT, N->getOperand(0), N->getOperand(1),
1960                     N->getOperand(2), N->getOperand(3));
1961  }
1962
1963  case ISD::INTRINSIC_VOID:
1964  case ISD::INTRINSIC_W_CHAIN: {
1965    unsigned IntNo = cast<ConstantSDNode>(N->getOperand(1))->getZExtValue();
1966    switch (IntNo) {
1967    default:
1968      break;
1969
1970    case Intrinsic::arm_neon_vld1: {
1971      unsigned DOpcodes[] = { ARM::VLD1d8, ARM::VLD1d16,
1972                              ARM::VLD1d32, ARM::VLD1d64 };
1973      unsigned QOpcodes[] = { ARM::VLD1q8Pseudo, ARM::VLD1q16Pseudo,
1974                              ARM::VLD1q32Pseudo, ARM::VLD1q64Pseudo };
1975      return SelectVLD(N, 1, DOpcodes, QOpcodes, 0);
1976    }
1977
1978    case Intrinsic::arm_neon_vld2: {
1979      unsigned DOpcodes[] = { ARM::VLD2d8Pseudo, ARM::VLD2d16Pseudo,
1980                              ARM::VLD2d32Pseudo, ARM::VLD1q64Pseudo };
1981      unsigned QOpcodes[] = { ARM::VLD2q8Pseudo, ARM::VLD2q16Pseudo,
1982                              ARM::VLD2q32Pseudo };
1983      return SelectVLD(N, 2, DOpcodes, QOpcodes, 0);
1984    }
1985
1986    case Intrinsic::arm_neon_vld3: {
1987      unsigned DOpcodes[] = { ARM::VLD3d8Pseudo, ARM::VLD3d16Pseudo,
1988                              ARM::VLD3d32Pseudo, ARM::VLD1d64TPseudo };
1989      unsigned QOpcodes0[] = { ARM::VLD3q8Pseudo_UPD,
1990                               ARM::VLD3q16Pseudo_UPD,
1991                               ARM::VLD3q32Pseudo_UPD };
1992      unsigned QOpcodes1[] = { ARM::VLD3q8oddPseudo_UPD,
1993                               ARM::VLD3q16oddPseudo_UPD,
1994                               ARM::VLD3q32oddPseudo_UPD };
1995      return SelectVLD(N, 3, DOpcodes, QOpcodes0, QOpcodes1);
1996    }
1997
1998    case Intrinsic::arm_neon_vld4: {
1999      unsigned DOpcodes[] = { ARM::VLD4d8Pseudo, ARM::VLD4d16Pseudo,
2000                              ARM::VLD4d32Pseudo, ARM::VLD1d64QPseudo };
2001      unsigned QOpcodes0[] = { ARM::VLD4q8Pseudo_UPD,
2002                               ARM::VLD4q16Pseudo_UPD,
2003                               ARM::VLD4q32Pseudo_UPD };
2004      unsigned QOpcodes1[] = { ARM::VLD4q8oddPseudo_UPD,
2005                               ARM::VLD4q16oddPseudo_UPD,
2006                               ARM::VLD4q32oddPseudo_UPD };
2007      return SelectVLD(N, 4, DOpcodes, QOpcodes0, QOpcodes1);
2008    }
2009
2010    case Intrinsic::arm_neon_vld2lane: {
2011      unsigned DOpcodes[] = { ARM::VLD2LNd8Pseudo, ARM::VLD2LNd16Pseudo,
2012                              ARM::VLD2LNd32Pseudo };
2013      unsigned QOpcodes[] = { ARM::VLD2LNq16Pseudo, ARM::VLD2LNq32Pseudo };
2014      return SelectVLDSTLane(N, true, 2, DOpcodes, QOpcodes);
2015    }
2016
2017    case Intrinsic::arm_neon_vld3lane: {
2018      unsigned DOpcodes[] = { ARM::VLD3LNd8Pseudo, ARM::VLD3LNd16Pseudo,
2019                              ARM::VLD3LNd32Pseudo };
2020      unsigned QOpcodes[] = { ARM::VLD3LNq16Pseudo, ARM::VLD3LNq32Pseudo };
2021      return SelectVLDSTLane(N, true, 3, DOpcodes, QOpcodes);
2022    }
2023
2024    case Intrinsic::arm_neon_vld4lane: {
2025      unsigned DOpcodes[] = { ARM::VLD4LNd8Pseudo, ARM::VLD4LNd16Pseudo,
2026                              ARM::VLD4LNd32Pseudo };
2027      unsigned QOpcodes[] = { ARM::VLD4LNq16Pseudo, ARM::VLD4LNq32Pseudo };
2028      return SelectVLDSTLane(N, true, 4, DOpcodes, QOpcodes);
2029    }
2030
2031    case Intrinsic::arm_neon_vst1: {
2032      unsigned DOpcodes[] = { ARM::VST1d8, ARM::VST1d16,
2033                              ARM::VST1d32, ARM::VST1d64 };
2034      unsigned QOpcodes[] = { ARM::VST1q8Pseudo, ARM::VST1q16Pseudo,
2035                              ARM::VST1q32Pseudo, ARM::VST1q64Pseudo };
2036      return SelectVST(N, 1, DOpcodes, QOpcodes, 0);
2037    }
2038
2039    case Intrinsic::arm_neon_vst2: {
2040      unsigned DOpcodes[] = { ARM::VST2d8Pseudo, ARM::VST2d16Pseudo,
2041                              ARM::VST2d32Pseudo, ARM::VST1q64Pseudo };
2042      unsigned QOpcodes[] = { ARM::VST2q8Pseudo, ARM::VST2q16Pseudo,
2043                              ARM::VST2q32Pseudo };
2044      return SelectVST(N, 2, DOpcodes, QOpcodes, 0);
2045    }
2046
2047    case Intrinsic::arm_neon_vst3: {
2048      unsigned DOpcodes[] = { ARM::VST3d8Pseudo, ARM::VST3d16Pseudo,
2049                              ARM::VST3d32Pseudo, ARM::VST1d64TPseudo };
2050      unsigned QOpcodes0[] = { ARM::VST3q8Pseudo_UPD,
2051                               ARM::VST3q16Pseudo_UPD,
2052                               ARM::VST3q32Pseudo_UPD };
2053      unsigned QOpcodes1[] = { ARM::VST3q8oddPseudo_UPD,
2054                               ARM::VST3q16oddPseudo_UPD,
2055                               ARM::VST3q32oddPseudo_UPD };
2056      return SelectVST(N, 3, DOpcodes, QOpcodes0, QOpcodes1);
2057    }
2058
2059    case Intrinsic::arm_neon_vst4: {
2060      unsigned DOpcodes[] = { ARM::VST4d8Pseudo, ARM::VST4d16Pseudo,
2061                              ARM::VST4d32Pseudo, ARM::VST1d64QPseudo };
2062      unsigned QOpcodes0[] = { ARM::VST4q8Pseudo_UPD,
2063                               ARM::VST4q16Pseudo_UPD,
2064                               ARM::VST4q32Pseudo_UPD };
2065      unsigned QOpcodes1[] = { ARM::VST4q8oddPseudo_UPD,
2066                               ARM::VST4q16oddPseudo_UPD,
2067                               ARM::VST4q32oddPseudo_UPD };
2068      return SelectVST(N, 4, DOpcodes, QOpcodes0, QOpcodes1);
2069    }
2070
2071    case Intrinsic::arm_neon_vst2lane: {
2072      unsigned DOpcodes[] = { ARM::VST2LNd8Pseudo, ARM::VST2LNd16Pseudo,
2073                              ARM::VST2LNd32Pseudo };
2074      unsigned QOpcodes[] = { ARM::VST2LNq16Pseudo, ARM::VST2LNq32Pseudo };
2075      return SelectVLDSTLane(N, false, 2, DOpcodes, QOpcodes);
2076    }
2077
2078    case Intrinsic::arm_neon_vst3lane: {
2079      unsigned DOpcodes[] = { ARM::VST3LNd8Pseudo, ARM::VST3LNd16Pseudo,
2080                              ARM::VST3LNd32Pseudo };
2081      unsigned QOpcodes[] = { ARM::VST3LNq16Pseudo, ARM::VST3LNq32Pseudo };
2082      return SelectVLDSTLane(N, false, 3, DOpcodes, QOpcodes);
2083    }
2084
2085    case Intrinsic::arm_neon_vst4lane: {
2086      unsigned DOpcodes[] = { ARM::VST4LNd8Pseudo, ARM::VST4LNd16Pseudo,
2087                              ARM::VST4LNd32Pseudo };
2088      unsigned QOpcodes[] = { ARM::VST4LNq16Pseudo, ARM::VST4LNq32Pseudo };
2089      return SelectVLDSTLane(N, false, 4, DOpcodes, QOpcodes);
2090    }
2091    }
2092    break;
2093  }
2094
2095  case ISD::INTRINSIC_WO_CHAIN: {
2096    unsigned IntNo = cast<ConstantSDNode>(N->getOperand(0))->getZExtValue();
2097    switch (IntNo) {
2098    default:
2099      break;
2100
2101    case Intrinsic::arm_neon_vtbl2:
2102      return SelectVTBL(N, false, 2, ARM::VTBL2);
2103    case Intrinsic::arm_neon_vtbl3:
2104      return SelectVTBL(N, false, 3, ARM::VTBL3);
2105    case Intrinsic::arm_neon_vtbl4:
2106      return SelectVTBL(N, false, 4, ARM::VTBL4);
2107
2108    case Intrinsic::arm_neon_vtbx2:
2109      return SelectVTBL(N, true, 2, ARM::VTBX2);
2110    case Intrinsic::arm_neon_vtbx3:
2111      return SelectVTBL(N, true, 3, ARM::VTBX3);
2112    case Intrinsic::arm_neon_vtbx4:
2113      return SelectVTBL(N, true, 4, ARM::VTBX4);
2114    }
2115    break;
2116  }
2117
2118  case ISD::CONCAT_VECTORS:
2119    return SelectConcatVector(N);
2120  }
2121
2122  return SelectCode(N);
2123}
2124
2125bool ARMDAGToDAGISel::
2126SelectInlineAsmMemoryOperand(const SDValue &Op, char ConstraintCode,
2127                             std::vector<SDValue> &OutOps) {
2128  assert(ConstraintCode == 'm' && "unexpected asm memory constraint");
2129  // Require the address to be in a register.  That is safe for all ARM
2130  // variants and it is hard to do anything much smarter without knowing
2131  // how the operand is used.
2132  OutOps.push_back(Op);
2133  return false;
2134}
2135
2136/// createARMISelDag - This pass converts a legalized DAG into a
2137/// ARM-specific DAG, ready for instruction scheduling.
2138///
2139FunctionPass *llvm::createARMISelDag(ARMBaseTargetMachine &TM,
2140                                     CodeGenOpt::Level OptLevel) {
2141  return new ARMDAGToDAGISel(TM, OptLevel);
2142}
2143