ARMISelDAGToDAG.cpp revision 681a2ad40357fbf0415977d76323e9a03ada84ae 
1//===-- ARMISelDAGToDAG.cpp - A dag to dag inst selector for ARM ----------===//
2//
3//                     The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10// This file defines an instruction selector for the ARM target.
11//
12//===----------------------------------------------------------------------===//
13
14#include "ARM.h"
15#include "ARMAddressingModes.h"
16#include "ARMConstantPoolValue.h"
17#include "ARMISelLowering.h"
18#include "ARMTargetMachine.h"
19#include "llvm/CallingConv.h"
20#include "llvm/Constants.h"
21#include "llvm/DerivedTypes.h"
22#include "llvm/Function.h"
23#include "llvm/Intrinsics.h"
24#include "llvm/LLVMContext.h"
25#include "llvm/CodeGen/MachineFrameInfo.h"
26#include "llvm/CodeGen/MachineFunction.h"
27#include "llvm/CodeGen/MachineInstrBuilder.h"
28#include "llvm/CodeGen/SelectionDAG.h"
29#include "llvm/CodeGen/SelectionDAGISel.h"
30#include "llvm/Target/TargetLowering.h"
31#include "llvm/Target/TargetOptions.h"
32#include "llvm/Support/Compiler.h"
33#include "llvm/Support/Debug.h"
34#include "llvm/Support/ErrorHandling.h"
35#include "llvm/Support/raw_ostream.h"
36
37using namespace llvm;
38
39//===--------------------------------------------------------------------===//
40/// ARMDAGToDAGISel - ARM specific code to select ARM machine
41/// instructions for SelectionDAG operations.
42///
43namespace {
44class ARMDAGToDAGISel : public SelectionDAGISel {
45  ARMBaseTargetMachine &TM;
46
47  /// Subtarget - Keep a pointer to the ARMSubtarget around so that we can
48  /// make the right decision when generating code for different targets.
49  const ARMSubtarget *Subtarget;
50
51public:
52  explicit ARMDAGToDAGISel(ARMBaseTargetMachine &tm,
53                           CodeGenOpt::Level OptLevel)
54    : SelectionDAGISel(tm, OptLevel), TM(tm),
55    Subtarget(&TM.getSubtarget<ARMSubtarget>()) {
56  }
57
58  virtual const char *getPassName() const {
59    return "ARM Instruction Selection";
60  }
61
62  /// getI32Imm - Return a target constant of type i32 with the specified
63  /// value.
64  inline SDValue getI32Imm(unsigned Imm) {
65    return CurDAG->getTargetConstant(Imm, MVT::i32);
66  }
67
68  SDNode *Select(SDValue Op);
69  virtual void InstructionSelect();
70  bool SelectShifterOperandReg(SDValue Op, SDValue N, SDValue &A,
71                               SDValue &B, SDValue &C);
72  bool SelectAddrMode2(SDValue Op, SDValue N, SDValue &Base,
73                       SDValue &Offset, SDValue &Opc);
74  bool SelectAddrMode2Offset(SDValue Op, SDValue N,
75                             SDValue &Offset, SDValue &Opc);
76  bool SelectAddrMode3(SDValue Op, SDValue N, SDValue &Base,
77                       SDValue &Offset, SDValue &Opc);
78  bool SelectAddrMode3Offset(SDValue Op, SDValue N,
79                             SDValue &Offset, SDValue &Opc);
80  bool SelectAddrMode4(SDValue Op, SDValue N, SDValue &Addr,
81                       SDValue &Mode);
82  bool SelectAddrMode5(SDValue Op, SDValue N, SDValue &Base,
83                       SDValue &Offset);
84  bool SelectAddrMode6(SDValue Op, SDValue N, SDValue &Addr, SDValue &Update,
85                       SDValue &Opc);
86
87  bool SelectAddrModePC(SDValue Op, SDValue N, SDValue &Offset,
88                        SDValue &Label);
89
90  bool SelectThumbAddrModeRR(SDValue Op, SDValue N, SDValue &Base,
91                             SDValue &Offset);
92  bool SelectThumbAddrModeRI5(SDValue Op, SDValue N, unsigned Scale,
93                              SDValue &Base, SDValue &OffImm,
94                              SDValue &Offset);
95  bool SelectThumbAddrModeS1(SDValue Op, SDValue N, SDValue &Base,
96                             SDValue &OffImm, SDValue &Offset);
97  bool SelectThumbAddrModeS2(SDValue Op, SDValue N, SDValue &Base,
98                             SDValue &OffImm, SDValue &Offset);
99  bool SelectThumbAddrModeS4(SDValue Op, SDValue N, SDValue &Base,
100                             SDValue &OffImm, SDValue &Offset);
101  bool SelectThumbAddrModeSP(SDValue Op, SDValue N, SDValue &Base,
102                             SDValue &OffImm);
103
104  bool SelectT2ShifterOperandReg(SDValue Op, SDValue N,
105                                 SDValue &BaseReg, SDValue &Opc);
106  bool SelectT2AddrModeImm12(SDValue Op, SDValue N, SDValue &Base,
107                             SDValue &OffImm);
108  bool SelectT2AddrModeImm8(SDValue Op, SDValue N, SDValue &Base,
109                            SDValue &OffImm);
110  bool SelectT2AddrModeImm8Offset(SDValue Op, SDValue N,
111                                 SDValue &OffImm);
112  bool SelectT2AddrModeImm8s4(SDValue Op, SDValue N, SDValue &Base,
113                              SDValue &OffImm);
114  bool SelectT2AddrModeSoReg(SDValue Op, SDValue N, SDValue &Base,
115                             SDValue &OffReg, SDValue &ShImm);
116
117  // Include the pieces autogenerated from the target description.
118#include "ARMGenDAGISel.inc"
119
120private:
121  /// SelectARMIndexedLoad - Indexed (pre/post inc/dec) load matching code for
122  /// ARM.
123  SDNode *SelectARMIndexedLoad(SDValue Op);
124  SDNode *SelectT2IndexedLoad(SDValue Op);
125
126  /// SelectDYN_ALLOC - Select dynamic alloc for Thumb.
127  SDNode *SelectDYN_ALLOC(SDValue Op);
128
129  /// SelectVLD - Select NEON load intrinsics.  NumVecs should
130  /// be 2, 3 or 4.  The opcode arrays specify the instructions used for
131  /// loads of D registers and even subregs and odd subregs of Q registers.
132  /// For NumVecs == 2, QOpcodes1 is not used.
133  SDNode *SelectVLD(SDValue Op, unsigned NumVecs, unsigned *DOpcodes,
134                    unsigned *QOpcodes0, unsigned *QOpcodes1);
135
136  /// SelectVST - Select NEON store intrinsics.  NumVecs should
137  /// be 2, 3 or 4.  The opcode arrays specify the instructions used for
138  /// stores of D registers and even subregs and odd subregs of Q registers.
139  /// For NumVecs == 2, QOpcodes1 is not used.
140  SDNode *SelectVST(SDValue Op, unsigned NumVecs, unsigned *DOpcodes,
141                    unsigned *QOpcodes0, unsigned *QOpcodes1);
142
143  /// SelectVLDSTLane - Select NEON load/store lane intrinsics.  NumVecs should
144  /// be 2, 3 or 4.  The opcode arrays specify the instructions used for
145  /// load/store of D registers and even subregs and odd subregs of Q registers.
146  SDNode *SelectVLDSTLane(SDValue Op, bool IsLoad, unsigned NumVecs,
147                          unsigned *DOpcodes, unsigned *QOpcodes0,
148                          unsigned *QOpcodes1);
149
150  /// SelectV6T2BitfieldExtractOp - Select SBFX/UBFX instructions for ARM.
151  SDNode *SelectV6T2BitfieldExtractOp(SDValue Op, unsigned Opc);
152
153  /// SelectInlineAsmMemoryOperand - Implement addressing mode selection for
154  /// inline asm expressions.
155  virtual bool SelectInlineAsmMemoryOperand(const SDValue &Op,
156                                            char ConstraintCode,
157                                            std::vector<SDValue> &OutOps);
158
159  /// PairDRegs - Insert a pair of double registers into an implicit def to
160  /// form a quad register.
161  SDNode *PairDRegs(EVT VT, SDValue V0, SDValue V1);
162};
163}
164
165/// isInt32Immediate - This method tests to see if the node is a 32-bit constant
166/// operand. If so Imm will receive the 32-bit value.
167static bool isInt32Immediate(SDNode *N, unsigned &Imm) {
168  if (N->getOpcode() == ISD::Constant && N->getValueType(0) == MVT::i32) {
169    Imm = cast<ConstantSDNode>(N)->getZExtValue();
170    return true;
171  }
172  return false;
173}
174
175// isInt32Immediate - This method tests to see if a constant operand.
176// If so Imm will receive the 32 bit value.
177static bool isInt32Immediate(SDValue N, unsigned &Imm) {
178  return isInt32Immediate(N.getNode(), Imm);
179}
180
181// isOpcWithIntImmediate - This method tests to see if the node is a specific
182// opcode and that it has a immediate integer right operand.
183// If so Imm will receive the 32 bit value.
184static bool isOpcWithIntImmediate(SDNode *N, unsigned Opc, unsigned& Imm) {
185  return N->getOpcode() == Opc &&
186         isInt32Immediate(N->getOperand(1).getNode(), Imm);
187}
188
189
190void ARMDAGToDAGISel::InstructionSelect() {
191  DEBUG(BB->dump());
192
193  SelectRoot(*CurDAG);
194  CurDAG->RemoveDeadNodes();
195}
196
197bool ARMDAGToDAGISel::SelectShifterOperandReg(SDValue Op,
198                                              SDValue N,
199                                              SDValue &BaseReg,
200                                              SDValue &ShReg,
201                                              SDValue &Opc) {
202  ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(N);
203
204  // Don't match base register only case. That is matched to a separate
205  // lower complexity pattern with explicit register operand.
206  if (ShOpcVal == ARM_AM::no_shift) return false;
207
208  BaseReg = N.getOperand(0);
209  unsigned ShImmVal = 0;
210  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
211    ShReg = CurDAG->getRegister(0, MVT::i32);
212    ShImmVal = RHS->getZExtValue() & 31;
213  } else {
214    ShReg = N.getOperand(1);
215  }
216  Opc = CurDAG->getTargetConstant(ARM_AM::getSORegOpc(ShOpcVal, ShImmVal),
217                                  MVT::i32);
218  return true;
219}
220
221bool ARMDAGToDAGISel::SelectAddrMode2(SDValue Op, SDValue N,
222                                      SDValue &Base, SDValue &Offset,
223                                      SDValue &Opc) {
224  if (N.getOpcode() == ISD::MUL) {
225    if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
226      // X * [3,5,9] -> X + X * [2,4,8] etc.
227      int RHSC = (int)RHS->getZExtValue();
228      if (RHSC & 1) {
229        RHSC = RHSC & ~1;
230        ARM_AM::AddrOpc AddSub = ARM_AM::add;
231        if (RHSC < 0) {
232          AddSub = ARM_AM::sub;
233          RHSC = - RHSC;
234        }
235        if (isPowerOf2_32(RHSC)) {
236          unsigned ShAmt = Log2_32(RHSC);
237          Base = Offset = N.getOperand(0);
238          Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, ShAmt,
239                                                            ARM_AM::lsl),
240                                          MVT::i32);
241          return true;
242        }
243      }
244    }
245  }
246
247  if (N.getOpcode() != ISD::ADD && N.getOpcode() != ISD::SUB) {
248    Base = N;
249    if (N.getOpcode() == ISD::FrameIndex) {
250      int FI = cast<FrameIndexSDNode>(N)->getIndex();
251      Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
252    } else if (N.getOpcode() == ARMISD::Wrapper) {
253      Base = N.getOperand(0);
254    }
255    Offset = CurDAG->getRegister(0, MVT::i32);
256    Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(ARM_AM::add, 0,
257                                                      ARM_AM::no_shift),
258                                    MVT::i32);
259    return true;
260  }
261
262  // Match simple R +/- imm12 operands.
263  if (N.getOpcode() == ISD::ADD)
264    if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
265      int RHSC = (int)RHS->getZExtValue();
266      if ((RHSC >= 0 && RHSC < 0x1000) ||
267          (RHSC < 0 && RHSC > -0x1000)) { // 12 bits.
268        Base = N.getOperand(0);
269        if (Base.getOpcode() == ISD::FrameIndex) {
270          int FI = cast<FrameIndexSDNode>(Base)->getIndex();
271          Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
272        }
273        Offset = CurDAG->getRegister(0, MVT::i32);
274
275        ARM_AM::AddrOpc AddSub = ARM_AM::add;
276        if (RHSC < 0) {
277          AddSub = ARM_AM::sub;
278          RHSC = - RHSC;
279        }
280        Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, RHSC,
281                                                          ARM_AM::no_shift),
282                                        MVT::i32);
283        return true;
284      }
285    }
286
287  // Otherwise this is R +/- [possibly shifted] R
288  ARM_AM::AddrOpc AddSub = N.getOpcode() == ISD::ADD ? ARM_AM::add:ARM_AM::sub;
289  ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(N.getOperand(1));
290  unsigned ShAmt = 0;
291
292  Base   = N.getOperand(0);
293  Offset = N.getOperand(1);
294
295  if (ShOpcVal != ARM_AM::no_shift) {
296    // Check to see if the RHS of the shift is a constant, if not, we can't fold
297    // it.
298    if (ConstantSDNode *Sh =
299           dyn_cast<ConstantSDNode>(N.getOperand(1).getOperand(1))) {
300      ShAmt = Sh->getZExtValue();
301      Offset = N.getOperand(1).getOperand(0);
302    } else {
303      ShOpcVal = ARM_AM::no_shift;
304    }
305  }
306
307  // Try matching (R shl C) + (R).
308  if (N.getOpcode() == ISD::ADD && ShOpcVal == ARM_AM::no_shift) {
309    ShOpcVal = ARM_AM::getShiftOpcForNode(N.getOperand(0));
310    if (ShOpcVal != ARM_AM::no_shift) {
311      // Check to see if the RHS of the shift is a constant, if not, we can't
312      // fold it.
313      if (ConstantSDNode *Sh =
314          dyn_cast<ConstantSDNode>(N.getOperand(0).getOperand(1))) {
315        ShAmt = Sh->getZExtValue();
316        Offset = N.getOperand(0).getOperand(0);
317        Base = N.getOperand(1);
318      } else {
319        ShOpcVal = ARM_AM::no_shift;
320      }
321    }
322  }
323
324  Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, ShAmt, ShOpcVal),
325                                  MVT::i32);
326  return true;
327}
328
329bool ARMDAGToDAGISel::SelectAddrMode2Offset(SDValue Op, SDValue N,
330                                            SDValue &Offset, SDValue &Opc) {
331  unsigned Opcode = Op.getOpcode();
332  ISD::MemIndexedMode AM = (Opcode == ISD::LOAD)
333    ? cast<LoadSDNode>(Op)->getAddressingMode()
334    : cast<StoreSDNode>(Op)->getAddressingMode();
335  ARM_AM::AddrOpc AddSub = (AM == ISD::PRE_INC || AM == ISD::POST_INC)
336    ? ARM_AM::add : ARM_AM::sub;
337  if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(N)) {
338    int Val = (int)C->getZExtValue();
339    if (Val >= 0 && Val < 0x1000) { // 12 bits.
340      Offset = CurDAG->getRegister(0, MVT::i32);
341      Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, Val,
342                                                        ARM_AM::no_shift),
343                                      MVT::i32);
344      return true;
345    }
346  }
347
348  Offset = N;
349  ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(N);
350  unsigned ShAmt = 0;
351  if (ShOpcVal != ARM_AM::no_shift) {
352    // Check to see if the RHS of the shift is a constant, if not, we can't fold
353    // it.
354    if (ConstantSDNode *Sh = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
355      ShAmt = Sh->getZExtValue();
356      Offset = N.getOperand(0);
357    } else {
358      ShOpcVal = ARM_AM::no_shift;
359    }
360  }
361
362  Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, ShAmt, ShOpcVal),
363                                  MVT::i32);
364  return true;
365}
366
367
368bool ARMDAGToDAGISel::SelectAddrMode3(SDValue Op, SDValue N,
369                                      SDValue &Base, SDValue &Offset,
370                                      SDValue &Opc) {
371  if (N.getOpcode() == ISD::SUB) {
372    // X - C  is canonicalize to X + -C, no need to handle it here.
373    Base = N.getOperand(0);
374    Offset = N.getOperand(1);
375    Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(ARM_AM::sub, 0),MVT::i32);
376    return true;
377  }
378
379  if (N.getOpcode() != ISD::ADD) {
380    Base = N;
381    if (N.getOpcode() == ISD::FrameIndex) {
382      int FI = cast<FrameIndexSDNode>(N)->getIndex();
383      Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
384    }
385    Offset = CurDAG->getRegister(0, MVT::i32);
386    Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(ARM_AM::add, 0),MVT::i32);
387    return true;
388  }
389
390  // If the RHS is +/- imm8, fold into addr mode.
391  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
392    int RHSC = (int)RHS->getZExtValue();
393    if ((RHSC >= 0 && RHSC < 256) ||
394        (RHSC < 0 && RHSC > -256)) { // note -256 itself isn't allowed.
395      Base = N.getOperand(0);
396      if (Base.getOpcode() == ISD::FrameIndex) {
397        int FI = cast<FrameIndexSDNode>(Base)->getIndex();
398        Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
399      }
400      Offset = CurDAG->getRegister(0, MVT::i32);
401
402      ARM_AM::AddrOpc AddSub = ARM_AM::add;
403      if (RHSC < 0) {
404        AddSub = ARM_AM::sub;
405        RHSC = - RHSC;
406      }
407      Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(AddSub, RHSC),MVT::i32);
408      return true;
409    }
410  }
411
412  Base = N.getOperand(0);
413  Offset = N.getOperand(1);
414  Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(ARM_AM::add, 0), MVT::i32);
415  return true;
416}
417
418bool ARMDAGToDAGISel::SelectAddrMode3Offset(SDValue Op, SDValue N,
419                                            SDValue &Offset, SDValue &Opc) {
420  unsigned Opcode = Op.getOpcode();
421  ISD::MemIndexedMode AM = (Opcode == ISD::LOAD)
422    ? cast<LoadSDNode>(Op)->getAddressingMode()
423    : cast<StoreSDNode>(Op)->getAddressingMode();
424  ARM_AM::AddrOpc AddSub = (AM == ISD::PRE_INC || AM == ISD::POST_INC)
425    ? ARM_AM::add : ARM_AM::sub;
426  if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(N)) {
427    int Val = (int)C->getZExtValue();
428    if (Val >= 0 && Val < 256) {
429      Offset = CurDAG->getRegister(0, MVT::i32);
430      Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(AddSub, Val), MVT::i32);
431      return true;
432    }
433  }
434
435  Offset = N;
436  Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(AddSub, 0), MVT::i32);
437  return true;
438}
439
440bool ARMDAGToDAGISel::SelectAddrMode4(SDValue Op, SDValue N,
441                                      SDValue &Addr, SDValue &Mode) {
442  Addr = N;
443  Mode = CurDAG->getTargetConstant(0, MVT::i32);
444  return true;
445}
446
447bool ARMDAGToDAGISel::SelectAddrMode5(SDValue Op, SDValue N,
448                                      SDValue &Base, SDValue &Offset) {
449  if (N.getOpcode() != ISD::ADD) {
450    Base = N;
451    if (N.getOpcode() == ISD::FrameIndex) {
452      int FI = cast<FrameIndexSDNode>(N)->getIndex();
453      Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
454    } else if (N.getOpcode() == ARMISD::Wrapper) {
455      Base = N.getOperand(0);
456    }
457    Offset = CurDAG->getTargetConstant(ARM_AM::getAM5Opc(ARM_AM::add, 0),
458                                       MVT::i32);
459    return true;
460  }
461
462  // If the RHS is +/- imm8, fold into addr mode.
463  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
464    int RHSC = (int)RHS->getZExtValue();
465    if ((RHSC & 3) == 0) {  // The constant is implicitly multiplied by 4.
466      RHSC >>= 2;
467      if ((RHSC >= 0 && RHSC < 256) ||
468          (RHSC < 0 && RHSC > -256)) { // note -256 itself isn't allowed.
469        Base = N.getOperand(0);
470        if (Base.getOpcode() == ISD::FrameIndex) {
471          int FI = cast<FrameIndexSDNode>(Base)->getIndex();
472          Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
473        }
474
475        ARM_AM::AddrOpc AddSub = ARM_AM::add;
476        if (RHSC < 0) {
477          AddSub = ARM_AM::sub;
478          RHSC = - RHSC;
479        }
480        Offset = CurDAG->getTargetConstant(ARM_AM::getAM5Opc(AddSub, RHSC),
481                                           MVT::i32);
482        return true;
483      }
484    }
485  }
486
487  Base = N;
488  Offset = CurDAG->getTargetConstant(ARM_AM::getAM5Opc(ARM_AM::add, 0),
489                                     MVT::i32);
490  return true;
491}
492
493bool ARMDAGToDAGISel::SelectAddrMode6(SDValue Op, SDValue N,
494                                      SDValue &Addr, SDValue &Update,
495                                      SDValue &Opc) {
496  Addr = N;
497  // Default to no writeback.
498  Update = CurDAG->getRegister(0, MVT::i32);
499  Opc = CurDAG->getTargetConstant(ARM_AM::getAM6Opc(false), MVT::i32);
500  return true;
501}
502
503bool ARMDAGToDAGISel::SelectAddrModePC(SDValue Op, SDValue N,
504                                       SDValue &Offset, SDValue &Label) {
505  if (N.getOpcode() == ARMISD::PIC_ADD && N.hasOneUse()) {
506    Offset = N.getOperand(0);
507    SDValue N1 = N.getOperand(1);
508    Label  = CurDAG->getTargetConstant(cast<ConstantSDNode>(N1)->getZExtValue(),
509                                       MVT::i32);
510    return true;
511  }
512  return false;
513}
514
515bool ARMDAGToDAGISel::SelectThumbAddrModeRR(SDValue Op, SDValue N,
516                                            SDValue &Base, SDValue &Offset){
517  // FIXME dl should come from the parent load or store, not the address
518  DebugLoc dl = Op.getDebugLoc();
519  if (N.getOpcode() != ISD::ADD) {
520    ConstantSDNode *NC = dyn_cast<ConstantSDNode>(N);
521    if (!NC || NC->getZExtValue() != 0)
522      return false;
523
524    Base = Offset = N;
525    return true;
526  }
527
528  Base = N.getOperand(0);
529  Offset = N.getOperand(1);
530  return true;
531}
532
533bool
534ARMDAGToDAGISel::SelectThumbAddrModeRI5(SDValue Op, SDValue N,
535                                        unsigned Scale, SDValue &Base,
536                                        SDValue &OffImm, SDValue &Offset) {
537  if (Scale == 4) {
538    SDValue TmpBase, TmpOffImm;
539    if (SelectThumbAddrModeSP(Op, N, TmpBase, TmpOffImm))
540      return false;  // We want to select tLDRspi / tSTRspi instead.
541    if (N.getOpcode() == ARMISD::Wrapper &&
542        N.getOperand(0).getOpcode() == ISD::TargetConstantPool)
543      return false;  // We want to select tLDRpci instead.
544  }
545
546  if (N.getOpcode() != ISD::ADD) {
547    Base = (N.getOpcode() == ARMISD::Wrapper) ? N.getOperand(0) : N;
548    Offset = CurDAG->getRegister(0, MVT::i32);
549    OffImm = CurDAG->getTargetConstant(0, MVT::i32);
550    return true;
551  }
552
553  // Thumb does not have [sp, r] address mode.
554  RegisterSDNode *LHSR = dyn_cast<RegisterSDNode>(N.getOperand(0));
555  RegisterSDNode *RHSR = dyn_cast<RegisterSDNode>(N.getOperand(1));
556  if ((LHSR && LHSR->getReg() == ARM::SP) ||
557      (RHSR && RHSR->getReg() == ARM::SP)) {
558    Base = N;
559    Offset = CurDAG->getRegister(0, MVT::i32);
560    OffImm = CurDAG->getTargetConstant(0, MVT::i32);
561    return true;
562  }
563
564  // If the RHS is + imm5 * scale, fold into addr mode.
565  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
566    int RHSC = (int)RHS->getZExtValue();
567    if ((RHSC & (Scale-1)) == 0) {  // The constant is implicitly multiplied.
568      RHSC /= Scale;
569      if (RHSC >= 0 && RHSC < 32) {
570        Base = N.getOperand(0);
571        Offset = CurDAG->getRegister(0, MVT::i32);
572        OffImm = CurDAG->getTargetConstant(RHSC, MVT::i32);
573        return true;
574      }
575    }
576  }
577
578  Base = N.getOperand(0);
579  Offset = N.getOperand(1);
580  OffImm = CurDAG->getTargetConstant(0, MVT::i32);
581  return true;
582}
583
584bool ARMDAGToDAGISel::SelectThumbAddrModeS1(SDValue Op, SDValue N,
585                                            SDValue &Base, SDValue &OffImm,
586                                            SDValue &Offset) {
587  return SelectThumbAddrModeRI5(Op, N, 1, Base, OffImm, Offset);
588}
589
590bool ARMDAGToDAGISel::SelectThumbAddrModeS2(SDValue Op, SDValue N,
591                                            SDValue &Base, SDValue &OffImm,
592                                            SDValue &Offset) {
593  return SelectThumbAddrModeRI5(Op, N, 2, Base, OffImm, Offset);
594}
595
596bool ARMDAGToDAGISel::SelectThumbAddrModeS4(SDValue Op, SDValue N,
597                                            SDValue &Base, SDValue &OffImm,
598                                            SDValue &Offset) {
599  return SelectThumbAddrModeRI5(Op, N, 4, Base, OffImm, Offset);
600}
601
602bool ARMDAGToDAGISel::SelectThumbAddrModeSP(SDValue Op, SDValue N,
603                                           SDValue &Base, SDValue &OffImm) {
604  if (N.getOpcode() == ISD::FrameIndex) {
605    int FI = cast<FrameIndexSDNode>(N)->getIndex();
606    Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
607    OffImm = CurDAG->getTargetConstant(0, MVT::i32);
608    return true;
609  }
610
611  if (N.getOpcode() != ISD::ADD)
612    return false;
613
614  RegisterSDNode *LHSR = dyn_cast<RegisterSDNode>(N.getOperand(0));
615  if (N.getOperand(0).getOpcode() == ISD::FrameIndex ||
616      (LHSR && LHSR->getReg() == ARM::SP)) {
617    // If the RHS is + imm8 * scale, fold into addr mode.
618    if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
619      int RHSC = (int)RHS->getZExtValue();
620      if ((RHSC & 3) == 0) {  // The constant is implicitly multiplied.
621        RHSC >>= 2;
622        if (RHSC >= 0 && RHSC < 256) {
623          Base = N.getOperand(0);
624          if (Base.getOpcode() == ISD::FrameIndex) {
625            int FI = cast<FrameIndexSDNode>(Base)->getIndex();
626            Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
627          }
628          OffImm = CurDAG->getTargetConstant(RHSC, MVT::i32);
629          return true;
630        }
631      }
632    }
633  }
634
635  return false;
636}
637
638bool ARMDAGToDAGISel::SelectT2ShifterOperandReg(SDValue Op, SDValue N,
639                                                SDValue &BaseReg,
640                                                SDValue &Opc) {
641  ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(N);
642
643  // Don't match base register only case. That is matched to a separate
644  // lower complexity pattern with explicit register operand.
645  if (ShOpcVal == ARM_AM::no_shift) return false;
646
647  BaseReg = N.getOperand(0);
648  unsigned ShImmVal = 0;
649  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
650    ShImmVal = RHS->getZExtValue() & 31;
651    Opc = getI32Imm(ARM_AM::getSORegOpc(ShOpcVal, ShImmVal));
652    return true;
653  }
654
655  return false;
656}
657
658bool ARMDAGToDAGISel::SelectT2AddrModeImm12(SDValue Op, SDValue N,
659                                            SDValue &Base, SDValue &OffImm) {
660  // Match simple R + imm12 operands.
661
662  // Base only.
663  if (N.getOpcode() != ISD::ADD && N.getOpcode() != ISD::SUB) {
664    if (N.getOpcode() == ISD::FrameIndex) {
665      // Match frame index...
666      int FI = cast<FrameIndexSDNode>(N)->getIndex();
667      Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
668      OffImm  = CurDAG->getTargetConstant(0, MVT::i32);
669      return true;
670    } else if (N.getOpcode() == ARMISD::Wrapper) {
671      Base = N.getOperand(0);
672      if (Base.getOpcode() == ISD::TargetConstantPool)
673        return false;  // We want to select t2LDRpci instead.
674    } else
675      Base = N;
676    OffImm  = CurDAG->getTargetConstant(0, MVT::i32);
677    return true;
678  }
679
680  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
681    if (SelectT2AddrModeImm8(Op, N, Base, OffImm))
682      // Let t2LDRi8 handle (R - imm8).
683      return false;
684
685    int RHSC = (int)RHS->getZExtValue();
686    if (N.getOpcode() == ISD::SUB)
687      RHSC = -RHSC;
688
689    if (RHSC >= 0 && RHSC < 0x1000) { // 12 bits (unsigned)
690      Base   = N.getOperand(0);
691      if (Base.getOpcode() == ISD::FrameIndex) {
692        int FI = cast<FrameIndexSDNode>(Base)->getIndex();
693        Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
694      }
695      OffImm = CurDAG->getTargetConstant(RHSC, MVT::i32);
696      return true;
697    }
698  }
699
700  // Base only.
701  Base = N;
702  OffImm  = CurDAG->getTargetConstant(0, MVT::i32);
703  return true;
704}
705
706bool ARMDAGToDAGISel::SelectT2AddrModeImm8(SDValue Op, SDValue N,
707                                           SDValue &Base, SDValue &OffImm) {
708  // Match simple R - imm8 operands.
709  if (N.getOpcode() == ISD::ADD || N.getOpcode() == ISD::SUB) {
710    if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
711      int RHSC = (int)RHS->getSExtValue();
712      if (N.getOpcode() == ISD::SUB)
713        RHSC = -RHSC;
714
715      if ((RHSC >= -255) && (RHSC < 0)) { // 8 bits (always negative)
716        Base = N.getOperand(0);
717        if (Base.getOpcode() == ISD::FrameIndex) {
718          int FI = cast<FrameIndexSDNode>(Base)->getIndex();
719          Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
720        }
721        OffImm = CurDAG->getTargetConstant(RHSC, MVT::i32);
722        return true;
723      }
724    }
725  }
726
727  return false;
728}
729
730bool ARMDAGToDAGISel::SelectT2AddrModeImm8Offset(SDValue Op, SDValue N,
731                                                 SDValue &OffImm){
732  unsigned Opcode = Op.getOpcode();
733  ISD::MemIndexedMode AM = (Opcode == ISD::LOAD)
734    ? cast<LoadSDNode>(Op)->getAddressingMode()
735    : cast<StoreSDNode>(Op)->getAddressingMode();
736  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N)) {
737    int RHSC = (int)RHS->getZExtValue();
738    if (RHSC >= 0 && RHSC < 0x100) { // 8 bits.
739      OffImm = ((AM == ISD::PRE_INC) || (AM == ISD::POST_INC))
740        ? CurDAG->getTargetConstant(RHSC, MVT::i32)
741        : CurDAG->getTargetConstant(-RHSC, MVT::i32);
742      return true;
743    }
744  }
745
746  return false;
747}
748
749bool ARMDAGToDAGISel::SelectT2AddrModeImm8s4(SDValue Op, SDValue N,
750                                             SDValue &Base, SDValue &OffImm) {
751  if (N.getOpcode() == ISD::ADD) {
752    if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
753      int RHSC = (int)RHS->getZExtValue();
754      if (((RHSC & 0x3) == 0) &&
755          ((RHSC >= 0 && RHSC < 0x400) || (RHSC < 0 && RHSC > -0x400))) { // 8 bits.
756        Base   = N.getOperand(0);
757        OffImm = CurDAG->getTargetConstant(RHSC, MVT::i32);
758        return true;
759      }
760    }
761  } else if (N.getOpcode() == ISD::SUB) {
762    if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
763      int RHSC = (int)RHS->getZExtValue();
764      if (((RHSC & 0x3) == 0) && (RHSC >= 0 && RHSC < 0x400)) { // 8 bits.
765        Base   = N.getOperand(0);
766        OffImm = CurDAG->getTargetConstant(-RHSC, MVT::i32);
767        return true;
768      }
769    }
770  }
771
772  return false;
773}
774
775bool ARMDAGToDAGISel::SelectT2AddrModeSoReg(SDValue Op, SDValue N,
776                                            SDValue &Base,
777                                            SDValue &OffReg, SDValue &ShImm) {
778  // (R - imm8) should be handled by t2LDRi8. The rest are handled by t2LDRi12.
779  if (N.getOpcode() != ISD::ADD)
780    return false;
781
782  // Leave (R + imm12) for t2LDRi12, (R - imm8) for t2LDRi8.
783  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
784    int RHSC = (int)RHS->getZExtValue();
785    if (RHSC >= 0 && RHSC < 0x1000) // 12 bits (unsigned)
786      return false;
787    else if (RHSC < 0 && RHSC >= -255) // 8 bits
788      return false;
789  }
790
791  // Look for (R + R) or (R + (R << [1,2,3])).
792  unsigned ShAmt = 0;
793  Base   = N.getOperand(0);
794  OffReg = N.getOperand(1);
795
796  // Swap if it is ((R << c) + R).
797  ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(OffReg);
798  if (ShOpcVal != ARM_AM::lsl) {
799    ShOpcVal = ARM_AM::getShiftOpcForNode(Base);
800    if (ShOpcVal == ARM_AM::lsl)
801      std::swap(Base, OffReg);
802  }
803
804  if (ShOpcVal == ARM_AM::lsl) {
805    // Check to see if the RHS of the shift is a constant, if not, we can't fold
806    // it.
807    if (ConstantSDNode *Sh = dyn_cast<ConstantSDNode>(OffReg.getOperand(1))) {
808      ShAmt = Sh->getZExtValue();
809      if (ShAmt >= 4) {
810        ShAmt = 0;
811        ShOpcVal = ARM_AM::no_shift;
812      } else
813        OffReg = OffReg.getOperand(0);
814    } else {
815      ShOpcVal = ARM_AM::no_shift;
816    }
817  }
818
819  ShImm = CurDAG->getTargetConstant(ShAmt, MVT::i32);
820
821  return true;
822}
823
824//===--------------------------------------------------------------------===//
825
826/// getAL - Returns a ARMCC::AL immediate node.
827static inline SDValue getAL(SelectionDAG *CurDAG) {
828  return CurDAG->getTargetConstant((uint64_t)ARMCC::AL, MVT::i32);
829}
830
831SDNode *ARMDAGToDAGISel::SelectARMIndexedLoad(SDValue Op) {
832  LoadSDNode *LD = cast<LoadSDNode>(Op);
833  ISD::MemIndexedMode AM = LD->getAddressingMode();
834  if (AM == ISD::UNINDEXED)
835    return NULL;
836
837  EVT LoadedVT = LD->getMemoryVT();
838  SDValue Offset, AMOpc;
839  bool isPre = (AM == ISD::PRE_INC) || (AM == ISD::PRE_DEC);
840  unsigned Opcode = 0;
841  bool Match = false;
842  if (LoadedVT == MVT::i32 &&
843      SelectAddrMode2Offset(Op, LD->getOffset(), Offset, AMOpc)) {
844    Opcode = isPre ? ARM::LDR_PRE : ARM::LDR_POST;
845    Match = true;
846  } else if (LoadedVT == MVT::i16 &&
847             SelectAddrMode3Offset(Op, LD->getOffset(), Offset, AMOpc)) {
848    Match = true;
849    Opcode = (LD->getExtensionType() == ISD::SEXTLOAD)
850      ? (isPre ? ARM::LDRSH_PRE : ARM::LDRSH_POST)
851      : (isPre ? ARM::LDRH_PRE : ARM::LDRH_POST);
852  } else if (LoadedVT == MVT::i8 || LoadedVT == MVT::i1) {
853    if (LD->getExtensionType() == ISD::SEXTLOAD) {
854      if (SelectAddrMode3Offset(Op, LD->getOffset(), Offset, AMOpc)) {
855        Match = true;
856        Opcode = isPre ? ARM::LDRSB_PRE : ARM::LDRSB_POST;
857      }
858    } else {
859      if (SelectAddrMode2Offset(Op, LD->getOffset(), Offset, AMOpc)) {
860        Match = true;
861        Opcode = isPre ? ARM::LDRB_PRE : ARM::LDRB_POST;
862      }
863    }
864  }
865
866  if (Match) {
867    SDValue Chain = LD->getChain();
868    SDValue Base = LD->getBasePtr();
869    SDValue Ops[]= { Base, Offset, AMOpc, getAL(CurDAG),
870                     CurDAG->getRegister(0, MVT::i32), Chain };
871    return CurDAG->getMachineNode(Opcode, Op.getDebugLoc(), MVT::i32, MVT::i32,
872                                  MVT::Other, Ops, 6);
873  }
874
875  return NULL;
876}
877
878SDNode *ARMDAGToDAGISel::SelectT2IndexedLoad(SDValue Op) {
879  LoadSDNode *LD = cast<LoadSDNode>(Op);
880  ISD::MemIndexedMode AM = LD->getAddressingMode();
881  if (AM == ISD::UNINDEXED)
882    return NULL;
883
884  EVT LoadedVT = LD->getMemoryVT();
885  bool isSExtLd = LD->getExtensionType() == ISD::SEXTLOAD;
886  SDValue Offset;
887  bool isPre = (AM == ISD::PRE_INC) || (AM == ISD::PRE_DEC);
888  unsigned Opcode = 0;
889  bool Match = false;
890  if (SelectT2AddrModeImm8Offset(Op, LD->getOffset(), Offset)) {
891    switch (LoadedVT.getSimpleVT().SimpleTy) {
892    case MVT::i32:
893      Opcode = isPre ? ARM::t2LDR_PRE : ARM::t2LDR_POST;
894      break;
895    case MVT::i16:
896      if (isSExtLd)
897        Opcode = isPre ? ARM::t2LDRSH_PRE : ARM::t2LDRSH_POST;
898      else
899        Opcode = isPre ? ARM::t2LDRH_PRE : ARM::t2LDRH_POST;
900      break;
901    case MVT::i8:
902    case MVT::i1:
903      if (isSExtLd)
904        Opcode = isPre ? ARM::t2LDRSB_PRE : ARM::t2LDRSB_POST;
905      else
906        Opcode = isPre ? ARM::t2LDRB_PRE : ARM::t2LDRB_POST;
907      break;
908    default:
909      return NULL;
910    }
911    Match = true;
912  }
913
914  if (Match) {
915    SDValue Chain = LD->getChain();
916    SDValue Base = LD->getBasePtr();
917    SDValue Ops[]= { Base, Offset, getAL(CurDAG),
918                     CurDAG->getRegister(0, MVT::i32), Chain };
919    return CurDAG->getMachineNode(Opcode, Op.getDebugLoc(), MVT::i32, MVT::i32,
920                                  MVT::Other, Ops, 5);
921  }
922
923  return NULL;
924}
925
926SDNode *ARMDAGToDAGISel::SelectDYN_ALLOC(SDValue Op) {
927  SDNode *N = Op.getNode();
928  DebugLoc dl = N->getDebugLoc();
929  EVT VT = Op.getValueType();
930  SDValue Chain = Op.getOperand(0);
931  SDValue Size = Op.getOperand(1);
932  SDValue Align = Op.getOperand(2);
933  SDValue SP = CurDAG->getRegister(ARM::SP, MVT::i32);
934  int32_t AlignVal = cast<ConstantSDNode>(Align)->getSExtValue();
935  if (AlignVal < 0)
936    // We need to align the stack. Use Thumb1 tAND which is the only thumb
937    // instruction that can read and write SP. This matches to a pseudo
938    // instruction that has a chain to ensure the result is written back to
939    // the stack pointer.
940    SP = SDValue(CurDAG->getMachineNode(ARM::tANDsp, dl, VT, SP, Align), 0);
941
942  bool isC = isa<ConstantSDNode>(Size);
943  uint32_t C = isC ? cast<ConstantSDNode>(Size)->getZExtValue() : ~0UL;
944  // Handle the most common case for both Thumb1 and Thumb2:
945  // tSUBspi - immediate is between 0 ... 508 inclusive.
946  if (C <= 508 && ((C & 3) == 0))
947    // FIXME: tSUBspi encode scale 4 implicitly.
948    return CurDAG->SelectNodeTo(N, ARM::tSUBspi_, VT, MVT::Other, SP,
949                                CurDAG->getTargetConstant(C/4, MVT::i32),
950                                Chain);
951
952  if (Subtarget->isThumb1Only()) {
953    // Use tADDspr since Thumb1 does not have a sub r, sp, r. ARMISelLowering
954    // should have negated the size operand already. FIXME: We can't insert
955    // new target independent node at this stage so we are forced to negate
956    // it earlier. Is there a better solution?
957    return CurDAG->SelectNodeTo(N, ARM::tADDspr_, VT, MVT::Other, SP, Size,
958                                Chain);
959  } else if (Subtarget->isThumb2()) {
960    if (isC && Predicate_t2_so_imm(Size.getNode())) {
961      // t2SUBrSPi
962      SDValue Ops[] = { SP, CurDAG->getTargetConstant(C, MVT::i32), Chain };
963      return CurDAG->SelectNodeTo(N, ARM::t2SUBrSPi_, VT, MVT::Other, Ops, 3);
964    } else if (isC && Predicate_imm0_4095(Size.getNode())) {
965      // t2SUBrSPi12
966      SDValue Ops[] = { SP, CurDAG->getTargetConstant(C, MVT::i32), Chain };
967      return CurDAG->SelectNodeTo(N, ARM::t2SUBrSPi12_, VT, MVT::Other, Ops, 3);
968    } else {
969      // t2SUBrSPs
970      SDValue Ops[] = { SP, Size,
971                        getI32Imm(ARM_AM::getSORegOpc(ARM_AM::lsl,0)), Chain };
972      return CurDAG->SelectNodeTo(N, ARM::t2SUBrSPs_, VT, MVT::Other, Ops, 4);
973    }
974  }
975
976  // FIXME: Add ADD / SUB sp instructions for ARM.
977  return 0;
978}
979
980/// PairDRegs - Insert a pair of double registers into an implicit def to
981/// form a quad register.
982SDNode *ARMDAGToDAGISel::PairDRegs(EVT VT, SDValue V0, SDValue V1) {
983  DebugLoc dl = V0.getNode()->getDebugLoc();
984  SDValue Undef =
985    SDValue(CurDAG->getMachineNode(TargetInstrInfo::IMPLICIT_DEF, dl, VT), 0);
986  SDValue SubReg0 = CurDAG->getTargetConstant(ARM::DSUBREG_0, MVT::i32);
987  SDValue SubReg1 = CurDAG->getTargetConstant(ARM::DSUBREG_1, MVT::i32);
988  SDNode *Pair = CurDAG->getMachineNode(TargetInstrInfo::INSERT_SUBREG, dl,
989                                        VT, Undef, V0, SubReg0);
990  return CurDAG->getMachineNode(TargetInstrInfo::INSERT_SUBREG, dl,
991                                VT, SDValue(Pair, 0), V1, SubReg1);
992}
993
994/// GetNEONSubregVT - Given a type for a 128-bit NEON vector, return the type
995/// for a 64-bit subregister of the vector.
996static EVT GetNEONSubregVT(EVT VT) {
997  switch (VT.getSimpleVT().SimpleTy) {
998  default: llvm_unreachable("unhandled NEON type");
999  case MVT::v16i8: return MVT::v8i8;
1000  case MVT::v8i16: return MVT::v4i16;
1001  case MVT::v4f32: return MVT::v2f32;
1002  case MVT::v4i32: return MVT::v2i32;
1003  case MVT::v2i64: return MVT::v1i64;
1004  }
1005}
1006
1007SDNode *ARMDAGToDAGISel::SelectVLD(SDValue Op, unsigned NumVecs,
1008                                   unsigned *DOpcodes, unsigned *QOpcodes0,
1009                                   unsigned *QOpcodes1) {
1010  assert(NumVecs >=2 && NumVecs <= 4 && "VLD NumVecs out-of-range");
1011  SDNode *N = Op.getNode();
1012  DebugLoc dl = N->getDebugLoc();
1013
1014  SDValue MemAddr, MemUpdate, MemOpc;
1015  if (!SelectAddrMode6(Op, N->getOperand(2), MemAddr, MemUpdate, MemOpc))
1016    return NULL;
1017
1018  SDValue Chain = N->getOperand(0);
1019  EVT VT = N->getValueType(0);
1020  bool is64BitVector = VT.is64BitVector();
1021
1022  unsigned OpcodeIndex;
1023  switch (VT.getSimpleVT().SimpleTy) {
1024  default: llvm_unreachable("unhandled vld type");
1025    // Double-register operations:
1026  case MVT::v8i8:  OpcodeIndex = 0; break;
1027  case MVT::v4i16: OpcodeIndex = 1; break;
1028  case MVT::v2f32:
1029  case MVT::v2i32: OpcodeIndex = 2; break;
1030  case MVT::v1i64: OpcodeIndex = 3; break;
1031    // Quad-register operations:
1032  case MVT::v16i8: OpcodeIndex = 0; break;
1033  case MVT::v8i16: OpcodeIndex = 1; break;
1034  case MVT::v4f32:
1035  case MVT::v4i32: OpcodeIndex = 2; break;
1036  }
1037
1038  if (is64BitVector) {
1039    unsigned Opc = DOpcodes[OpcodeIndex];
1040    const SDValue Ops[] = { MemAddr, MemUpdate, MemOpc, Chain };
1041    std::vector<EVT> ResTys(NumVecs, VT);
1042    ResTys.push_back(MVT::Other);
1043    return CurDAG->getMachineNode(Opc, dl, ResTys, Ops, 4);
1044  }
1045
1046  EVT RegVT = GetNEONSubregVT(VT);
1047  if (NumVecs == 2) {
1048    // Quad registers are directly supported for VLD2,
1049    // loading 2 pairs of D regs.
1050    unsigned Opc = QOpcodes0[OpcodeIndex];
1051    const SDValue Ops[] = { MemAddr, MemUpdate, MemOpc, Chain };
1052    std::vector<EVT> ResTys(4, VT);
1053    ResTys.push_back(MVT::Other);
1054    SDNode *VLd = CurDAG->getMachineNode(Opc, dl, ResTys, Ops, 4);
1055    Chain = SDValue(VLd, 4);
1056
1057    // Combine the even and odd subregs to produce the result.
1058    for (unsigned Vec = 0; Vec < NumVecs; ++Vec) {
1059      SDNode *Q = PairDRegs(VT, SDValue(VLd, 2*Vec), SDValue(VLd, 2*Vec+1));
1060      ReplaceUses(SDValue(N, Vec), SDValue(Q, 0));
1061    }
1062  } else {
1063    // Otherwise, quad registers are loaded with two separate instructions,
1064    // where one loads the even registers and the other loads the odd registers.
1065
1066    // Enable writeback to the address register.
1067    MemOpc = CurDAG->getTargetConstant(ARM_AM::getAM6Opc(true), MVT::i32);
1068
1069    std::vector<EVT> ResTys(NumVecs, RegVT);
1070    ResTys.push_back(MemAddr.getValueType());
1071    ResTys.push_back(MVT::Other);
1072
1073    // Load the even subregs.
1074    unsigned Opc = QOpcodes0[OpcodeIndex];
1075    const SDValue OpsA[] = { MemAddr, MemUpdate, MemOpc, Chain };
1076    SDNode *VLdA = CurDAG->getMachineNode(Opc, dl, ResTys, OpsA, 4);
1077    Chain = SDValue(VLdA, NumVecs+1);
1078
1079    // Load the odd subregs.
1080    Opc = QOpcodes1[OpcodeIndex];
1081    const SDValue OpsB[] = { SDValue(VLdA, NumVecs), MemUpdate, MemOpc, Chain };
1082    SDNode *VLdB = CurDAG->getMachineNode(Opc, dl, ResTys, OpsB, 4);
1083    Chain = SDValue(VLdB, NumVecs+1);
1084
1085    // Combine the even and odd subregs to produce the result.
1086    for (unsigned Vec = 0; Vec < NumVecs; ++Vec) {
1087      SDNode *Q = PairDRegs(VT, SDValue(VLdA, Vec), SDValue(VLdB, Vec));
1088      ReplaceUses(SDValue(N, Vec), SDValue(Q, 0));
1089    }
1090  }
1091  ReplaceUses(SDValue(N, NumVecs), Chain);
1092  return NULL;
1093}
1094
1095SDNode *ARMDAGToDAGISel::SelectVST(SDValue Op, unsigned NumVecs,
1096                                   unsigned *DOpcodes, unsigned *QOpcodes0,
1097                                   unsigned *QOpcodes1) {
1098  assert(NumVecs >=2 && NumVecs <= 4 && "VST NumVecs out-of-range");
1099  SDNode *N = Op.getNode();
1100  DebugLoc dl = N->getDebugLoc();
1101
1102  SDValue MemAddr, MemUpdate, MemOpc;
1103  if (!SelectAddrMode6(Op, N->getOperand(2), MemAddr, MemUpdate, MemOpc))
1104    return NULL;
1105
1106  SDValue Chain = N->getOperand(0);
1107  EVT VT = N->getOperand(3).getValueType();
1108  bool is64BitVector = VT.is64BitVector();
1109
1110  unsigned OpcodeIndex;
1111  switch (VT.getSimpleVT().SimpleTy) {
1112  default: llvm_unreachable("unhandled vst type");
1113    // Double-register operations:
1114  case MVT::v8i8:  OpcodeIndex = 0; break;
1115  case MVT::v4i16: OpcodeIndex = 1; break;
1116  case MVT::v2f32:
1117  case MVT::v2i32: OpcodeIndex = 2; break;
1118  case MVT::v1i64: OpcodeIndex = 3; break;
1119    // Quad-register operations:
1120  case MVT::v16i8: OpcodeIndex = 0; break;
1121  case MVT::v8i16: OpcodeIndex = 1; break;
1122  case MVT::v4f32:
1123  case MVT::v4i32: OpcodeIndex = 2; break;
1124  }
1125
1126  SmallVector<SDValue, 8> Ops;
1127  Ops.push_back(MemAddr);
1128  Ops.push_back(MemUpdate);
1129  Ops.push_back(MemOpc);
1130
1131  if (is64BitVector) {
1132    unsigned Opc = DOpcodes[OpcodeIndex];
1133    for (unsigned Vec = 0; Vec < NumVecs; ++Vec)
1134      Ops.push_back(N->getOperand(Vec+3));
1135    Ops.push_back(Chain);
1136    return CurDAG->getMachineNode(Opc, dl, MVT::Other, Ops.data(), NumVecs+4);
1137  }
1138
1139  EVT RegVT = GetNEONSubregVT(VT);
1140  if (NumVecs == 2) {
1141    // Quad registers are directly supported for VST2,
1142    // storing 2 pairs of D regs.
1143    unsigned Opc = QOpcodes0[OpcodeIndex];
1144    for (unsigned Vec = 0; Vec < NumVecs; ++Vec) {
1145      Ops.push_back(CurDAG->getTargetExtractSubreg(ARM::DSUBREG_0, dl, RegVT,
1146                                                   N->getOperand(Vec+3)));
1147      Ops.push_back(CurDAG->getTargetExtractSubreg(ARM::DSUBREG_1, dl, RegVT,
1148                                                   N->getOperand(Vec+3)));
1149    }
1150    Ops.push_back(Chain);
1151    return CurDAG->getMachineNode(Opc, dl, MVT::Other, Ops.data(), 8);
1152  }
1153
1154  // Otherwise, quad registers are stored with two separate instructions,
1155  // where one stores the even registers and the other stores the odd registers.
1156
1157  // Enable writeback to the address register.
1158  MemOpc = CurDAG->getTargetConstant(ARM_AM::getAM6Opc(true), MVT::i32);
1159
1160  // Store the even subregs.
1161  for (unsigned Vec = 0; Vec < NumVecs; ++Vec)
1162    Ops.push_back(CurDAG->getTargetExtractSubreg(ARM::DSUBREG_0, dl, RegVT,
1163                                                 N->getOperand(Vec+3)));
1164  Ops.push_back(Chain);
1165  unsigned Opc = QOpcodes0[OpcodeIndex];
1166  SDNode *VStA = CurDAG->getMachineNode(Opc, dl, MemAddr.getValueType(),
1167                                        MVT::Other, Ops.data(), NumVecs+4);
1168  Chain = SDValue(VStA, 1);
1169
1170  // Store the odd subregs.
1171  Ops[0] = SDValue(VStA, 0); // MemAddr
1172  for (unsigned Vec = 0; Vec < NumVecs; ++Vec)
1173    Ops[Vec+3] = CurDAG->getTargetExtractSubreg(ARM::DSUBREG_1, dl, RegVT,
1174                                                N->getOperand(Vec+3));
1175  Ops[NumVecs+3] = Chain;
1176  Opc = QOpcodes1[OpcodeIndex];
1177  SDNode *VStB = CurDAG->getMachineNode(Opc, dl, MemAddr.getValueType(),
1178                                        MVT::Other, Ops.data(), NumVecs+4);
1179  Chain = SDValue(VStB, 1);
1180  ReplaceUses(SDValue(N, 0), Chain);
1181  return NULL;
1182}
1183
1184SDNode *ARMDAGToDAGISel::SelectVLDSTLane(SDValue Op, bool IsLoad,
1185                                         unsigned NumVecs, unsigned *DOpcodes,
1186                                         unsigned *QOpcodes0,
1187                                         unsigned *QOpcodes1) {
1188  assert(NumVecs >=2 && NumVecs <= 4 && "VLDSTLane NumVecs out-of-range");
1189  SDNode *N = Op.getNode();
1190  DebugLoc dl = N->getDebugLoc();
1191
1192  SDValue MemAddr, MemUpdate, MemOpc;
1193  if (!SelectAddrMode6(Op, N->getOperand(2), MemAddr, MemUpdate, MemOpc))
1194    return NULL;
1195
1196  SDValue Chain = N->getOperand(0);
1197  unsigned Lane =
1198    cast<ConstantSDNode>(N->getOperand(NumVecs+3))->getZExtValue();
1199  EVT VT = IsLoad ? N->getValueType(0) : N->getOperand(3).getValueType();
1200  bool is64BitVector = VT.is64BitVector();
1201
1202  // Quad registers are handled by load/store of subregs. Find the subreg info.
1203  unsigned NumElts = 0;
1204  int SubregIdx = 0;
1205  EVT RegVT = VT;
1206  if (!is64BitVector) {
1207    RegVT = GetNEONSubregVT(VT);
1208    NumElts = RegVT.getVectorNumElements();
1209    SubregIdx = (Lane < NumElts) ? ARM::DSUBREG_0 : ARM::DSUBREG_1;
1210  }
1211
1212  unsigned OpcodeIndex;
1213  switch (VT.getSimpleVT().SimpleTy) {
1214  default: llvm_unreachable("unhandled vld/vst lane type");
1215    // Double-register operations:
1216  case MVT::v8i8:  OpcodeIndex = 0; break;
1217  case MVT::v4i16: OpcodeIndex = 1; break;
1218  case MVT::v2f32:
1219  case MVT::v2i32: OpcodeIndex = 2; break;
1220    // Quad-register operations:
1221  case MVT::v8i16: OpcodeIndex = 0; break;
1222  case MVT::v4f32:
1223  case MVT::v4i32: OpcodeIndex = 1; break;
1224  }
1225
1226  SmallVector<SDValue, 9> Ops;
1227  Ops.push_back(MemAddr);
1228  Ops.push_back(MemUpdate);
1229  Ops.push_back(MemOpc);
1230
1231  unsigned Opc = 0;
1232  if (is64BitVector) {
1233    Opc = DOpcodes[OpcodeIndex];
1234    for (unsigned Vec = 0; Vec < NumVecs; ++Vec)
1235      Ops.push_back(N->getOperand(Vec+3));
1236  } else {
1237    // Check if this is loading the even or odd subreg of a Q register.
1238    if (Lane < NumElts) {
1239      Opc = QOpcodes0[OpcodeIndex];
1240    } else {
1241      Lane -= NumElts;
1242      Opc = QOpcodes1[OpcodeIndex];
1243    }
1244    // Extract the subregs of the input vector.
1245    for (unsigned Vec = 0; Vec < NumVecs; ++Vec)
1246      Ops.push_back(CurDAG->getTargetExtractSubreg(SubregIdx, dl, RegVT,
1247                                                   N->getOperand(Vec+3)));
1248  }
1249  Ops.push_back(getI32Imm(Lane));
1250  Ops.push_back(Chain);
1251
1252  if (!IsLoad)
1253    return CurDAG->getMachineNode(Opc, dl, MVT::Other, Ops.data(), NumVecs+5);
1254
1255  std::vector<EVT> ResTys(NumVecs, RegVT);
1256  ResTys.push_back(MVT::Other);
1257  SDNode *VLdLn =
1258    CurDAG->getMachineNode(Opc, dl, ResTys, Ops.data(), NumVecs+5);
1259  // For a 64-bit vector load to D registers, nothing more needs to be done.
1260  if (is64BitVector)
1261    return VLdLn;
1262
1263  // For 128-bit vectors, take the 64-bit results of the load and insert them
1264  // as subregs into the result.
1265  for (unsigned Vec = 0; Vec < NumVecs; ++Vec) {
1266    SDValue QuadVec = CurDAG->getTargetInsertSubreg(SubregIdx, dl, VT,
1267                                                    N->getOperand(Vec+3),
1268                                                    SDValue(VLdLn, Vec));
1269    ReplaceUses(SDValue(N, Vec), QuadVec);
1270  }
1271
1272  Chain = SDValue(VLdLn, NumVecs);
1273  ReplaceUses(SDValue(N, NumVecs), Chain);
1274  return NULL;
1275}
1276
1277SDNode *ARMDAGToDAGISel::SelectV6T2BitfieldExtractOp(SDValue Op,
1278                                                     unsigned Opc) {
1279  if (!Subtarget->hasV6T2Ops())
1280    return NULL;
1281
1282  unsigned Shl_imm = 0;
1283  if (isOpcWithIntImmediate(Op.getOperand(0).getNode(), ISD::SHL, Shl_imm)){
1284    assert(Shl_imm > 0 && Shl_imm < 32 && "bad amount in shift node!");
1285    unsigned Srl_imm = 0;
1286    if (isInt32Immediate(Op.getOperand(1), Srl_imm)) {
1287      assert(Srl_imm > 0 && Srl_imm < 32 && "bad amount in shift node!");
1288      unsigned Width = 32 - Srl_imm;
1289      int LSB = Srl_imm - Shl_imm;
1290      if ((LSB + Width) > 32)
1291        return NULL;
1292      SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
1293      SDValue Ops[] = { Op.getOperand(0).getOperand(0),
1294                        CurDAG->getTargetConstant(LSB, MVT::i32),
1295                        CurDAG->getTargetConstant(Width, MVT::i32),
1296                        getAL(CurDAG), Reg0 };
1297      return CurDAG->SelectNodeTo(Op.getNode(), Opc, MVT::i32, Ops, 5);
1298    }
1299  }
1300  return NULL;
1301}
1302
1303SDNode *ARMDAGToDAGISel::Select(SDValue Op) {
1304  SDNode *N = Op.getNode();
1305  DebugLoc dl = N->getDebugLoc();
1306
1307  if (N->isMachineOpcode())
1308    return NULL;   // Already selected.
1309
1310  switch (N->getOpcode()) {
1311  default: break;
1312  case ISD::Constant: {
1313    unsigned Val = cast<ConstantSDNode>(N)->getZExtValue();
1314    bool UseCP = true;
1315    if (Subtarget->hasThumb2())
1316      // Thumb2-aware targets have the MOVT instruction, so all immediates can
1317      // be done with MOV + MOVT, at worst.
1318      UseCP = 0;
1319    else {
1320      if (Subtarget->isThumb()) {
1321        UseCP = (Val > 255 &&                          // MOV
1322                 ~Val > 255 &&                         // MOV + MVN
1323                 !ARM_AM::isThumbImmShiftedVal(Val));  // MOV + LSL
1324      } else
1325        UseCP = (ARM_AM::getSOImmVal(Val) == -1 &&     // MOV
1326                 ARM_AM::getSOImmVal(~Val) == -1 &&    // MVN
1327                 !ARM_AM::isSOImmTwoPartVal(Val));     // two instrs.
1328    }
1329
1330    if (UseCP) {
1331      SDValue CPIdx =
1332        CurDAG->getTargetConstantPool(ConstantInt::get(
1333                                  Type::getInt32Ty(*CurDAG->getContext()), Val),
1334                                      TLI.getPointerTy());
1335
1336      SDNode *ResNode;
1337      if (Subtarget->isThumb1Only()) {
1338        SDValue Pred = CurDAG->getTargetConstant(0xEULL, MVT::i32);
1339        SDValue PredReg = CurDAG->getRegister(0, MVT::i32);
1340        SDValue Ops[] = { CPIdx, Pred, PredReg, CurDAG->getEntryNode() };
1341        ResNode = CurDAG->getMachineNode(ARM::tLDRcp, dl, MVT::i32, MVT::Other,
1342                                         Ops, 4);
1343      } else {
1344        SDValue Ops[] = {
1345          CPIdx,
1346          CurDAG->getRegister(0, MVT::i32),
1347          CurDAG->getTargetConstant(0, MVT::i32),
1348          getAL(CurDAG),
1349          CurDAG->getRegister(0, MVT::i32),
1350          CurDAG->getEntryNode()
1351        };
1352        ResNode=CurDAG->getMachineNode(ARM::LDRcp, dl, MVT::i32, MVT::Other,
1353                                       Ops, 6);
1354      }
1355      ReplaceUses(Op, SDValue(ResNode, 0));
1356      return NULL;
1357    }
1358
1359    // Other cases are autogenerated.
1360    break;
1361  }
1362  case ISD::FrameIndex: {
1363    // Selects to ADDri FI, 0 which in turn will become ADDri SP, imm.
1364    int FI = cast<FrameIndexSDNode>(N)->getIndex();
1365    SDValue TFI = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
1366    if (Subtarget->isThumb1Only()) {
1367      return CurDAG->SelectNodeTo(N, ARM::tADDrSPi, MVT::i32, TFI,
1368                                  CurDAG->getTargetConstant(0, MVT::i32));
1369    } else {
1370      unsigned Opc = ((Subtarget->isThumb() && Subtarget->hasThumb2()) ?
1371                      ARM::t2ADDri : ARM::ADDri);
1372      SDValue Ops[] = { TFI, CurDAG->getTargetConstant(0, MVT::i32),
1373                        getAL(CurDAG), CurDAG->getRegister(0, MVT::i32),
1374                        CurDAG->getRegister(0, MVT::i32) };
1375      return CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops, 5);
1376    }
1377  }
1378  case ARMISD::DYN_ALLOC:
1379    return SelectDYN_ALLOC(Op);
1380  case ISD::SRL:
1381    if (SDNode *I = SelectV6T2BitfieldExtractOp(Op,
1382                      Subtarget->isThumb() ? ARM::t2UBFX : ARM::UBFX))
1383      return I;
1384    break;
1385  case ISD::SRA:
1386    if (SDNode *I = SelectV6T2BitfieldExtractOp(Op,
1387                      Subtarget->isThumb() ? ARM::t2SBFX : ARM::SBFX))
1388      return I;
1389    break;
1390  case ISD::MUL:
1391    if (Subtarget->isThumb1Only())
1392      break;
1393    if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
1394      unsigned RHSV = C->getZExtValue();
1395      if (!RHSV) break;
1396      if (isPowerOf2_32(RHSV-1)) {  // 2^n+1?
1397        unsigned ShImm = Log2_32(RHSV-1);
1398        if (ShImm >= 32)
1399          break;
1400        SDValue V = Op.getOperand(0);
1401        ShImm = ARM_AM::getSORegOpc(ARM_AM::lsl, ShImm);
1402        SDValue ShImmOp = CurDAG->getTargetConstant(ShImm, MVT::i32);
1403        SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
1404        if (Subtarget->isThumb()) {
1405          SDValue Ops[] = { V, V, ShImmOp, getAL(CurDAG), Reg0, Reg0 };
1406          return CurDAG->SelectNodeTo(N, ARM::t2ADDrs, MVT::i32, Ops, 6);
1407        } else {
1408          SDValue Ops[] = { V, V, Reg0, ShImmOp, getAL(CurDAG), Reg0, Reg0 };
1409          return CurDAG->SelectNodeTo(N, ARM::ADDrs, MVT::i32, Ops, 7);
1410        }
1411      }
1412      if (isPowerOf2_32(RHSV+1)) {  // 2^n-1?
1413        unsigned ShImm = Log2_32(RHSV+1);
1414        if (ShImm >= 32)
1415          break;
1416        SDValue V = Op.getOperand(0);
1417        ShImm = ARM_AM::getSORegOpc(ARM_AM::lsl, ShImm);
1418        SDValue ShImmOp = CurDAG->getTargetConstant(ShImm, MVT::i32);
1419        SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
1420        if (Subtarget->isThumb()) {
1421          SDValue Ops[] = { V, V, ShImmOp, getAL(CurDAG), Reg0 };
1422          return CurDAG->SelectNodeTo(N, ARM::t2RSBrs, MVT::i32, Ops, 5);
1423        } else {
1424          SDValue Ops[] = { V, V, Reg0, ShImmOp, getAL(CurDAG), Reg0, Reg0 };
1425          return CurDAG->SelectNodeTo(N, ARM::RSBrs, MVT::i32, Ops, 7);
1426        }
1427      }
1428    }
1429    break;
1430  case ARMISD::FMRRD:
1431    return CurDAG->getMachineNode(ARM::FMRRD, dl, MVT::i32, MVT::i32,
1432                                  Op.getOperand(0), getAL(CurDAG),
1433                                  CurDAG->getRegister(0, MVT::i32));
1434  case ISD::UMUL_LOHI: {
1435    if (Subtarget->isThumb1Only())
1436      break;
1437    if (Subtarget->isThumb()) {
1438      SDValue Ops[] = { Op.getOperand(0), Op.getOperand(1),
1439                        getAL(CurDAG), CurDAG->getRegister(0, MVT::i32),
1440                        CurDAG->getRegister(0, MVT::i32) };
1441      return CurDAG->getMachineNode(ARM::t2UMULL, dl, MVT::i32, MVT::i32, Ops,4);
1442    } else {
1443      SDValue Ops[] = { Op.getOperand(0), Op.getOperand(1),
1444                        getAL(CurDAG), CurDAG->getRegister(0, MVT::i32),
1445                        CurDAG->getRegister(0, MVT::i32) };
1446      return CurDAG->getMachineNode(ARM::UMULL, dl, MVT::i32, MVT::i32, Ops, 5);
1447    }
1448  }
1449  case ISD::SMUL_LOHI: {
1450    if (Subtarget->isThumb1Only())
1451      break;
1452    if (Subtarget->isThumb()) {
1453      SDValue Ops[] = { Op.getOperand(0), Op.getOperand(1),
1454                        getAL(CurDAG), CurDAG->getRegister(0, MVT::i32) };
1455      return CurDAG->getMachineNode(ARM::t2SMULL, dl, MVT::i32, MVT::i32, Ops,4);
1456    } else {
1457      SDValue Ops[] = { Op.getOperand(0), Op.getOperand(1),
1458                        getAL(CurDAG), CurDAG->getRegister(0, MVT::i32),
1459                        CurDAG->getRegister(0, MVT::i32) };
1460      return CurDAG->getMachineNode(ARM::SMULL, dl, MVT::i32, MVT::i32, Ops, 5);
1461    }
1462  }
1463  case ISD::LOAD: {
1464    SDNode *ResNode = 0;
1465    if (Subtarget->isThumb() && Subtarget->hasThumb2())
1466      ResNode = SelectT2IndexedLoad(Op);
1467    else
1468      ResNode = SelectARMIndexedLoad(Op);
1469    if (ResNode)
1470      return ResNode;
1471    // Other cases are autogenerated.
1472    break;
1473  }
1474  case ARMISD::BRCOND: {
1475    // Pattern: (ARMbrcond:void (bb:Other):$dst, (imm:i32):$cc)
1476    // Emits: (Bcc:void (bb:Other):$dst, (imm:i32):$cc)
1477    // Pattern complexity = 6  cost = 1  size = 0
1478
1479    // Pattern: (ARMbrcond:void (bb:Other):$dst, (imm:i32):$cc)
1480    // Emits: (tBcc:void (bb:Other):$dst, (imm:i32):$cc)
1481    // Pattern complexity = 6  cost = 1  size = 0
1482
1483    // Pattern: (ARMbrcond:void (bb:Other):$dst, (imm:i32):$cc)
1484    // Emits: (t2Bcc:void (bb:Other):$dst, (imm:i32):$cc)
1485    // Pattern complexity = 6  cost = 1  size = 0
1486
1487    unsigned Opc = Subtarget->isThumb() ?
1488      ((Subtarget->hasThumb2()) ? ARM::t2Bcc : ARM::tBcc) : ARM::Bcc;
1489    SDValue Chain = Op.getOperand(0);
1490    SDValue N1 = Op.getOperand(1);
1491    SDValue N2 = Op.getOperand(2);
1492    SDValue N3 = Op.getOperand(3);
1493    SDValue InFlag = Op.getOperand(4);
1494    assert(N1.getOpcode() == ISD::BasicBlock);
1495    assert(N2.getOpcode() == ISD::Constant);
1496    assert(N3.getOpcode() == ISD::Register);
1497
1498    SDValue Tmp2 = CurDAG->getTargetConstant(((unsigned)
1499                               cast<ConstantSDNode>(N2)->getZExtValue()),
1500                               MVT::i32);
1501    SDValue Ops[] = { N1, Tmp2, N3, Chain, InFlag };
1502    SDNode *ResNode = CurDAG->getMachineNode(Opc, dl, MVT::Other,
1503                                             MVT::Flag, Ops, 5);
1504    Chain = SDValue(ResNode, 0);
1505    if (Op.getNode()->getNumValues() == 2) {
1506      InFlag = SDValue(ResNode, 1);
1507      ReplaceUses(SDValue(Op.getNode(), 1), InFlag);
1508    }
1509    ReplaceUses(SDValue(Op.getNode(), 0), SDValue(Chain.getNode(), Chain.getResNo()));
1510    return NULL;
1511  }
1512  case ARMISD::CMOV: {
1513    EVT VT = Op.getValueType();
1514    SDValue N0 = Op.getOperand(0);
1515    SDValue N1 = Op.getOperand(1);
1516    SDValue N2 = Op.getOperand(2);
1517    SDValue N3 = Op.getOperand(3);
1518    SDValue InFlag = Op.getOperand(4);
1519    assert(N2.getOpcode() == ISD::Constant);
1520    assert(N3.getOpcode() == ISD::Register);
1521
1522    if (!Subtarget->isThumb1Only() && VT == MVT::i32) {
1523      // Pattern: (ARMcmov:i32 GPR:i32:$false, so_reg:i32:$true, (imm:i32):$cc)
1524      // Emits: (MOVCCs:i32 GPR:i32:$false, so_reg:i32:$true, (imm:i32):$cc)
1525      // Pattern complexity = 18  cost = 1  size = 0
1526      SDValue CPTmp0;
1527      SDValue CPTmp1;
1528      SDValue CPTmp2;
1529      if (Subtarget->isThumb()) {
1530        if (SelectT2ShifterOperandReg(Op, N1, CPTmp0, CPTmp1)) {
1531          unsigned SOVal = cast<ConstantSDNode>(CPTmp1)->getZExtValue();
1532          unsigned SOShOp = ARM_AM::getSORegShOp(SOVal);
1533          unsigned Opc = 0;
1534          switch (SOShOp) {
1535          case ARM_AM::lsl: Opc = ARM::t2MOVCClsl; break;
1536          case ARM_AM::lsr: Opc = ARM::t2MOVCClsr; break;
1537          case ARM_AM::asr: Opc = ARM::t2MOVCCasr; break;
1538          case ARM_AM::ror: Opc = ARM::t2MOVCCror; break;
1539          default:
1540            llvm_unreachable("Unknown so_reg opcode!");
1541            break;
1542          }
1543          SDValue SOShImm =
1544            CurDAG->getTargetConstant(ARM_AM::getSORegOffset(SOVal), MVT::i32);
1545          SDValue Tmp2 = CurDAG->getTargetConstant(((unsigned)
1546                                   cast<ConstantSDNode>(N2)->getZExtValue()),
1547                                   MVT::i32);
1548          SDValue Ops[] = { N0, CPTmp0, SOShImm, Tmp2, N3, InFlag };
1549          return CurDAG->SelectNodeTo(Op.getNode(), Opc, MVT::i32,Ops, 6);
1550        }
1551      } else {
1552        if (SelectShifterOperandReg(Op, N1, CPTmp0, CPTmp1, CPTmp2)) {
1553          SDValue Tmp2 = CurDAG->getTargetConstant(((unsigned)
1554                                   cast<ConstantSDNode>(N2)->getZExtValue()),
1555                                   MVT::i32);
1556          SDValue Ops[] = { N0, CPTmp0, CPTmp1, CPTmp2, Tmp2, N3, InFlag };
1557          return CurDAG->SelectNodeTo(Op.getNode(),
1558                                      ARM::MOVCCs, MVT::i32, Ops, 7);
1559        }
1560      }
1561
1562      // Pattern: (ARMcmov:i32 GPR:i32:$false,
1563      //             (imm:i32)<<P:Predicate_so_imm>>:$true,
1564      //             (imm:i32):$cc)
1565      // Emits: (MOVCCi:i32 GPR:i32:$false,
1566      //           (so_imm:i32 (imm:i32):$true), (imm:i32):$cc)
1567      // Pattern complexity = 10  cost = 1  size = 0
1568      if (N3.getOpcode() == ISD::Constant) {
1569        if (Subtarget->isThumb()) {
1570          if (Predicate_t2_so_imm(N3.getNode())) {
1571            SDValue Tmp1 = CurDAG->getTargetConstant(((unsigned)
1572                                     cast<ConstantSDNode>(N1)->getZExtValue()),
1573                                     MVT::i32);
1574            SDValue Tmp2 = CurDAG->getTargetConstant(((unsigned)
1575                                     cast<ConstantSDNode>(N2)->getZExtValue()),
1576                                     MVT::i32);
1577            SDValue Ops[] = { N0, Tmp1, Tmp2, N3, InFlag };
1578            return CurDAG->SelectNodeTo(Op.getNode(),
1579                                        ARM::t2MOVCCi, MVT::i32, Ops, 5);
1580          }
1581        } else {
1582          if (Predicate_so_imm(N3.getNode())) {
1583            SDValue Tmp1 = CurDAG->getTargetConstant(((unsigned)
1584                                     cast<ConstantSDNode>(N1)->getZExtValue()),
1585                                     MVT::i32);
1586            SDValue Tmp2 = CurDAG->getTargetConstant(((unsigned)
1587                                     cast<ConstantSDNode>(N2)->getZExtValue()),
1588                                     MVT::i32);
1589            SDValue Ops[] = { N0, Tmp1, Tmp2, N3, InFlag };
1590            return CurDAG->SelectNodeTo(Op.getNode(),
1591                                        ARM::MOVCCi, MVT::i32, Ops, 5);
1592          }
1593        }
1594      }
1595    }
1596
1597    // Pattern: (ARMcmov:i32 GPR:i32:$false, GPR:i32:$true, (imm:i32):$cc)
1598    // Emits: (MOVCCr:i32 GPR:i32:$false, GPR:i32:$true, (imm:i32):$cc)
1599    // Pattern complexity = 6  cost = 1  size = 0
1600    //
1601    // Pattern: (ARMcmov:i32 GPR:i32:$false, GPR:i32:$true, (imm:i32):$cc)
1602    // Emits: (tMOVCCr:i32 GPR:i32:$false, GPR:i32:$true, (imm:i32):$cc)
1603    // Pattern complexity = 6  cost = 11  size = 0
1604    //
1605    // Also FCPYScc and FCPYDcc.
1606    SDValue Tmp2 = CurDAG->getTargetConstant(((unsigned)
1607                               cast<ConstantSDNode>(N2)->getZExtValue()),
1608                               MVT::i32);
1609    SDValue Ops[] = { N0, N1, Tmp2, N3, InFlag };
1610    unsigned Opc = 0;
1611    switch (VT.getSimpleVT().SimpleTy) {
1612    default: assert(false && "Illegal conditional move type!");
1613      break;
1614    case MVT::i32:
1615      Opc = Subtarget->isThumb()
1616        ? (Subtarget->hasThumb2() ? ARM::t2MOVCCr : ARM::tMOVCCr_pseudo)
1617        : ARM::MOVCCr;
1618      break;
1619    case MVT::f32:
1620      Opc = ARM::FCPYScc;
1621      break;
1622    case MVT::f64:
1623      Opc = ARM::FCPYDcc;
1624      break;
1625    }
1626    return CurDAG->SelectNodeTo(Op.getNode(), Opc, VT, Ops, 5);
1627  }
1628  case ARMISD::CNEG: {
1629    EVT VT = Op.getValueType();
1630    SDValue N0 = Op.getOperand(0);
1631    SDValue N1 = Op.getOperand(1);
1632    SDValue N2 = Op.getOperand(2);
1633    SDValue N3 = Op.getOperand(3);
1634    SDValue InFlag = Op.getOperand(4);
1635    assert(N2.getOpcode() == ISD::Constant);
1636    assert(N3.getOpcode() == ISD::Register);
1637
1638    SDValue Tmp2 = CurDAG->getTargetConstant(((unsigned)
1639                               cast<ConstantSDNode>(N2)->getZExtValue()),
1640                               MVT::i32);
1641    SDValue Ops[] = { N0, N1, Tmp2, N3, InFlag };
1642    unsigned Opc = 0;
1643    switch (VT.getSimpleVT().SimpleTy) {
1644    default: assert(false && "Illegal conditional move type!");
1645      break;
1646    case MVT::f32:
1647      Opc = ARM::FNEGScc;
1648      break;
1649    case MVT::f64:
1650      Opc = ARM::FNEGDcc;
1651      break;
1652    }
1653    return CurDAG->SelectNodeTo(Op.getNode(), Opc, VT, Ops, 5);
1654  }
1655
1656  case ARMISD::VZIP: {
1657    unsigned Opc = 0;
1658    EVT VT = N->getValueType(0);
1659    switch (VT.getSimpleVT().SimpleTy) {
1660    default: return NULL;
1661    case MVT::v8i8:  Opc = ARM::VZIPd8; break;
1662    case MVT::v4i16: Opc = ARM::VZIPd16; break;
1663    case MVT::v2f32:
1664    case MVT::v2i32: Opc = ARM::VZIPd32; break;
1665    case MVT::v16i8: Opc = ARM::VZIPq8; break;
1666    case MVT::v8i16: Opc = ARM::VZIPq16; break;
1667    case MVT::v4f32:
1668    case MVT::v4i32: Opc = ARM::VZIPq32; break;
1669    }
1670    return CurDAG->getMachineNode(Opc, dl, VT, VT,
1671                                  N->getOperand(0), N->getOperand(1));
1672  }
1673  case ARMISD::VUZP: {
1674    unsigned Opc = 0;
1675    EVT VT = N->getValueType(0);
1676    switch (VT.getSimpleVT().SimpleTy) {
1677    default: return NULL;
1678    case MVT::v8i8:  Opc = ARM::VUZPd8; break;
1679    case MVT::v4i16: Opc = ARM::VUZPd16; break;
1680    case MVT::v2f32:
1681    case MVT::v2i32: Opc = ARM::VUZPd32; break;
1682    case MVT::v16i8: Opc = ARM::VUZPq8; break;
1683    case MVT::v8i16: Opc = ARM::VUZPq16; break;
1684    case MVT::v4f32:
1685    case MVT::v4i32: Opc = ARM::VUZPq32; break;
1686    }
1687    return CurDAG->getMachineNode(Opc, dl, VT, VT,
1688                                  N->getOperand(0), N->getOperand(1));
1689  }
1690  case ARMISD::VTRN: {
1691    unsigned Opc = 0;
1692    EVT VT = N->getValueType(0);
1693    switch (VT.getSimpleVT().SimpleTy) {
1694    default: return NULL;
1695    case MVT::v8i8:  Opc = ARM::VTRNd8; break;
1696    case MVT::v4i16: Opc = ARM::VTRNd16; break;
1697    case MVT::v2f32:
1698    case MVT::v2i32: Opc = ARM::VTRNd32; break;
1699    case MVT::v16i8: Opc = ARM::VTRNq8; break;
1700    case MVT::v8i16: Opc = ARM::VTRNq16; break;
1701    case MVT::v4f32:
1702    case MVT::v4i32: Opc = ARM::VTRNq32; break;
1703    }
1704    return CurDAG->getMachineNode(Opc, dl, VT, VT,
1705                                  N->getOperand(0), N->getOperand(1));
1706  }
1707
1708  case ISD::INTRINSIC_VOID:
1709  case ISD::INTRINSIC_W_CHAIN: {
1710    unsigned IntNo = cast<ConstantSDNode>(N->getOperand(1))->getZExtValue();
1711    switch (IntNo) {
1712    default:
1713      break;
1714
1715    case Intrinsic::arm_neon_vld2: {
1716      unsigned DOpcodes[] = { ARM::VLD2d8, ARM::VLD2d16,
1717                              ARM::VLD2d32, ARM::VLD2d64 };
1718      unsigned QOpcodes[] = { ARM::VLD2q8, ARM::VLD2q16, ARM::VLD2q32 };
1719      return SelectVLD(Op, 2, DOpcodes, QOpcodes, 0);
1720    }
1721
1722    case Intrinsic::arm_neon_vld3: {
1723      unsigned DOpcodes[] = { ARM::VLD3d8, ARM::VLD3d16,
1724                              ARM::VLD3d32, ARM::VLD3d64 };
1725      unsigned QOpcodes0[] = { ARM::VLD3q8a, ARM::VLD3q16a, ARM::VLD3q32a };
1726      unsigned QOpcodes1[] = { ARM::VLD3q8b, ARM::VLD3q16b, ARM::VLD3q32b };
1727      return SelectVLD(Op, 3, DOpcodes, QOpcodes0, QOpcodes1);
1728    }
1729
1730    case Intrinsic::arm_neon_vld4: {
1731      unsigned DOpcodes[] = { ARM::VLD4d8, ARM::VLD4d16,
1732                              ARM::VLD4d32, ARM::VLD4d64 };
1733      unsigned QOpcodes0[] = { ARM::VLD4q8a, ARM::VLD4q16a, ARM::VLD4q32a };
1734      unsigned QOpcodes1[] = { ARM::VLD4q8b, ARM::VLD4q16b, ARM::VLD4q32b };
1735      return SelectVLD(Op, 4, DOpcodes, QOpcodes0, QOpcodes1);
1736    }
1737
1738    case Intrinsic::arm_neon_vld2lane: {
1739      unsigned DOpcodes[] = { ARM::VLD2LNd8, ARM::VLD2LNd16, ARM::VLD2LNd32 };
1740      unsigned QOpcodes0[] = { ARM::VLD2LNq16a, ARM::VLD2LNq32a };
1741      unsigned QOpcodes1[] = { ARM::VLD2LNq16b, ARM::VLD2LNq32b };
1742      return SelectVLDSTLane(Op, true, 2, DOpcodes, QOpcodes0, QOpcodes1);
1743    }
1744
1745    case Intrinsic::arm_neon_vld3lane: {
1746      unsigned DOpcodes[] = { ARM::VLD3LNd8, ARM::VLD3LNd16, ARM::VLD3LNd32 };
1747      unsigned QOpcodes0[] = { ARM::VLD3LNq16a, ARM::VLD3LNq32a };
1748      unsigned QOpcodes1[] = { ARM::VLD3LNq16b, ARM::VLD3LNq32b };
1749      return SelectVLDSTLane(Op, true, 3, DOpcodes, QOpcodes0, QOpcodes1);
1750    }
1751
1752    case Intrinsic::arm_neon_vld4lane: {
1753      unsigned DOpcodes[] = { ARM::VLD4LNd8, ARM::VLD4LNd16, ARM::VLD4LNd32 };
1754      unsigned QOpcodes0[] = { ARM::VLD4LNq16a, ARM::VLD4LNq32a };
1755      unsigned QOpcodes1[] = { ARM::VLD4LNq16b, ARM::VLD4LNq32b };
1756      return SelectVLDSTLane(Op, true, 4, DOpcodes, QOpcodes0, QOpcodes1);
1757    }
1758
1759    case Intrinsic::arm_neon_vst2: {
1760      unsigned DOpcodes[] = { ARM::VST2d8, ARM::VST2d16,
1761                              ARM::VST2d32, ARM::VST2d64 };
1762      unsigned QOpcodes[] = { ARM::VST2q8, ARM::VST2q16, ARM::VST2q32 };
1763      return SelectVST(Op, 2, DOpcodes, QOpcodes, 0);
1764    }
1765
1766    case Intrinsic::arm_neon_vst3: {
1767      unsigned DOpcodes[] = { ARM::VST3d8, ARM::VST3d16,
1768                              ARM::VST3d32, ARM::VST3d64 };
1769      unsigned QOpcodes0[] = { ARM::VST3q8a, ARM::VST3q16a, ARM::VST3q32a };
1770      unsigned QOpcodes1[] = { ARM::VST3q8b, ARM::VST3q16b, ARM::VST3q32b };
1771      return SelectVST(Op, 3, DOpcodes, QOpcodes0, QOpcodes1);
1772    }
1773
1774    case Intrinsic::arm_neon_vst4: {
1775      unsigned DOpcodes[] = { ARM::VST4d8, ARM::VST4d16,
1776                              ARM::VST4d32, ARM::VST4d64 };
1777      unsigned QOpcodes0[] = { ARM::VST4q8a, ARM::VST4q16a, ARM::VST4q32a };
1778      unsigned QOpcodes1[] = { ARM::VST4q8b, ARM::VST4q16b, ARM::VST4q32b };
1779      return SelectVST(Op, 4, DOpcodes, QOpcodes0, QOpcodes1);
1780    }
1781
1782    case Intrinsic::arm_neon_vst2lane: {
1783      unsigned DOpcodes[] = { ARM::VST2LNd8, ARM::VST2LNd16, ARM::VST2LNd32 };
1784      unsigned QOpcodes0[] = { ARM::VST2LNq16a, ARM::VST2LNq32a };
1785      unsigned QOpcodes1[] = { ARM::VST2LNq16b, ARM::VST2LNq32b };
1786      return SelectVLDSTLane(Op, false, 2, DOpcodes, QOpcodes0, QOpcodes1);
1787    }
1788
1789    case Intrinsic::arm_neon_vst3lane: {
1790      unsigned DOpcodes[] = { ARM::VST3LNd8, ARM::VST3LNd16, ARM::VST3LNd32 };
1791      unsigned QOpcodes0[] = { ARM::VST3LNq16a, ARM::VST3LNq32a };
1792      unsigned QOpcodes1[] = { ARM::VST3LNq16b, ARM::VST3LNq32b };
1793      return SelectVLDSTLane(Op, false, 3, DOpcodes, QOpcodes0, QOpcodes1);
1794    }
1795
1796    case Intrinsic::arm_neon_vst4lane: {
1797      unsigned DOpcodes[] = { ARM::VST4LNd8, ARM::VST4LNd16, ARM::VST4LNd32 };
1798      unsigned QOpcodes0[] = { ARM::VST4LNq16a, ARM::VST4LNq32a };
1799      unsigned QOpcodes1[] = { ARM::VST4LNq16b, ARM::VST4LNq32b };
1800      return SelectVLDSTLane(Op, false, 4, DOpcodes, QOpcodes0, QOpcodes1);
1801    }
1802    }
1803  }
1804  }
1805
1806  return SelectCode(Op);
1807}
1808
1809bool ARMDAGToDAGISel::
1810SelectInlineAsmMemoryOperand(const SDValue &Op, char ConstraintCode,
1811                             std::vector<SDValue> &OutOps) {
1812  assert(ConstraintCode == 'm' && "unexpected asm memory constraint");
1813  // Require the address to be in a register.  That is safe for all ARM
1814  // variants and it is hard to do anything much smarter without knowing
1815  // how the operand is used.
1816  OutOps.push_back(Op);
1817  return false;
1818}
1819
1820/// createARMISelDag - This pass converts a legalized DAG into a
1821/// ARM-specific DAG, ready for instruction scheduling.
1822///
1823FunctionPass *llvm::createARMISelDag(ARMBaseTargetMachine &TM,
1824                                     CodeGenOpt::Level OptLevel) {
1825  return new ARMDAGToDAGISel(TM, OptLevel);
1826}
1827