ARMISelDAGToDAG.cpp revision 5de5d4b6d0eb3fd379fa571d82f6fa764460b3b8 
1//===-- ARMISelDAGToDAG.cpp - A dag to dag inst selector for ARM ----------===//
2//
3//                     The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10// This file defines an instruction selector for the ARM target.
11//
12//===----------------------------------------------------------------------===//
13
14#define DEBUG_TYPE "arm-isel"
15#include "ARM.h"
16#include "ARMBaseInstrInfo.h"
17#include "ARMAddressingModes.h"
18#include "ARMTargetMachine.h"
19#include "llvm/CallingConv.h"
20#include "llvm/Constants.h"
21#include "llvm/DerivedTypes.h"
22#include "llvm/Function.h"
23#include "llvm/Intrinsics.h"
24#include "llvm/LLVMContext.h"
25#include "llvm/CodeGen/MachineFrameInfo.h"
26#include "llvm/CodeGen/MachineFunction.h"
27#include "llvm/CodeGen/MachineInstrBuilder.h"
28#include "llvm/CodeGen/SelectionDAG.h"
29#include "llvm/CodeGen/SelectionDAGISel.h"
30#include "llvm/Target/TargetLowering.h"
31#include "llvm/Target/TargetOptions.h"
32#include "llvm/Support/CommandLine.h"
33#include "llvm/Support/Compiler.h"
34#include "llvm/Support/Debug.h"
35#include "llvm/Support/ErrorHandling.h"
36#include "llvm/Support/raw_ostream.h"
37
38using namespace llvm;
39
40static cl::opt<bool>
41DisableShifterOp("disable-shifter-op", cl::Hidden,
42  cl::desc("Disable isel of shifter-op"),
43  cl::init(false));
44
45static cl::opt<bool>
46CheckVMLxHazard("check-vmlx-hazard", cl::Hidden,
47  cl::desc("Check fp vmla / vmls hazard at isel time"),
48  cl::init(false));
49
50//===--------------------------------------------------------------------===//
51/// ARMDAGToDAGISel - ARM specific code to select ARM machine
52/// instructions for SelectionDAG operations.
53///
54namespace {
55
56enum AddrMode2Type {
57  AM2_BASE, // Simple AM2 (+-imm12)
58  AM2_SHOP  // Shifter-op AM2
59};
60
61class ARMDAGToDAGISel : public SelectionDAGISel {
62  ARMBaseTargetMachine &TM;
63  const ARMBaseInstrInfo *TII;
64
65  /// Subtarget - Keep a pointer to the ARMSubtarget around so that we can
66  /// make the right decision when generating code for different targets.
67  const ARMSubtarget *Subtarget;
68
69public:
70  explicit ARMDAGToDAGISel(ARMBaseTargetMachine &tm,
71                           CodeGenOpt::Level OptLevel)
72    : SelectionDAGISel(tm, OptLevel), TM(tm),
73      TII(static_cast<const ARMBaseInstrInfo*>(TM.getInstrInfo())),
74      Subtarget(&TM.getSubtarget<ARMSubtarget>()) {
75  }
76
77  virtual const char *getPassName() const {
78    return "ARM Instruction Selection";
79  }
80
81  /// getI32Imm - Return a target constant of type i32 with the specified
82  /// value.
83  inline SDValue getI32Imm(unsigned Imm) {
84    return CurDAG->getTargetConstant(Imm, MVT::i32);
85  }
86
87  SDNode *Select(SDNode *N);
88
89
90  bool hasNoVMLxHazardUse(SDNode *N) const;
91  bool isShifterOpProfitable(const SDValue &Shift,
92                             ARM_AM::ShiftOpc ShOpcVal, unsigned ShAmt);
93  bool SelectShifterOperandReg(SDValue N, SDValue &A,
94                               SDValue &B, SDValue &C);
95  bool SelectShiftShifterOperandReg(SDValue N, SDValue &A,
96                                    SDValue &B, SDValue &C);
97  bool SelectAddrModeImm12(SDValue N, SDValue &Base, SDValue &OffImm);
98  bool SelectLdStSOReg(SDValue N, SDValue &Base, SDValue &Offset, SDValue &Opc);
99
100  AddrMode2Type SelectAddrMode2Worker(SDValue N, SDValue &Base,
101                                      SDValue &Offset, SDValue &Opc);
102  bool SelectAddrMode2Base(SDValue N, SDValue &Base, SDValue &Offset,
103                           SDValue &Opc) {
104    return SelectAddrMode2Worker(N, Base, Offset, Opc) == AM2_BASE;
105  }
106
107  bool SelectAddrMode2ShOp(SDValue N, SDValue &Base, SDValue &Offset,
108                           SDValue &Opc) {
109    return SelectAddrMode2Worker(N, Base, Offset, Opc) == AM2_SHOP;
110  }
111
112  bool SelectAddrMode2(SDValue N, SDValue &Base, SDValue &Offset,
113                       SDValue &Opc) {
114    SelectAddrMode2Worker(N, Base, Offset, Opc);
115//    return SelectAddrMode2ShOp(N, Base, Offset, Opc);
116    // This always matches one way or another.
117    return true;
118  }
119
120  bool SelectAddrMode2Offset(SDNode *Op, SDValue N,
121                             SDValue &Offset, SDValue &Opc);
122  bool SelectAddrMode3(SDValue N, SDValue &Base,
123                       SDValue &Offset, SDValue &Opc);
124  bool SelectAddrMode3Offset(SDNode *Op, SDValue N,
125                             SDValue &Offset, SDValue &Opc);
126  bool SelectAddrMode5(SDValue N, SDValue &Base,
127                       SDValue &Offset);
128  bool SelectAddrMode6(SDNode *Parent, SDValue N, SDValue &Addr,SDValue &Align);
129
130  bool SelectAddrModePC(SDValue N, SDValue &Offset, SDValue &Label);
131
132  // Thumb Addressing Modes:
133  bool SelectThumbAddrModeRR(SDValue N, SDValue &Base, SDValue &Offset);
134  bool SelectThumbAddrModeRI(SDValue N, SDValue &Base, SDValue &Offset,
135                             unsigned Scale);
136  bool SelectThumbAddrModeRI5S1(SDValue N, SDValue &Base, SDValue &Offset);
137  bool SelectThumbAddrModeRI5S2(SDValue N, SDValue &Base, SDValue &Offset);
138  bool SelectThumbAddrModeRI5S4(SDValue N, SDValue &Base, SDValue &Offset);
139  bool SelectThumbAddrModeImm5S(SDValue N, unsigned Scale, SDValue &Base,
140                                SDValue &OffImm);
141  bool SelectThumbAddrModeImm5S1(SDValue N, SDValue &Base,
142                                 SDValue &OffImm);
143  bool SelectThumbAddrModeImm5S2(SDValue N, SDValue &Base,
144                                 SDValue &OffImm);
145  bool SelectThumbAddrModeImm5S4(SDValue N, SDValue &Base,
146                                 SDValue &OffImm);
147  bool SelectThumbAddrModeSP(SDValue N, SDValue &Base, SDValue &OffImm);
148
149  // Thumb 2 Addressing Modes:
150  bool SelectT2ShifterOperandReg(SDValue N,
151                                 SDValue &BaseReg, SDValue &Opc);
152  bool SelectT2AddrModeImm12(SDValue N, SDValue &Base, SDValue &OffImm);
153  bool SelectT2AddrModeImm8(SDValue N, SDValue &Base,
154                            SDValue &OffImm);
155  bool SelectT2AddrModeImm8Offset(SDNode *Op, SDValue N,
156                                 SDValue &OffImm);
157  bool SelectT2AddrModeSoReg(SDValue N, SDValue &Base,
158                             SDValue &OffReg, SDValue &ShImm);
159
160  inline bool is_so_imm(unsigned Imm) const {
161    return ARM_AM::getSOImmVal(Imm) != -1;
162  }
163
164  inline bool is_so_imm_not(unsigned Imm) const {
165    return ARM_AM::getSOImmVal(~Imm) != -1;
166  }
167
168  inline bool is_t2_so_imm(unsigned Imm) const {
169    return ARM_AM::getT2SOImmVal(Imm) != -1;
170  }
171
172  inline bool is_t2_so_imm_not(unsigned Imm) const {
173    return ARM_AM::getT2SOImmVal(~Imm) != -1;
174  }
175
176  inline bool Pred_so_imm(SDNode *inN) const {
177    ConstantSDNode *N = cast<ConstantSDNode>(inN);
178    return is_so_imm(N->getZExtValue());
179  }
180
181  inline bool Pred_t2_so_imm(SDNode *inN) const {
182    ConstantSDNode *N = cast<ConstantSDNode>(inN);
183    return is_t2_so_imm(N->getZExtValue());
184  }
185
186  // Include the pieces autogenerated from the target description.
187#include "ARMGenDAGISel.inc"
188
189private:
190  /// SelectARMIndexedLoad - Indexed (pre/post inc/dec) load matching code for
191  /// ARM.
192  SDNode *SelectARMIndexedLoad(SDNode *N);
193  SDNode *SelectT2IndexedLoad(SDNode *N);
194
195  /// SelectVLD - Select NEON load intrinsics.  NumVecs should be
196  /// 1, 2, 3 or 4.  The opcode arrays specify the instructions used for
197  /// loads of D registers and even subregs and odd subregs of Q registers.
198  /// For NumVecs <= 2, QOpcodes1 is not used.
199  SDNode *SelectVLD(SDNode *N, unsigned NumVecs, unsigned *DOpcodes,
200                    unsigned *QOpcodes0, unsigned *QOpcodes1);
201
202  /// SelectVST - Select NEON store intrinsics.  NumVecs should
203  /// be 1, 2, 3 or 4.  The opcode arrays specify the instructions used for
204  /// stores of D registers and even subregs and odd subregs of Q registers.
205  /// For NumVecs <= 2, QOpcodes1 is not used.
206  SDNode *SelectVST(SDNode *N, unsigned NumVecs, unsigned *DOpcodes,
207                    unsigned *QOpcodes0, unsigned *QOpcodes1);
208
209  /// SelectVLDSTLane - Select NEON load/store lane intrinsics.  NumVecs should
210  /// be 2, 3 or 4.  The opcode arrays specify the instructions used for
211  /// load/store of D registers and Q registers.
212  SDNode *SelectVLDSTLane(SDNode *N, bool IsLoad, unsigned NumVecs,
213                          unsigned *DOpcodes, unsigned *QOpcodes);
214
215  /// SelectVLDDup - Select NEON load-duplicate intrinsics.  NumVecs
216  /// should be 2, 3 or 4.  The opcode array specifies the instructions used
217  /// for loading D registers.  (Q registers are not supported.)
218  SDNode *SelectVLDDup(SDNode *N, unsigned NumVecs, unsigned *Opcodes);
219
220  /// SelectVTBL - Select NEON VTBL and VTBX intrinsics.  NumVecs should be 2,
221  /// 3 or 4.  These are custom-selected so that a REG_SEQUENCE can be
222  /// generated to force the table registers to be consecutive.
223  SDNode *SelectVTBL(SDNode *N, bool IsExt, unsigned NumVecs, unsigned Opc);
224
225  /// SelectV6T2BitfieldExtractOp - Select SBFX/UBFX instructions for ARM.
226  SDNode *SelectV6T2BitfieldExtractOp(SDNode *N, bool isSigned);
227
228  /// SelectCMOVOp - Select CMOV instructions for ARM.
229  SDNode *SelectCMOVOp(SDNode *N);
230  SDNode *SelectT2CMOVShiftOp(SDNode *N, SDValue FalseVal, SDValue TrueVal,
231                              ARMCC::CondCodes CCVal, SDValue CCR,
232                              SDValue InFlag);
233  SDNode *SelectARMCMOVShiftOp(SDNode *N, SDValue FalseVal, SDValue TrueVal,
234                               ARMCC::CondCodes CCVal, SDValue CCR,
235                               SDValue InFlag);
236  SDNode *SelectT2CMOVImmOp(SDNode *N, SDValue FalseVal, SDValue TrueVal,
237                              ARMCC::CondCodes CCVal, SDValue CCR,
238                              SDValue InFlag);
239  SDNode *SelectARMCMOVImmOp(SDNode *N, SDValue FalseVal, SDValue TrueVal,
240                               ARMCC::CondCodes CCVal, SDValue CCR,
241                               SDValue InFlag);
242
243  SDNode *SelectConcatVector(SDNode *N);
244
245  /// SelectInlineAsmMemoryOperand - Implement addressing mode selection for
246  /// inline asm expressions.
247  virtual bool SelectInlineAsmMemoryOperand(const SDValue &Op,
248                                            char ConstraintCode,
249                                            std::vector<SDValue> &OutOps);
250
251  // Form pairs of consecutive S, D, or Q registers.
252  SDNode *PairSRegs(EVT VT, SDValue V0, SDValue V1);
253  SDNode *PairDRegs(EVT VT, SDValue V0, SDValue V1);
254  SDNode *PairQRegs(EVT VT, SDValue V0, SDValue V1);
255
256  // Form sequences of 4 consecutive S, D, or Q registers.
257  SDNode *QuadSRegs(EVT VT, SDValue V0, SDValue V1, SDValue V2, SDValue V3);
258  SDNode *QuadDRegs(EVT VT, SDValue V0, SDValue V1, SDValue V2, SDValue V3);
259  SDNode *QuadQRegs(EVT VT, SDValue V0, SDValue V1, SDValue V2, SDValue V3);
260
261  // Get the alignment operand for a NEON VLD or VST instruction.
262  SDValue GetVLDSTAlign(SDValue Align, unsigned NumVecs, bool is64BitVector);
263};
264}
265
266/// isInt32Immediate - This method tests to see if the node is a 32-bit constant
267/// operand. If so Imm will receive the 32-bit value.
268static bool isInt32Immediate(SDNode *N, unsigned &Imm) {
269  if (N->getOpcode() == ISD::Constant && N->getValueType(0) == MVT::i32) {
270    Imm = cast<ConstantSDNode>(N)->getZExtValue();
271    return true;
272  }
273  return false;
274}
275
276// isInt32Immediate - This method tests to see if a constant operand.
277// If so Imm will receive the 32 bit value.
278static bool isInt32Immediate(SDValue N, unsigned &Imm) {
279  return isInt32Immediate(N.getNode(), Imm);
280}
281
282// isOpcWithIntImmediate - This method tests to see if the node is a specific
283// opcode and that it has a immediate integer right operand.
284// If so Imm will receive the 32 bit value.
285static bool isOpcWithIntImmediate(SDNode *N, unsigned Opc, unsigned& Imm) {
286  return N->getOpcode() == Opc &&
287         isInt32Immediate(N->getOperand(1).getNode(), Imm);
288}
289
290/// hasNoVMLxHazardUse - Return true if it's desirable to select a FP MLA / MLS
291/// node. VFP / NEON fp VMLA / VMLS instructions have special RAW hazards (at
292/// least on current ARM implementations) which should be avoidded.
293bool ARMDAGToDAGISel::hasNoVMLxHazardUse(SDNode *N) const {
294  if (OptLevel == CodeGenOpt::None)
295    return true;
296
297  if (!CheckVMLxHazard)
298    return true;
299
300  if (!Subtarget->isCortexA8() && !Subtarget->isCortexA9())
301    return true;
302
303  if (!N->hasOneUse())
304    return false;
305
306  SDNode *Use = *N->use_begin();
307  if (Use->getOpcode() == ISD::CopyToReg)
308    return true;
309  if (Use->isMachineOpcode()) {
310    const TargetInstrDesc &TID = TII->get(Use->getMachineOpcode());
311    if (TID.mayStore())
312      return true;
313    unsigned Opcode = TID.getOpcode();
314    if (Opcode == ARM::VMOVRS || Opcode == ARM::VMOVRRD)
315      return true;
316    // vmlx feeding into another vmlx. We actually want to unfold
317    // the use later in the MLxExpansion pass. e.g.
318    // vmla
319    // vmla (stall 8 cycles)
320    //
321    // vmul (5 cycles)
322    // vadd (5 cycles)
323    // vmla
324    // This adds up to about 18 - 19 cycles.
325    //
326    // vmla
327    // vmul (stall 4 cycles)
328    // vadd adds up to about 14 cycles.
329    return TII->isFpMLxInstruction(Opcode);
330  }
331
332  return false;
333}
334
335bool ARMDAGToDAGISel::isShifterOpProfitable(const SDValue &Shift,
336                                            ARM_AM::ShiftOpc ShOpcVal,
337                                            unsigned ShAmt) {
338  if (!Subtarget->isCortexA9())
339    return true;
340  if (Shift.hasOneUse())
341    return true;
342  // R << 2 is free.
343  return ShOpcVal == ARM_AM::lsl && ShAmt == 2;
344}
345
346bool ARMDAGToDAGISel::SelectShifterOperandReg(SDValue N,
347                                              SDValue &BaseReg,
348                                              SDValue &ShReg,
349                                              SDValue &Opc) {
350  if (DisableShifterOp)
351    return false;
352
353  ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(N);
354
355  // Don't match base register only case. That is matched to a separate
356  // lower complexity pattern with explicit register operand.
357  if (ShOpcVal == ARM_AM::no_shift) return false;
358
359  BaseReg = N.getOperand(0);
360  unsigned ShImmVal = 0;
361  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
362    ShReg = CurDAG->getRegister(0, MVT::i32);
363    ShImmVal = RHS->getZExtValue() & 31;
364  } else {
365    ShReg = N.getOperand(1);
366    if (!isShifterOpProfitable(N, ShOpcVal, ShImmVal))
367      return false;
368  }
369  Opc = CurDAG->getTargetConstant(ARM_AM::getSORegOpc(ShOpcVal, ShImmVal),
370                                  MVT::i32);
371  return true;
372}
373
374bool ARMDAGToDAGISel::SelectShiftShifterOperandReg(SDValue N,
375                                                   SDValue &BaseReg,
376                                                   SDValue &ShReg,
377                                                   SDValue &Opc) {
378  ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(N);
379
380  // Don't match base register only case. That is matched to a separate
381  // lower complexity pattern with explicit register operand.
382  if (ShOpcVal == ARM_AM::no_shift) return false;
383
384  BaseReg = N.getOperand(0);
385  unsigned ShImmVal = 0;
386  // Do not check isShifterOpProfitable. This must return true.
387  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
388    ShReg = CurDAG->getRegister(0, MVT::i32);
389    ShImmVal = RHS->getZExtValue() & 31;
390  } else {
391    ShReg = N.getOperand(1);
392  }
393  Opc = CurDAG->getTargetConstant(ARM_AM::getSORegOpc(ShOpcVal, ShImmVal),
394                                  MVT::i32);
395  return true;
396}
397
398bool ARMDAGToDAGISel::SelectAddrModeImm12(SDValue N,
399                                          SDValue &Base,
400                                          SDValue &OffImm) {
401  // Match simple R + imm12 operands.
402
403  // Base only.
404  if (N.getOpcode() != ISD::ADD && N.getOpcode() != ISD::SUB) {
405    if (N.getOpcode() == ISD::FrameIndex) {
406      // Match frame index...
407      int FI = cast<FrameIndexSDNode>(N)->getIndex();
408      Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
409      OffImm  = CurDAG->getTargetConstant(0, MVT::i32);
410      return true;
411    } else if (N.getOpcode() == ARMISD::Wrapper &&
412               !(Subtarget->useMovt() &&
413                 N.getOperand(0).getOpcode() == ISD::TargetGlobalAddress)) {
414      Base = N.getOperand(0);
415    } else
416      Base = N;
417    OffImm  = CurDAG->getTargetConstant(0, MVT::i32);
418    return true;
419  }
420
421  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
422    int RHSC = (int)RHS->getZExtValue();
423    if (N.getOpcode() == ISD::SUB)
424      RHSC = -RHSC;
425
426    if (RHSC >= 0 && RHSC < 0x1000) { // 12 bits (unsigned)
427      Base   = N.getOperand(0);
428      if (Base.getOpcode() == ISD::FrameIndex) {
429        int FI = cast<FrameIndexSDNode>(Base)->getIndex();
430        Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
431      }
432      OffImm = CurDAG->getTargetConstant(RHSC, MVT::i32);
433      return true;
434    }
435  }
436
437  // Base only.
438  Base = N;
439  OffImm  = CurDAG->getTargetConstant(0, MVT::i32);
440  return true;
441}
442
443
444
445bool ARMDAGToDAGISel::SelectLdStSOReg(SDValue N, SDValue &Base, SDValue &Offset,
446                                      SDValue &Opc) {
447  if (N.getOpcode() == ISD::MUL &&
448      (!Subtarget->isCortexA9() || N.hasOneUse())) {
449    if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
450      // X * [3,5,9] -> X + X * [2,4,8] etc.
451      int RHSC = (int)RHS->getZExtValue();
452      if (RHSC & 1) {
453        RHSC = RHSC & ~1;
454        ARM_AM::AddrOpc AddSub = ARM_AM::add;
455        if (RHSC < 0) {
456          AddSub = ARM_AM::sub;
457          RHSC = - RHSC;
458        }
459        if (isPowerOf2_32(RHSC)) {
460          unsigned ShAmt = Log2_32(RHSC);
461          Base = Offset = N.getOperand(0);
462          Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, ShAmt,
463                                                            ARM_AM::lsl),
464                                          MVT::i32);
465          return true;
466        }
467      }
468    }
469  }
470
471  if (N.getOpcode() != ISD::ADD && N.getOpcode() != ISD::SUB)
472    return false;
473
474  // Leave simple R +/- imm12 operands for LDRi12
475  if (N.getOpcode() == ISD::ADD) {
476    if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
477      int RHSC = (int)RHS->getZExtValue();
478      if ((RHSC >= 0 && RHSC < 0x1000) ||
479          (RHSC < 0 && RHSC > -0x1000)) // 12 bits.
480        return false;
481    }
482  }
483
484  if (Subtarget->isCortexA9() && !N.hasOneUse())
485    // Compute R +/- (R << N) and reuse it.
486    return false;
487
488  // Otherwise this is R +/- [possibly shifted] R.
489  ARM_AM::AddrOpc AddSub = N.getOpcode() == ISD::ADD ? ARM_AM::add:ARM_AM::sub;
490  ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(N.getOperand(1));
491  unsigned ShAmt = 0;
492
493  Base   = N.getOperand(0);
494  Offset = N.getOperand(1);
495
496  if (ShOpcVal != ARM_AM::no_shift) {
497    // Check to see if the RHS of the shift is a constant, if not, we can't fold
498    // it.
499    if (ConstantSDNode *Sh =
500           dyn_cast<ConstantSDNode>(N.getOperand(1).getOperand(1))) {
501      ShAmt = Sh->getZExtValue();
502      if (isShifterOpProfitable(Offset, ShOpcVal, ShAmt))
503        Offset = N.getOperand(1).getOperand(0);
504      else {
505        ShAmt = 0;
506        ShOpcVal = ARM_AM::no_shift;
507      }
508    } else {
509      ShOpcVal = ARM_AM::no_shift;
510    }
511  }
512
513  // Try matching (R shl C) + (R).
514  if (N.getOpcode() == ISD::ADD && ShOpcVal == ARM_AM::no_shift &&
515      !(Subtarget->isCortexA9() || N.getOperand(0).hasOneUse())) {
516    ShOpcVal = ARM_AM::getShiftOpcForNode(N.getOperand(0));
517    if (ShOpcVal != ARM_AM::no_shift) {
518      // Check to see if the RHS of the shift is a constant, if not, we can't
519      // fold it.
520      if (ConstantSDNode *Sh =
521          dyn_cast<ConstantSDNode>(N.getOperand(0).getOperand(1))) {
522        ShAmt = Sh->getZExtValue();
523        if (!Subtarget->isCortexA9() ||
524            (N.hasOneUse() &&
525             isShifterOpProfitable(N.getOperand(0), ShOpcVal, ShAmt))) {
526          Offset = N.getOperand(0).getOperand(0);
527          Base = N.getOperand(1);
528        } else {
529          ShAmt = 0;
530          ShOpcVal = ARM_AM::no_shift;
531        }
532      } else {
533        ShOpcVal = ARM_AM::no_shift;
534      }
535    }
536  }
537
538  Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, ShAmt, ShOpcVal),
539                                  MVT::i32);
540  return true;
541}
542
543
544
545
546//-----
547
548AddrMode2Type ARMDAGToDAGISel::SelectAddrMode2Worker(SDValue N,
549                                                     SDValue &Base,
550                                                     SDValue &Offset,
551                                                     SDValue &Opc) {
552  if (N.getOpcode() == ISD::MUL &&
553      (!Subtarget->isCortexA9() || N.hasOneUse())) {
554    if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
555      // X * [3,5,9] -> X + X * [2,4,8] etc.
556      int RHSC = (int)RHS->getZExtValue();
557      if (RHSC & 1) {
558        RHSC = RHSC & ~1;
559        ARM_AM::AddrOpc AddSub = ARM_AM::add;
560        if (RHSC < 0) {
561          AddSub = ARM_AM::sub;
562          RHSC = - RHSC;
563        }
564        if (isPowerOf2_32(RHSC)) {
565          unsigned ShAmt = Log2_32(RHSC);
566          Base = Offset = N.getOperand(0);
567          Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, ShAmt,
568                                                            ARM_AM::lsl),
569                                          MVT::i32);
570          return AM2_SHOP;
571        }
572      }
573    }
574  }
575
576  if (N.getOpcode() != ISD::ADD && N.getOpcode() != ISD::SUB) {
577    Base = N;
578    if (N.getOpcode() == ISD::FrameIndex) {
579      int FI = cast<FrameIndexSDNode>(N)->getIndex();
580      Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
581    } else if (N.getOpcode() == ARMISD::Wrapper &&
582               !(Subtarget->useMovt() &&
583                 N.getOperand(0).getOpcode() == ISD::TargetGlobalAddress)) {
584      Base = N.getOperand(0);
585    }
586    Offset = CurDAG->getRegister(0, MVT::i32);
587    Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(ARM_AM::add, 0,
588                                                      ARM_AM::no_shift),
589                                    MVT::i32);
590    return AM2_BASE;
591  }
592
593  // Match simple R +/- imm12 operands.
594  if (N.getOpcode() == ISD::ADD) {
595    if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
596      int RHSC = (int)RHS->getZExtValue();
597      if ((RHSC >= 0 && RHSC < 0x1000) ||
598          (RHSC < 0 && RHSC > -0x1000)) { // 12 bits.
599        Base = N.getOperand(0);
600        if (Base.getOpcode() == ISD::FrameIndex) {
601          int FI = cast<FrameIndexSDNode>(Base)->getIndex();
602          Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
603        }
604        Offset = CurDAG->getRegister(0, MVT::i32);
605
606        ARM_AM::AddrOpc AddSub = ARM_AM::add;
607        if (RHSC < 0) {
608          AddSub = ARM_AM::sub;
609          RHSC = - RHSC;
610        }
611        Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, RHSC,
612                                                          ARM_AM::no_shift),
613                                        MVT::i32);
614        return AM2_BASE;
615      }
616    }
617  }
618
619  if (Subtarget->isCortexA9() && !N.hasOneUse()) {
620    // Compute R +/- (R << N) and reuse it.
621    Base = N;
622    Offset = CurDAG->getRegister(0, MVT::i32);
623    Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(ARM_AM::add, 0,
624                                                      ARM_AM::no_shift),
625                                    MVT::i32);
626    return AM2_BASE;
627  }
628
629  // Otherwise this is R +/- [possibly shifted] R.
630  ARM_AM::AddrOpc AddSub = N.getOpcode() == ISD::ADD ? ARM_AM::add:ARM_AM::sub;
631  ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(N.getOperand(1));
632  unsigned ShAmt = 0;
633
634  Base   = N.getOperand(0);
635  Offset = N.getOperand(1);
636
637  if (ShOpcVal != ARM_AM::no_shift) {
638    // Check to see if the RHS of the shift is a constant, if not, we can't fold
639    // it.
640    if (ConstantSDNode *Sh =
641           dyn_cast<ConstantSDNode>(N.getOperand(1).getOperand(1))) {
642      ShAmt = Sh->getZExtValue();
643      if (isShifterOpProfitable(Offset, ShOpcVal, ShAmt))
644        Offset = N.getOperand(1).getOperand(0);
645      else {
646        ShAmt = 0;
647        ShOpcVal = ARM_AM::no_shift;
648      }
649    } else {
650      ShOpcVal = ARM_AM::no_shift;
651    }
652  }
653
654  // Try matching (R shl C) + (R).
655  if (N.getOpcode() == ISD::ADD && ShOpcVal == ARM_AM::no_shift &&
656      !(Subtarget->isCortexA9() || N.getOperand(0).hasOneUse())) {
657    ShOpcVal = ARM_AM::getShiftOpcForNode(N.getOperand(0));
658    if (ShOpcVal != ARM_AM::no_shift) {
659      // Check to see if the RHS of the shift is a constant, if not, we can't
660      // fold it.
661      if (ConstantSDNode *Sh =
662          dyn_cast<ConstantSDNode>(N.getOperand(0).getOperand(1))) {
663        ShAmt = Sh->getZExtValue();
664        if (!Subtarget->isCortexA9() ||
665            (N.hasOneUse() &&
666             isShifterOpProfitable(N.getOperand(0), ShOpcVal, ShAmt))) {
667          Offset = N.getOperand(0).getOperand(0);
668          Base = N.getOperand(1);
669        } else {
670          ShAmt = 0;
671          ShOpcVal = ARM_AM::no_shift;
672        }
673      } else {
674        ShOpcVal = ARM_AM::no_shift;
675      }
676    }
677  }
678
679  Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, ShAmt, ShOpcVal),
680                                  MVT::i32);
681  return AM2_SHOP;
682}
683
684bool ARMDAGToDAGISel::SelectAddrMode2Offset(SDNode *Op, SDValue N,
685                                            SDValue &Offset, SDValue &Opc) {
686  unsigned Opcode = Op->getOpcode();
687  ISD::MemIndexedMode AM = (Opcode == ISD::LOAD)
688    ? cast<LoadSDNode>(Op)->getAddressingMode()
689    : cast<StoreSDNode>(Op)->getAddressingMode();
690  ARM_AM::AddrOpc AddSub = (AM == ISD::PRE_INC || AM == ISD::POST_INC)
691    ? ARM_AM::add : ARM_AM::sub;
692  if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(N)) {
693    int Val = (int)C->getZExtValue();
694    if (Val >= 0 && Val < 0x1000) { // 12 bits.
695      Offset = CurDAG->getRegister(0, MVT::i32);
696      Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, Val,
697                                                        ARM_AM::no_shift),
698                                      MVT::i32);
699      return true;
700    }
701  }
702
703  Offset = N;
704  ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(N);
705  unsigned ShAmt = 0;
706  if (ShOpcVal != ARM_AM::no_shift) {
707    // Check to see if the RHS of the shift is a constant, if not, we can't fold
708    // it.
709    if (ConstantSDNode *Sh = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
710      ShAmt = Sh->getZExtValue();
711      if (isShifterOpProfitable(N, ShOpcVal, ShAmt))
712        Offset = N.getOperand(0);
713      else {
714        ShAmt = 0;
715        ShOpcVal = ARM_AM::no_shift;
716      }
717    } else {
718      ShOpcVal = ARM_AM::no_shift;
719    }
720  }
721
722  Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, ShAmt, ShOpcVal),
723                                  MVT::i32);
724  return true;
725}
726
727
728bool ARMDAGToDAGISel::SelectAddrMode3(SDValue N,
729                                      SDValue &Base, SDValue &Offset,
730                                      SDValue &Opc) {
731  if (N.getOpcode() == ISD::SUB) {
732    // X - C  is canonicalize to X + -C, no need to handle it here.
733    Base = N.getOperand(0);
734    Offset = N.getOperand(1);
735    Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(ARM_AM::sub, 0),MVT::i32);
736    return true;
737  }
738
739  if (N.getOpcode() != ISD::ADD) {
740    Base = N;
741    if (N.getOpcode() == ISD::FrameIndex) {
742      int FI = cast<FrameIndexSDNode>(N)->getIndex();
743      Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
744    }
745    Offset = CurDAG->getRegister(0, MVT::i32);
746    Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(ARM_AM::add, 0),MVT::i32);
747    return true;
748  }
749
750  // If the RHS is +/- imm8, fold into addr mode.
751  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
752    int RHSC = (int)RHS->getZExtValue();
753    if ((RHSC >= 0 && RHSC < 256) ||
754        (RHSC < 0 && RHSC > -256)) { // note -256 itself isn't allowed.
755      Base = N.getOperand(0);
756      if (Base.getOpcode() == ISD::FrameIndex) {
757        int FI = cast<FrameIndexSDNode>(Base)->getIndex();
758        Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
759      }
760      Offset = CurDAG->getRegister(0, MVT::i32);
761
762      ARM_AM::AddrOpc AddSub = ARM_AM::add;
763      if (RHSC < 0) {
764        AddSub = ARM_AM::sub;
765        RHSC = - RHSC;
766      }
767      Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(AddSub, RHSC),MVT::i32);
768      return true;
769    }
770  }
771
772  Base = N.getOperand(0);
773  Offset = N.getOperand(1);
774  Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(ARM_AM::add, 0), MVT::i32);
775  return true;
776}
777
778bool ARMDAGToDAGISel::SelectAddrMode3Offset(SDNode *Op, SDValue N,
779                                            SDValue &Offset, SDValue &Opc) {
780  unsigned Opcode = Op->getOpcode();
781  ISD::MemIndexedMode AM = (Opcode == ISD::LOAD)
782    ? cast<LoadSDNode>(Op)->getAddressingMode()
783    : cast<StoreSDNode>(Op)->getAddressingMode();
784  ARM_AM::AddrOpc AddSub = (AM == ISD::PRE_INC || AM == ISD::POST_INC)
785    ? ARM_AM::add : ARM_AM::sub;
786  if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(N)) {
787    int Val = (int)C->getZExtValue();
788    if (Val >= 0 && Val < 256) {
789      Offset = CurDAG->getRegister(0, MVT::i32);
790      Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(AddSub, Val), MVT::i32);
791      return true;
792    }
793  }
794
795  Offset = N;
796  Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(AddSub, 0), MVT::i32);
797  return true;
798}
799
800bool ARMDAGToDAGISel::SelectAddrMode5(SDValue N,
801                                      SDValue &Base, SDValue &Offset) {
802  if (N.getOpcode() != ISD::ADD) {
803    Base = N;
804    if (N.getOpcode() == ISD::FrameIndex) {
805      int FI = cast<FrameIndexSDNode>(N)->getIndex();
806      Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
807    } else if (N.getOpcode() == ARMISD::Wrapper &&
808               !(Subtarget->useMovt() &&
809                 N.getOperand(0).getOpcode() == ISD::TargetGlobalAddress)) {
810      Base = N.getOperand(0);
811    }
812    Offset = CurDAG->getTargetConstant(ARM_AM::getAM5Opc(ARM_AM::add, 0),
813                                       MVT::i32);
814    return true;
815  }
816
817  // If the RHS is +/- imm8, fold into addr mode.
818  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
819    int RHSC = (int)RHS->getZExtValue();
820    if ((RHSC & 3) == 0) {  // The constant is implicitly multiplied by 4.
821      RHSC >>= 2;
822      if ((RHSC >= 0 && RHSC < 256) ||
823          (RHSC < 0 && RHSC > -256)) { // note -256 itself isn't allowed.
824        Base = N.getOperand(0);
825        if (Base.getOpcode() == ISD::FrameIndex) {
826          int FI = cast<FrameIndexSDNode>(Base)->getIndex();
827          Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
828        }
829
830        ARM_AM::AddrOpc AddSub = ARM_AM::add;
831        if (RHSC < 0) {
832          AddSub = ARM_AM::sub;
833          RHSC = - RHSC;
834        }
835        Offset = CurDAG->getTargetConstant(ARM_AM::getAM5Opc(AddSub, RHSC),
836                                           MVT::i32);
837        return true;
838      }
839    }
840  }
841
842  Base = N;
843  Offset = CurDAG->getTargetConstant(ARM_AM::getAM5Opc(ARM_AM::add, 0),
844                                     MVT::i32);
845  return true;
846}
847
848bool ARMDAGToDAGISel::SelectAddrMode6(SDNode *Parent, SDValue N, SDValue &Addr,
849                                      SDValue &Align) {
850  Addr = N;
851
852  unsigned Alignment = 0;
853  if (LSBaseSDNode *LSN = dyn_cast<LSBaseSDNode>(Parent)) {
854    // This case occurs only for VLD1-lane/dup and VST1-lane instructions.
855    // The maximum alignment is equal to the memory size being referenced.
856    unsigned LSNAlign = LSN->getAlignment();
857    unsigned MemSize = LSN->getMemoryVT().getSizeInBits() / 8;
858    if (LSNAlign > MemSize && MemSize > 1)
859      Alignment = MemSize;
860  } else {
861    // All other uses of addrmode6 are for intrinsics.  For now just record
862    // the raw alignment value; it will be refined later based on the legal
863    // alignment operands for the intrinsic.
864    Alignment = cast<MemIntrinsicSDNode>(Parent)->getAlignment();
865  }
866
867  Align = CurDAG->getTargetConstant(Alignment, MVT::i32);
868  return true;
869}
870
871bool ARMDAGToDAGISel::SelectAddrModePC(SDValue N,
872                                       SDValue &Offset, SDValue &Label) {
873  if (N.getOpcode() == ARMISD::PIC_ADD && N.hasOneUse()) {
874    Offset = N.getOperand(0);
875    SDValue N1 = N.getOperand(1);
876    Label  = CurDAG->getTargetConstant(cast<ConstantSDNode>(N1)->getZExtValue(),
877                                       MVT::i32);
878    return true;
879  }
880
881  return false;
882}
883
884
885//===----------------------------------------------------------------------===//
886//                         Thumb Addressing Modes
887//===----------------------------------------------------------------------===//
888
889
890bool ARMDAGToDAGISel::SelectThumbAddrModeRR(SDValue N,
891                                            SDValue &Base, SDValue &Offset){
892  // FIXME dl should come from the parent load or store, not the address
893  if (N.getOpcode() != ISD::ADD) {
894    ConstantSDNode *NC = dyn_cast<ConstantSDNode>(N);
895    if (!NC || !NC->isNullValue())
896      return false;
897
898    Base = Offset = N;
899    return true;
900  }
901
902  Base = N.getOperand(0);
903  Offset = N.getOperand(1);
904  return true;
905}
906
907bool
908ARMDAGToDAGISel::SelectThumbAddrModeRI(SDValue N, SDValue &Base,
909                                       SDValue &Offset, unsigned Scale) {
910  if (Scale == 4) {
911    SDValue TmpBase, TmpOffImm;
912    if (SelectThumbAddrModeSP(N, TmpBase, TmpOffImm))
913      return false;  // We want to select tLDRspi / tSTRspi instead.
914
915    if (N.getOpcode() == ARMISD::Wrapper &&
916        N.getOperand(0).getOpcode() == ISD::TargetConstantPool)
917      return false;  // We want to select tLDRpci instead.
918  }
919
920  if (N.getOpcode() != ISD::ADD)
921    return false;
922
923  // Thumb does not have [sp, r] address mode.
924  RegisterSDNode *LHSR = dyn_cast<RegisterSDNode>(N.getOperand(0));
925  RegisterSDNode *RHSR = dyn_cast<RegisterSDNode>(N.getOperand(1));
926  if ((LHSR && LHSR->getReg() == ARM::SP) ||
927      (RHSR && RHSR->getReg() == ARM::SP))
928    return false;
929
930  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
931    int RHSC = (int)RHS->getZExtValue();
932
933    if ((RHSC & (Scale - 1)) == 0) { // The constant is implicitly multiplied.
934      RHSC /= Scale;
935
936      if (RHSC >= 0 && RHSC < 32)
937        return false;
938    }
939  }
940
941  Base = N.getOperand(0);
942  Offset = N.getOperand(1);
943  return true;
944}
945
946bool
947ARMDAGToDAGISel::SelectThumbAddrModeRI5S1(SDValue N,
948                                          SDValue &Base,
949                                          SDValue &Offset) {
950  return SelectThumbAddrModeRI(N, Base, Offset, 1);
951}
952
953bool
954ARMDAGToDAGISel::SelectThumbAddrModeRI5S2(SDValue N,
955                                          SDValue &Base,
956                                          SDValue &Offset) {
957  return SelectThumbAddrModeRI(N, Base, Offset, 2);
958}
959
960bool
961ARMDAGToDAGISel::SelectThumbAddrModeRI5S4(SDValue N,
962                                          SDValue &Base,
963                                          SDValue &Offset) {
964  return SelectThumbAddrModeRI(N, Base, Offset, 4);
965}
966
967bool
968ARMDAGToDAGISel::SelectThumbAddrModeImm5S(SDValue N, unsigned Scale,
969                                          SDValue &Base, SDValue &OffImm) {
970  if (Scale == 4) {
971    SDValue TmpBase, TmpOffImm;
972    if (SelectThumbAddrModeSP(N, TmpBase, TmpOffImm))
973      return false;  // We want to select tLDRspi / tSTRspi instead.
974
975    if (N.getOpcode() == ARMISD::Wrapper &&
976        N.getOperand(0).getOpcode() == ISD::TargetConstantPool)
977      return false;  // We want to select tLDRpci instead.
978  }
979
980  if (N.getOpcode() != ISD::ADD) {
981    if (N.getOpcode() == ARMISD::Wrapper &&
982        !(Subtarget->useMovt() &&
983          N.getOperand(0).getOpcode() == ISD::TargetGlobalAddress)) {
984      Base = N.getOperand(0);
985    } else {
986      Base = N;
987    }
988
989    OffImm = CurDAG->getTargetConstant(0, MVT::i32);
990    return true;
991  }
992
993  RegisterSDNode *LHSR = dyn_cast<RegisterSDNode>(N.getOperand(0));
994  RegisterSDNode *RHSR = dyn_cast<RegisterSDNode>(N.getOperand(1));
995  if ((LHSR && LHSR->getReg() == ARM::SP) ||
996      (RHSR && RHSR->getReg() == ARM::SP)) {
997    ConstantSDNode *LHS = dyn_cast<ConstantSDNode>(N.getOperand(0));
998    ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1));
999    unsigned LHSC = LHS ? LHS->getZExtValue() : 0;
1000    unsigned RHSC = RHS ? RHS->getZExtValue() : 0;
1001
1002    // Thumb does not have [sp, #imm5] address mode for non-zero imm5.
1003    if (LHSC != 0 || RHSC != 0) return false;
1004
1005    Base = N;
1006    OffImm = CurDAG->getTargetConstant(0, MVT::i32);
1007    return true;
1008  }
1009
1010  // If the RHS is + imm5 * scale, fold into addr mode.
1011  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
1012    int RHSC = (int)RHS->getZExtValue();
1013
1014    if ((RHSC & (Scale - 1)) == 0) { // The constant is implicitly multiplied.
1015      RHSC /= Scale;
1016
1017      if (RHSC >= 0 && RHSC < 32) {
1018        Base = N.getOperand(0);
1019        OffImm = CurDAG->getTargetConstant(RHSC, MVT::i32);
1020        return true;
1021      }
1022    }
1023  }
1024
1025  Base = N.getOperand(0);
1026  OffImm = CurDAG->getTargetConstant(0, MVT::i32);
1027  return true;
1028}
1029
1030bool
1031ARMDAGToDAGISel::SelectThumbAddrModeImm5S4(SDValue N, SDValue &Base,
1032                                           SDValue &OffImm) {
1033  return SelectThumbAddrModeImm5S(N, 4, Base, OffImm);
1034}
1035
1036bool
1037ARMDAGToDAGISel::SelectThumbAddrModeImm5S2(SDValue N, SDValue &Base,
1038                                           SDValue &OffImm) {
1039  return SelectThumbAddrModeImm5S(N, 2, Base, OffImm);
1040}
1041
1042bool
1043ARMDAGToDAGISel::SelectThumbAddrModeImm5S1(SDValue N, SDValue &Base,
1044                                           SDValue &OffImm) {
1045  return SelectThumbAddrModeImm5S(N, 1, Base, OffImm);
1046}
1047
1048bool ARMDAGToDAGISel::SelectThumbAddrModeSP(SDValue N,
1049                                            SDValue &Base, SDValue &OffImm) {
1050  if (N.getOpcode() == ISD::FrameIndex) {
1051    int FI = cast<FrameIndexSDNode>(N)->getIndex();
1052    Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
1053    OffImm = CurDAG->getTargetConstant(0, MVT::i32);
1054    return true;
1055  }
1056
1057  if (N.getOpcode() != ISD::ADD)
1058    return false;
1059
1060  RegisterSDNode *LHSR = dyn_cast<RegisterSDNode>(N.getOperand(0));
1061  if (N.getOperand(0).getOpcode() == ISD::FrameIndex ||
1062      (LHSR && LHSR->getReg() == ARM::SP)) {
1063    // If the RHS is + imm8 * scale, fold into addr mode.
1064    if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
1065      int RHSC = (int)RHS->getZExtValue();
1066      if ((RHSC & 3) == 0) {  // The constant is implicitly multiplied.
1067        RHSC >>= 2;
1068        if (RHSC >= 0 && RHSC < 256) {
1069          Base = N.getOperand(0);
1070          if (Base.getOpcode() == ISD::FrameIndex) {
1071            int FI = cast<FrameIndexSDNode>(Base)->getIndex();
1072            Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
1073          }
1074          OffImm = CurDAG->getTargetConstant(RHSC, MVT::i32);
1075          return true;
1076        }
1077      }
1078    }
1079  }
1080
1081  return false;
1082}
1083
1084
1085//===----------------------------------------------------------------------===//
1086//                        Thumb 2 Addressing Modes
1087//===----------------------------------------------------------------------===//
1088
1089
1090bool ARMDAGToDAGISel::SelectT2ShifterOperandReg(SDValue N, SDValue &BaseReg,
1091                                                SDValue &Opc) {
1092  if (DisableShifterOp)
1093    return false;
1094
1095  ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(N);
1096
1097  // Don't match base register only case. That is matched to a separate
1098  // lower complexity pattern with explicit register operand.
1099  if (ShOpcVal == ARM_AM::no_shift) return false;
1100
1101  BaseReg = N.getOperand(0);
1102  unsigned ShImmVal = 0;
1103  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
1104    ShImmVal = RHS->getZExtValue() & 31;
1105    Opc = getI32Imm(ARM_AM::getSORegOpc(ShOpcVal, ShImmVal));
1106    return true;
1107  }
1108
1109  return false;
1110}
1111
1112bool ARMDAGToDAGISel::SelectT2AddrModeImm12(SDValue N,
1113                                            SDValue &Base, SDValue &OffImm) {
1114  // Match simple R + imm12 operands.
1115
1116  // Base only.
1117  if (N.getOpcode() != ISD::ADD && N.getOpcode() != ISD::SUB) {
1118    if (N.getOpcode() == ISD::FrameIndex) {
1119      // Match frame index...
1120      int FI = cast<FrameIndexSDNode>(N)->getIndex();
1121      Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
1122      OffImm  = CurDAG->getTargetConstant(0, MVT::i32);
1123      return true;
1124    } else if (N.getOpcode() == ARMISD::Wrapper &&
1125               !(Subtarget->useMovt() &&
1126                 N.getOperand(0).getOpcode() == ISD::TargetGlobalAddress)) {
1127      Base = N.getOperand(0);
1128      if (Base.getOpcode() == ISD::TargetConstantPool)
1129        return false;  // We want to select t2LDRpci instead.
1130    } else
1131      Base = N;
1132    OffImm  = CurDAG->getTargetConstant(0, MVT::i32);
1133    return true;
1134  }
1135
1136  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
1137    if (SelectT2AddrModeImm8(N, Base, OffImm))
1138      // Let t2LDRi8 handle (R - imm8).
1139      return false;
1140
1141    int RHSC = (int)RHS->getZExtValue();
1142    if (N.getOpcode() == ISD::SUB)
1143      RHSC = -RHSC;
1144
1145    if (RHSC >= 0 && RHSC < 0x1000) { // 12 bits (unsigned)
1146      Base   = N.getOperand(0);
1147      if (Base.getOpcode() == ISD::FrameIndex) {
1148        int FI = cast<FrameIndexSDNode>(Base)->getIndex();
1149        Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
1150      }
1151      OffImm = CurDAG->getTargetConstant(RHSC, MVT::i32);
1152      return true;
1153    }
1154  }
1155
1156  // Base only.
1157  Base = N;
1158  OffImm  = CurDAG->getTargetConstant(0, MVT::i32);
1159  return true;
1160}
1161
1162bool ARMDAGToDAGISel::SelectT2AddrModeImm8(SDValue N,
1163                                           SDValue &Base, SDValue &OffImm) {
1164  // Match simple R - imm8 operands.
1165  if (N.getOpcode() == ISD::ADD || N.getOpcode() == ISD::SUB) {
1166    if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
1167      int RHSC = (int)RHS->getSExtValue();
1168      if (N.getOpcode() == ISD::SUB)
1169        RHSC = -RHSC;
1170
1171      if ((RHSC >= -255) && (RHSC < 0)) { // 8 bits (always negative)
1172        Base = N.getOperand(0);
1173        if (Base.getOpcode() == ISD::FrameIndex) {
1174          int FI = cast<FrameIndexSDNode>(Base)->getIndex();
1175          Base = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
1176        }
1177        OffImm = CurDAG->getTargetConstant(RHSC, MVT::i32);
1178        return true;
1179      }
1180    }
1181  }
1182
1183  return false;
1184}
1185
1186bool ARMDAGToDAGISel::SelectT2AddrModeImm8Offset(SDNode *Op, SDValue N,
1187                                                 SDValue &OffImm){
1188  unsigned Opcode = Op->getOpcode();
1189  ISD::MemIndexedMode AM = (Opcode == ISD::LOAD)
1190    ? cast<LoadSDNode>(Op)->getAddressingMode()
1191    : cast<StoreSDNode>(Op)->getAddressingMode();
1192  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N)) {
1193    int RHSC = (int)RHS->getZExtValue();
1194    if (RHSC >= 0 && RHSC < 0x100) { // 8 bits.
1195      OffImm = ((AM == ISD::PRE_INC) || (AM == ISD::POST_INC))
1196        ? CurDAG->getTargetConstant(RHSC, MVT::i32)
1197        : CurDAG->getTargetConstant(-RHSC, MVT::i32);
1198      return true;
1199    }
1200  }
1201
1202  return false;
1203}
1204
1205bool ARMDAGToDAGISel::SelectT2AddrModeSoReg(SDValue N,
1206                                            SDValue &Base,
1207                                            SDValue &OffReg, SDValue &ShImm) {
1208  // (R - imm8) should be handled by t2LDRi8. The rest are handled by t2LDRi12.
1209  if (N.getOpcode() != ISD::ADD)
1210    return false;
1211
1212  // Leave (R + imm12) for t2LDRi12, (R - imm8) for t2LDRi8.
1213  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
1214    int RHSC = (int)RHS->getZExtValue();
1215    if (RHSC >= 0 && RHSC < 0x1000) // 12 bits (unsigned)
1216      return false;
1217    else if (RHSC < 0 && RHSC >= -255) // 8 bits
1218      return false;
1219  }
1220
1221  if (Subtarget->isCortexA9() && !N.hasOneUse()) {
1222    // Compute R + (R << [1,2,3]) and reuse it.
1223    Base = N;
1224    return false;
1225  }
1226
1227  // Look for (R + R) or (R + (R << [1,2,3])).
1228  unsigned ShAmt = 0;
1229  Base   = N.getOperand(0);
1230  OffReg = N.getOperand(1);
1231
1232  // Swap if it is ((R << c) + R).
1233  ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(OffReg);
1234  if (ShOpcVal != ARM_AM::lsl) {
1235    ShOpcVal = ARM_AM::getShiftOpcForNode(Base);
1236    if (ShOpcVal == ARM_AM::lsl)
1237      std::swap(Base, OffReg);
1238  }
1239
1240  if (ShOpcVal == ARM_AM::lsl) {
1241    // Check to see if the RHS of the shift is a constant, if not, we can't fold
1242    // it.
1243    if (ConstantSDNode *Sh = dyn_cast<ConstantSDNode>(OffReg.getOperand(1))) {
1244      ShAmt = Sh->getZExtValue();
1245      if (ShAmt < 4 && isShifterOpProfitable(OffReg, ShOpcVal, ShAmt))
1246        OffReg = OffReg.getOperand(0);
1247      else {
1248        ShAmt = 0;
1249        ShOpcVal = ARM_AM::no_shift;
1250      }
1251    } else {
1252      ShOpcVal = ARM_AM::no_shift;
1253    }
1254  }
1255
1256  ShImm = CurDAG->getTargetConstant(ShAmt, MVT::i32);
1257
1258  return true;
1259}
1260
1261//===--------------------------------------------------------------------===//
1262
1263/// getAL - Returns a ARMCC::AL immediate node.
1264static inline SDValue getAL(SelectionDAG *CurDAG) {
1265  return CurDAG->getTargetConstant((uint64_t)ARMCC::AL, MVT::i32);
1266}
1267
1268SDNode *ARMDAGToDAGISel::SelectARMIndexedLoad(SDNode *N) {
1269  LoadSDNode *LD = cast<LoadSDNode>(N);
1270  ISD::MemIndexedMode AM = LD->getAddressingMode();
1271  if (AM == ISD::UNINDEXED)
1272    return NULL;
1273
1274  EVT LoadedVT = LD->getMemoryVT();
1275  SDValue Offset, AMOpc;
1276  bool isPre = (AM == ISD::PRE_INC) || (AM == ISD::PRE_DEC);
1277  unsigned Opcode = 0;
1278  bool Match = false;
1279  if (LoadedVT == MVT::i32 &&
1280      SelectAddrMode2Offset(N, LD->getOffset(), Offset, AMOpc)) {
1281    Opcode = isPre ? ARM::LDR_PRE : ARM::LDR_POST;
1282    Match = true;
1283  } else if (LoadedVT == MVT::i16 &&
1284             SelectAddrMode3Offset(N, LD->getOffset(), Offset, AMOpc)) {
1285    Match = true;
1286    Opcode = (LD->getExtensionType() == ISD::SEXTLOAD)
1287      ? (isPre ? ARM::LDRSH_PRE : ARM::LDRSH_POST)
1288      : (isPre ? ARM::LDRH_PRE : ARM::LDRH_POST);
1289  } else if (LoadedVT == MVT::i8 || LoadedVT == MVT::i1) {
1290    if (LD->getExtensionType() == ISD::SEXTLOAD) {
1291      if (SelectAddrMode3Offset(N, LD->getOffset(), Offset, AMOpc)) {
1292        Match = true;
1293        Opcode = isPre ? ARM::LDRSB_PRE : ARM::LDRSB_POST;
1294      }
1295    } else {
1296      if (SelectAddrMode2Offset(N, LD->getOffset(), Offset, AMOpc)) {
1297        Match = true;
1298        Opcode = isPre ? ARM::LDRB_PRE : ARM::LDRB_POST;
1299      }
1300    }
1301  }
1302
1303  if (Match) {
1304    SDValue Chain = LD->getChain();
1305    SDValue Base = LD->getBasePtr();
1306    SDValue Ops[]= { Base, Offset, AMOpc, getAL(CurDAG),
1307                     CurDAG->getRegister(0, MVT::i32), Chain };
1308    return CurDAG->getMachineNode(Opcode, N->getDebugLoc(), MVT::i32, MVT::i32,
1309                                  MVT::Other, Ops, 6);
1310  }
1311
1312  return NULL;
1313}
1314
1315SDNode *ARMDAGToDAGISel::SelectT2IndexedLoad(SDNode *N) {
1316  LoadSDNode *LD = cast<LoadSDNode>(N);
1317  ISD::MemIndexedMode AM = LD->getAddressingMode();
1318  if (AM == ISD::UNINDEXED)
1319    return NULL;
1320
1321  EVT LoadedVT = LD->getMemoryVT();
1322  bool isSExtLd = LD->getExtensionType() == ISD::SEXTLOAD;
1323  SDValue Offset;
1324  bool isPre = (AM == ISD::PRE_INC) || (AM == ISD::PRE_DEC);
1325  unsigned Opcode = 0;
1326  bool Match = false;
1327  if (SelectT2AddrModeImm8Offset(N, LD->getOffset(), Offset)) {
1328    switch (LoadedVT.getSimpleVT().SimpleTy) {
1329    case MVT::i32:
1330      Opcode = isPre ? ARM::t2LDR_PRE : ARM::t2LDR_POST;
1331      break;
1332    case MVT::i16:
1333      if (isSExtLd)
1334        Opcode = isPre ? ARM::t2LDRSH_PRE : ARM::t2LDRSH_POST;
1335      else
1336        Opcode = isPre ? ARM::t2LDRH_PRE : ARM::t2LDRH_POST;
1337      break;
1338    case MVT::i8:
1339    case MVT::i1:
1340      if (isSExtLd)
1341        Opcode = isPre ? ARM::t2LDRSB_PRE : ARM::t2LDRSB_POST;
1342      else
1343        Opcode = isPre ? ARM::t2LDRB_PRE : ARM::t2LDRB_POST;
1344      break;
1345    default:
1346      return NULL;
1347    }
1348    Match = true;
1349  }
1350
1351  if (Match) {
1352    SDValue Chain = LD->getChain();
1353    SDValue Base = LD->getBasePtr();
1354    SDValue Ops[]= { Base, Offset, getAL(CurDAG),
1355                     CurDAG->getRegister(0, MVT::i32), Chain };
1356    return CurDAG->getMachineNode(Opcode, N->getDebugLoc(), MVT::i32, MVT::i32,
1357                                  MVT::Other, Ops, 5);
1358  }
1359
1360  return NULL;
1361}
1362
1363/// PairSRegs - Form a D register from a pair of S registers.
1364///
1365SDNode *ARMDAGToDAGISel::PairSRegs(EVT VT, SDValue V0, SDValue V1) {
1366  DebugLoc dl = V0.getNode()->getDebugLoc();
1367  SDValue SubReg0 = CurDAG->getTargetConstant(ARM::ssub_0, MVT::i32);
1368  SDValue SubReg1 = CurDAG->getTargetConstant(ARM::ssub_1, MVT::i32);
1369  const SDValue Ops[] = { V0, SubReg0, V1, SubReg1 };
1370  return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops, 4);
1371}
1372
1373/// PairDRegs - Form a quad register from a pair of D registers.
1374///
1375SDNode *ARMDAGToDAGISel::PairDRegs(EVT VT, SDValue V0, SDValue V1) {
1376  DebugLoc dl = V0.getNode()->getDebugLoc();
1377  SDValue SubReg0 = CurDAG->getTargetConstant(ARM::dsub_0, MVT::i32);
1378  SDValue SubReg1 = CurDAG->getTargetConstant(ARM::dsub_1, MVT::i32);
1379  const SDValue Ops[] = { V0, SubReg0, V1, SubReg1 };
1380  return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops, 4);
1381}
1382
1383/// PairQRegs - Form 4 consecutive D registers from a pair of Q registers.
1384///
1385SDNode *ARMDAGToDAGISel::PairQRegs(EVT VT, SDValue V0, SDValue V1) {
1386  DebugLoc dl = V0.getNode()->getDebugLoc();
1387  SDValue SubReg0 = CurDAG->getTargetConstant(ARM::qsub_0, MVT::i32);
1388  SDValue SubReg1 = CurDAG->getTargetConstant(ARM::qsub_1, MVT::i32);
1389  const SDValue Ops[] = { V0, SubReg0, V1, SubReg1 };
1390  return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops, 4);
1391}
1392
1393/// QuadSRegs - Form 4 consecutive S registers.
1394///
1395SDNode *ARMDAGToDAGISel::QuadSRegs(EVT VT, SDValue V0, SDValue V1,
1396                                   SDValue V2, SDValue V3) {
1397  DebugLoc dl = V0.getNode()->getDebugLoc();
1398  SDValue SubReg0 = CurDAG->getTargetConstant(ARM::ssub_0, MVT::i32);
1399  SDValue SubReg1 = CurDAG->getTargetConstant(ARM::ssub_1, MVT::i32);
1400  SDValue SubReg2 = CurDAG->getTargetConstant(ARM::ssub_2, MVT::i32);
1401  SDValue SubReg3 = CurDAG->getTargetConstant(ARM::ssub_3, MVT::i32);
1402  const SDValue Ops[] = { V0, SubReg0, V1, SubReg1, V2, SubReg2, V3, SubReg3 };
1403  return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops, 8);
1404}
1405
1406/// QuadDRegs - Form 4 consecutive D registers.
1407///
1408SDNode *ARMDAGToDAGISel::QuadDRegs(EVT VT, SDValue V0, SDValue V1,
1409                                   SDValue V2, SDValue V3) {
1410  DebugLoc dl = V0.getNode()->getDebugLoc();
1411  SDValue SubReg0 = CurDAG->getTargetConstant(ARM::dsub_0, MVT::i32);
1412  SDValue SubReg1 = CurDAG->getTargetConstant(ARM::dsub_1, MVT::i32);
1413  SDValue SubReg2 = CurDAG->getTargetConstant(ARM::dsub_2, MVT::i32);
1414  SDValue SubReg3 = CurDAG->getTargetConstant(ARM::dsub_3, MVT::i32);
1415  const SDValue Ops[] = { V0, SubReg0, V1, SubReg1, V2, SubReg2, V3, SubReg3 };
1416  return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops, 8);
1417}
1418
1419/// QuadQRegs - Form 4 consecutive Q registers.
1420///
1421SDNode *ARMDAGToDAGISel::QuadQRegs(EVT VT, SDValue V0, SDValue V1,
1422                                   SDValue V2, SDValue V3) {
1423  DebugLoc dl = V0.getNode()->getDebugLoc();
1424  SDValue SubReg0 = CurDAG->getTargetConstant(ARM::qsub_0, MVT::i32);
1425  SDValue SubReg1 = CurDAG->getTargetConstant(ARM::qsub_1, MVT::i32);
1426  SDValue SubReg2 = CurDAG->getTargetConstant(ARM::qsub_2, MVT::i32);
1427  SDValue SubReg3 = CurDAG->getTargetConstant(ARM::qsub_3, MVT::i32);
1428  const SDValue Ops[] = { V0, SubReg0, V1, SubReg1, V2, SubReg2, V3, SubReg3 };
1429  return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops, 8);
1430}
1431
1432/// GetVLDSTAlign - Get the alignment (in bytes) for the alignment operand
1433/// of a NEON VLD or VST instruction.  The supported values depend on the
1434/// number of registers being loaded.
1435SDValue ARMDAGToDAGISel::GetVLDSTAlign(SDValue Align, unsigned NumVecs,
1436                                       bool is64BitVector) {
1437  unsigned NumRegs = NumVecs;
1438  if (!is64BitVector && NumVecs < 3)
1439    NumRegs *= 2;
1440
1441  unsigned Alignment = cast<ConstantSDNode>(Align)->getZExtValue();
1442  if (Alignment >= 32 && NumRegs == 4)
1443    Alignment = 32;
1444  else if (Alignment >= 16 && (NumRegs == 2 || NumRegs == 4))
1445    Alignment = 16;
1446  else if (Alignment >= 8)
1447    Alignment = 8;
1448  else
1449    Alignment = 0;
1450
1451  return CurDAG->getTargetConstant(Alignment, MVT::i32);
1452}
1453
1454SDNode *ARMDAGToDAGISel::SelectVLD(SDNode *N, unsigned NumVecs,
1455                                   unsigned *DOpcodes, unsigned *QOpcodes0,
1456                                   unsigned *QOpcodes1) {
1457  assert(NumVecs >= 1 && NumVecs <= 4 && "VLD NumVecs out-of-range");
1458  DebugLoc dl = N->getDebugLoc();
1459
1460  SDValue MemAddr, Align;
1461  if (!SelectAddrMode6(N, N->getOperand(2), MemAddr, Align))
1462    return NULL;
1463
1464  SDValue Chain = N->getOperand(0);
1465  EVT VT = N->getValueType(0);
1466  bool is64BitVector = VT.is64BitVector();
1467  Align = GetVLDSTAlign(Align, NumVecs, is64BitVector);
1468
1469  unsigned OpcodeIndex;
1470  switch (VT.getSimpleVT().SimpleTy) {
1471  default: llvm_unreachable("unhandled vld type");
1472    // Double-register operations:
1473  case MVT::v8i8:  OpcodeIndex = 0; break;
1474  case MVT::v4i16: OpcodeIndex = 1; break;
1475  case MVT::v2f32:
1476  case MVT::v2i32: OpcodeIndex = 2; break;
1477  case MVT::v1i64: OpcodeIndex = 3; break;
1478    // Quad-register operations:
1479  case MVT::v16i8: OpcodeIndex = 0; break;
1480  case MVT::v8i16: OpcodeIndex = 1; break;
1481  case MVT::v4f32:
1482  case MVT::v4i32: OpcodeIndex = 2; break;
1483  case MVT::v2i64: OpcodeIndex = 3;
1484    assert(NumVecs == 1 && "v2i64 type only supported for VLD1");
1485    break;
1486  }
1487
1488  EVT ResTy;
1489  if (NumVecs == 1)
1490    ResTy = VT;
1491  else {
1492    unsigned ResTyElts = (NumVecs == 3) ? 4 : NumVecs;
1493    if (!is64BitVector)
1494      ResTyElts *= 2;
1495    ResTy = EVT::getVectorVT(*CurDAG->getContext(), MVT::i64, ResTyElts);
1496  }
1497
1498  SDValue Pred = getAL(CurDAG);
1499  SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
1500  SDValue SuperReg;
1501  if (is64BitVector) {
1502    unsigned Opc = DOpcodes[OpcodeIndex];
1503    const SDValue Ops[] = { MemAddr, Align, Pred, Reg0, Chain };
1504    SDNode *VLd = CurDAG->getMachineNode(Opc, dl, ResTy, MVT::Other, Ops, 5);
1505    if (NumVecs == 1)
1506      return VLd;
1507
1508    SuperReg = SDValue(VLd, 0);
1509    assert(ARM::dsub_7 == ARM::dsub_0+7 && "Unexpected subreg numbering");
1510    for (unsigned Vec = 0; Vec < NumVecs; ++Vec) {
1511      SDValue D = CurDAG->getTargetExtractSubreg(ARM::dsub_0+Vec,
1512                                                 dl, VT, SuperReg);
1513      ReplaceUses(SDValue(N, Vec), D);
1514    }
1515    ReplaceUses(SDValue(N, NumVecs), SDValue(VLd, 1));
1516    return NULL;
1517  }
1518
1519  if (NumVecs <= 2) {
1520    // Quad registers are directly supported for VLD1 and VLD2,
1521    // loading pairs of D regs.
1522    unsigned Opc = QOpcodes0[OpcodeIndex];
1523    const SDValue Ops[] = { MemAddr, Align, Pred, Reg0, Chain };
1524    SDNode *VLd = CurDAG->getMachineNode(Opc, dl, ResTy, MVT::Other, Ops, 5);
1525    if (NumVecs == 1)
1526      return VLd;
1527
1528    SuperReg = SDValue(VLd, 0);
1529    Chain = SDValue(VLd, 1);
1530
1531  } else {
1532    // Otherwise, quad registers are loaded with two separate instructions,
1533    // where one loads the even registers and the other loads the odd registers.
1534    EVT AddrTy = MemAddr.getValueType();
1535
1536    // Load the even subregs.
1537    unsigned Opc = QOpcodes0[OpcodeIndex];
1538    SDValue ImplDef =
1539      SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, dl, ResTy), 0);
1540    const SDValue OpsA[] = { MemAddr, Align, Reg0, ImplDef, Pred, Reg0, Chain };
1541    SDNode *VLdA =
1542      CurDAG->getMachineNode(Opc, dl, ResTy, AddrTy, MVT::Other, OpsA, 7);
1543    Chain = SDValue(VLdA, 2);
1544
1545    // Load the odd subregs.
1546    Opc = QOpcodes1[OpcodeIndex];
1547    const SDValue OpsB[] = { SDValue(VLdA, 1), Align, Reg0, SDValue(VLdA, 0),
1548                             Pred, Reg0, Chain };
1549    SDNode *VLdB =
1550      CurDAG->getMachineNode(Opc, dl, ResTy, AddrTy, MVT::Other, OpsB, 7);
1551    SuperReg = SDValue(VLdB, 0);
1552    Chain = SDValue(VLdB, 2);
1553  }
1554
1555  // Extract out the Q registers.
1556  assert(ARM::qsub_3 == ARM::qsub_0+3 && "Unexpected subreg numbering");
1557  for (unsigned Vec = 0; Vec < NumVecs; ++Vec) {
1558    SDValue Q = CurDAG->getTargetExtractSubreg(ARM::qsub_0+Vec,
1559                                               dl, VT, SuperReg);
1560    ReplaceUses(SDValue(N, Vec), Q);
1561  }
1562  ReplaceUses(SDValue(N, NumVecs), Chain);
1563  return NULL;
1564}
1565
1566SDNode *ARMDAGToDAGISel::SelectVST(SDNode *N, unsigned NumVecs,
1567                                   unsigned *DOpcodes, unsigned *QOpcodes0,
1568                                   unsigned *QOpcodes1) {
1569  assert(NumVecs >= 1 && NumVecs <= 4 && "VST NumVecs out-of-range");
1570  DebugLoc dl = N->getDebugLoc();
1571
1572  SDValue MemAddr, Align;
1573  if (!SelectAddrMode6(N, N->getOperand(2), MemAddr, Align))
1574    return NULL;
1575
1576  SDValue Chain = N->getOperand(0);
1577  EVT VT = N->getOperand(3).getValueType();
1578  bool is64BitVector = VT.is64BitVector();
1579  Align = GetVLDSTAlign(Align, NumVecs, is64BitVector);
1580
1581  unsigned OpcodeIndex;
1582  switch (VT.getSimpleVT().SimpleTy) {
1583  default: llvm_unreachable("unhandled vst type");
1584    // Double-register operations:
1585  case MVT::v8i8:  OpcodeIndex = 0; break;
1586  case MVT::v4i16: OpcodeIndex = 1; break;
1587  case MVT::v2f32:
1588  case MVT::v2i32: OpcodeIndex = 2; break;
1589  case MVT::v1i64: OpcodeIndex = 3; break;
1590    // Quad-register operations:
1591  case MVT::v16i8: OpcodeIndex = 0; break;
1592  case MVT::v8i16: OpcodeIndex = 1; break;
1593  case MVT::v4f32:
1594  case MVT::v4i32: OpcodeIndex = 2; break;
1595  case MVT::v2i64: OpcodeIndex = 3;
1596    assert(NumVecs == 1 && "v2i64 type only supported for VST1");
1597    break;
1598  }
1599
1600  SDValue Pred = getAL(CurDAG);
1601  SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
1602
1603  SmallVector<SDValue, 7> Ops;
1604  Ops.push_back(MemAddr);
1605  Ops.push_back(Align);
1606
1607  if (is64BitVector) {
1608    if (NumVecs == 1) {
1609      Ops.push_back(N->getOperand(3));
1610    } else {
1611      SDValue RegSeq;
1612      SDValue V0 = N->getOperand(0+3);
1613      SDValue V1 = N->getOperand(1+3);
1614
1615      // Form a REG_SEQUENCE to force register allocation.
1616      if (NumVecs == 2)
1617        RegSeq = SDValue(PairDRegs(MVT::v2i64, V0, V1), 0);
1618      else {
1619        SDValue V2 = N->getOperand(2+3);
1620        // If it's a vld3, form a quad D-register and leave the last part as
1621        // an undef.
1622        SDValue V3 = (NumVecs == 3)
1623          ? SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF,dl,VT), 0)
1624          : N->getOperand(3+3);
1625        RegSeq = SDValue(QuadDRegs(MVT::v4i64, V0, V1, V2, V3), 0);
1626      }
1627      Ops.push_back(RegSeq);
1628    }
1629    Ops.push_back(Pred);
1630    Ops.push_back(Reg0); // predicate register
1631    Ops.push_back(Chain);
1632    unsigned Opc = DOpcodes[OpcodeIndex];
1633    return CurDAG->getMachineNode(Opc, dl, MVT::Other, Ops.data(), 6);
1634  }
1635
1636  if (NumVecs <= 2) {
1637    // Quad registers are directly supported for VST1 and VST2.
1638    unsigned Opc = QOpcodes0[OpcodeIndex];
1639    if (NumVecs == 1) {
1640      Ops.push_back(N->getOperand(3));
1641    } else {
1642      // Form a QQ register.
1643      SDValue Q0 = N->getOperand(3);
1644      SDValue Q1 = N->getOperand(4);
1645      Ops.push_back(SDValue(PairQRegs(MVT::v4i64, Q0, Q1), 0));
1646    }
1647    Ops.push_back(Pred);
1648    Ops.push_back(Reg0); // predicate register
1649    Ops.push_back(Chain);
1650    return CurDAG->getMachineNode(Opc, dl, MVT::Other, Ops.data(), 6);
1651  }
1652
1653  // Otherwise, quad registers are stored with two separate instructions,
1654  // where one stores the even registers and the other stores the odd registers.
1655
1656  // Form the QQQQ REG_SEQUENCE.
1657  SDValue V0 = N->getOperand(0+3);
1658  SDValue V1 = N->getOperand(1+3);
1659  SDValue V2 = N->getOperand(2+3);
1660  SDValue V3 = (NumVecs == 3)
1661    ? SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, dl, VT), 0)
1662    : N->getOperand(3+3);
1663  SDValue RegSeq = SDValue(QuadQRegs(MVT::v8i64, V0, V1, V2, V3), 0);
1664
1665  // Store the even D registers.
1666  Ops.push_back(Reg0); // post-access address offset
1667  Ops.push_back(RegSeq);
1668  Ops.push_back(Pred);
1669  Ops.push_back(Reg0); // predicate register
1670  Ops.push_back(Chain);
1671  unsigned Opc = QOpcodes0[OpcodeIndex];
1672  SDNode *VStA = CurDAG->getMachineNode(Opc, dl, MemAddr.getValueType(),
1673                                        MVT::Other, Ops.data(), 7);
1674  Chain = SDValue(VStA, 1);
1675
1676  // Store the odd D registers.
1677  Ops[0] = SDValue(VStA, 0); // MemAddr
1678  Ops[6] = Chain;
1679  Opc = QOpcodes1[OpcodeIndex];
1680  SDNode *VStB = CurDAG->getMachineNode(Opc, dl, MemAddr.getValueType(),
1681                                        MVT::Other, Ops.data(), 7);
1682  Chain = SDValue(VStB, 1);
1683  ReplaceUses(SDValue(N, 0), Chain);
1684  return NULL;
1685}
1686
1687SDNode *ARMDAGToDAGISel::SelectVLDSTLane(SDNode *N, bool IsLoad,
1688                                         unsigned NumVecs, unsigned *DOpcodes,
1689                                         unsigned *QOpcodes) {
1690  assert(NumVecs >=2 && NumVecs <= 4 && "VLDSTLane NumVecs out-of-range");
1691  DebugLoc dl = N->getDebugLoc();
1692
1693  SDValue MemAddr, Align;
1694  if (!SelectAddrMode6(N, N->getOperand(2), MemAddr, Align))
1695    return NULL;
1696
1697  SDValue Chain = N->getOperand(0);
1698  unsigned Lane =
1699    cast<ConstantSDNode>(N->getOperand(NumVecs+3))->getZExtValue();
1700  EVT VT = IsLoad ? N->getValueType(0) : N->getOperand(3).getValueType();
1701  bool is64BitVector = VT.is64BitVector();
1702
1703  unsigned Alignment = 0;
1704  if (NumVecs != 3) {
1705    Alignment = cast<ConstantSDNode>(Align)->getZExtValue();
1706    unsigned NumBytes = NumVecs * VT.getVectorElementType().getSizeInBits()/8;
1707    if (Alignment > NumBytes)
1708      Alignment = NumBytes;
1709    if (Alignment < 8 && Alignment < NumBytes)
1710      Alignment = 0;
1711    // Alignment must be a power of two; make sure of that.
1712    Alignment = (Alignment & -Alignment);
1713    if (Alignment == 1)
1714      Alignment = 0;
1715  }
1716  Align = CurDAG->getTargetConstant(Alignment, MVT::i32);
1717
1718  unsigned OpcodeIndex;
1719  switch (VT.getSimpleVT().SimpleTy) {
1720  default: llvm_unreachable("unhandled vld/vst lane type");
1721    // Double-register operations:
1722  case MVT::v8i8:  OpcodeIndex = 0; break;
1723  case MVT::v4i16: OpcodeIndex = 1; break;
1724  case MVT::v2f32:
1725  case MVT::v2i32: OpcodeIndex = 2; break;
1726    // Quad-register operations:
1727  case MVT::v8i16: OpcodeIndex = 0; break;
1728  case MVT::v4f32:
1729  case MVT::v4i32: OpcodeIndex = 1; break;
1730  }
1731
1732  SDValue Pred = getAL(CurDAG);
1733  SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
1734
1735  SmallVector<SDValue, 7> Ops;
1736  Ops.push_back(MemAddr);
1737  Ops.push_back(Align);
1738
1739  unsigned Opc = (is64BitVector ? DOpcodes[OpcodeIndex] :
1740                                  QOpcodes[OpcodeIndex]);
1741
1742  SDValue SuperReg;
1743  SDValue V0 = N->getOperand(0+3);
1744  SDValue V1 = N->getOperand(1+3);
1745  if (NumVecs == 2) {
1746    if (is64BitVector)
1747      SuperReg = SDValue(PairDRegs(MVT::v2i64, V0, V1), 0);
1748    else
1749      SuperReg = SDValue(PairQRegs(MVT::v4i64, V0, V1), 0);
1750  } else {
1751    SDValue V2 = N->getOperand(2+3);
1752    SDValue V3 = (NumVecs == 3)
1753      ? SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF,dl,VT), 0)
1754      : N->getOperand(3+3);
1755    if (is64BitVector)
1756      SuperReg = SDValue(QuadDRegs(MVT::v4i64, V0, V1, V2, V3), 0);
1757    else
1758      SuperReg = SDValue(QuadQRegs(MVT::v8i64, V0, V1, V2, V3), 0);
1759  }
1760  Ops.push_back(SuperReg);
1761  Ops.push_back(getI32Imm(Lane));
1762  Ops.push_back(Pred);
1763  Ops.push_back(Reg0);
1764  Ops.push_back(Chain);
1765
1766  if (!IsLoad)
1767    return CurDAG->getMachineNode(Opc, dl, MVT::Other, Ops.data(), 7);
1768
1769  EVT ResTy;
1770  unsigned ResTyElts = (NumVecs == 3) ? 4 : NumVecs;
1771  if (!is64BitVector)
1772    ResTyElts *= 2;
1773  ResTy = EVT::getVectorVT(*CurDAG->getContext(), MVT::i64, ResTyElts);
1774
1775  SDNode *VLdLn = CurDAG->getMachineNode(Opc, dl, ResTy, MVT::Other,
1776                                         Ops.data(), 7);
1777  SuperReg = SDValue(VLdLn, 0);
1778  Chain = SDValue(VLdLn, 1);
1779
1780  // Extract the subregisters.
1781  assert(ARM::dsub_7 == ARM::dsub_0+7 && "Unexpected subreg numbering");
1782  assert(ARM::qsub_3 == ARM::qsub_0+3 && "Unexpected subreg numbering");
1783  unsigned SubIdx = is64BitVector ? ARM::dsub_0 : ARM::qsub_0;
1784  for (unsigned Vec = 0; Vec < NumVecs; ++Vec)
1785    ReplaceUses(SDValue(N, Vec),
1786                CurDAG->getTargetExtractSubreg(SubIdx+Vec, dl, VT, SuperReg));
1787  ReplaceUses(SDValue(N, NumVecs), Chain);
1788  return NULL;
1789}
1790
1791SDNode *ARMDAGToDAGISel::SelectVLDDup(SDNode *N, unsigned NumVecs,
1792                                      unsigned *Opcodes) {
1793  assert(NumVecs >=2 && NumVecs <= 4 && "VLDDup NumVecs out-of-range");
1794  DebugLoc dl = N->getDebugLoc();
1795
1796  SDValue MemAddr, Align;
1797  if (!SelectAddrMode6(N, N->getOperand(1), MemAddr, Align))
1798    return NULL;
1799
1800  SDValue Chain = N->getOperand(0);
1801  EVT VT = N->getValueType(0);
1802
1803  unsigned Alignment = 0;
1804  if (NumVecs != 3) {
1805    Alignment = cast<ConstantSDNode>(Align)->getZExtValue();
1806    unsigned NumBytes = NumVecs * VT.getVectorElementType().getSizeInBits()/8;
1807    if (Alignment > NumBytes)
1808      Alignment = NumBytes;
1809    if (Alignment < 8 && Alignment < NumBytes)
1810      Alignment = 0;
1811    // Alignment must be a power of two; make sure of that.
1812    Alignment = (Alignment & -Alignment);
1813    if (Alignment == 1)
1814      Alignment = 0;
1815  }
1816  Align = CurDAG->getTargetConstant(Alignment, MVT::i32);
1817
1818  unsigned OpcodeIndex;
1819  switch (VT.getSimpleVT().SimpleTy) {
1820  default: llvm_unreachable("unhandled vld-dup type");
1821  case MVT::v8i8:  OpcodeIndex = 0; break;
1822  case MVT::v4i16: OpcodeIndex = 1; break;
1823  case MVT::v2f32:
1824  case MVT::v2i32: OpcodeIndex = 2; break;
1825  }
1826
1827  SDValue Pred = getAL(CurDAG);
1828  SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
1829  SDValue SuperReg;
1830  unsigned Opc = Opcodes[OpcodeIndex];
1831  const SDValue Ops[] = { MemAddr, Align, Pred, Reg0, Chain };
1832
1833  unsigned ResTyElts = (NumVecs == 3) ? 4 : NumVecs;
1834  EVT ResTy = EVT::getVectorVT(*CurDAG->getContext(), MVT::i64, ResTyElts);
1835  SDNode *VLdDup = CurDAG->getMachineNode(Opc, dl, ResTy, MVT::Other, Ops, 5);
1836  SuperReg = SDValue(VLdDup, 0);
1837  Chain = SDValue(VLdDup, 1);
1838
1839  // Extract the subregisters.
1840  assert(ARM::dsub_7 == ARM::dsub_0+7 && "Unexpected subreg numbering");
1841  unsigned SubIdx = ARM::dsub_0;
1842  for (unsigned Vec = 0; Vec < NumVecs; ++Vec)
1843    ReplaceUses(SDValue(N, Vec),
1844                CurDAG->getTargetExtractSubreg(SubIdx+Vec, dl, VT, SuperReg));
1845  ReplaceUses(SDValue(N, NumVecs), Chain);
1846  return NULL;
1847}
1848
1849SDNode *ARMDAGToDAGISel::SelectVTBL(SDNode *N, bool IsExt, unsigned NumVecs,
1850                                    unsigned Opc) {
1851  assert(NumVecs >= 2 && NumVecs <= 4 && "VTBL NumVecs out-of-range");
1852  DebugLoc dl = N->getDebugLoc();
1853  EVT VT = N->getValueType(0);
1854  unsigned FirstTblReg = IsExt ? 2 : 1;
1855
1856  // Form a REG_SEQUENCE to force register allocation.
1857  SDValue RegSeq;
1858  SDValue V0 = N->getOperand(FirstTblReg + 0);
1859  SDValue V1 = N->getOperand(FirstTblReg + 1);
1860  if (NumVecs == 2)
1861    RegSeq = SDValue(PairDRegs(MVT::v16i8, V0, V1), 0);
1862  else {
1863    SDValue V2 = N->getOperand(FirstTblReg + 2);
1864    // If it's a vtbl3, form a quad D-register and leave the last part as
1865    // an undef.
1866    SDValue V3 = (NumVecs == 3)
1867      ? SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, dl, VT), 0)
1868      : N->getOperand(FirstTblReg + 3);
1869    RegSeq = SDValue(QuadDRegs(MVT::v4i64, V0, V1, V2, V3), 0);
1870  }
1871
1872  SmallVector<SDValue, 6> Ops;
1873  if (IsExt)
1874    Ops.push_back(N->getOperand(1));
1875  Ops.push_back(RegSeq);
1876  Ops.push_back(N->getOperand(FirstTblReg + NumVecs));
1877  Ops.push_back(getAL(CurDAG)); // predicate
1878  Ops.push_back(CurDAG->getRegister(0, MVT::i32)); // predicate register
1879  return CurDAG->getMachineNode(Opc, dl, VT, Ops.data(), Ops.size());
1880}
1881
1882SDNode *ARMDAGToDAGISel::SelectV6T2BitfieldExtractOp(SDNode *N,
1883                                                     bool isSigned) {
1884  if (!Subtarget->hasV6T2Ops())
1885    return NULL;
1886
1887  unsigned Opc = isSigned ? (Subtarget->isThumb() ? ARM::t2SBFX : ARM::SBFX)
1888    : (Subtarget->isThumb() ? ARM::t2UBFX : ARM::UBFX);
1889
1890
1891  // For unsigned extracts, check for a shift right and mask
1892  unsigned And_imm = 0;
1893  if (N->getOpcode() == ISD::AND) {
1894    if (isOpcWithIntImmediate(N, ISD::AND, And_imm)) {
1895
1896      // The immediate is a mask of the low bits iff imm & (imm+1) == 0
1897      if (And_imm & (And_imm + 1))
1898        return NULL;
1899
1900      unsigned Srl_imm = 0;
1901      if (isOpcWithIntImmediate(N->getOperand(0).getNode(), ISD::SRL,
1902                                Srl_imm)) {
1903        assert(Srl_imm > 0 && Srl_imm < 32 && "bad amount in shift node!");
1904
1905        unsigned Width = CountTrailingOnes_32(And_imm);
1906        unsigned LSB = Srl_imm;
1907        SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
1908        SDValue Ops[] = { N->getOperand(0).getOperand(0),
1909                          CurDAG->getTargetConstant(LSB, MVT::i32),
1910                          CurDAG->getTargetConstant(Width, MVT::i32),
1911          getAL(CurDAG), Reg0 };
1912        return CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops, 5);
1913      }
1914    }
1915    return NULL;
1916  }
1917
1918  // Otherwise, we're looking for a shift of a shift
1919  unsigned Shl_imm = 0;
1920  if (isOpcWithIntImmediate(N->getOperand(0).getNode(), ISD::SHL, Shl_imm)) {
1921    assert(Shl_imm > 0 && Shl_imm < 32 && "bad amount in shift node!");
1922    unsigned Srl_imm = 0;
1923    if (isInt32Immediate(N->getOperand(1), Srl_imm)) {
1924      assert(Srl_imm > 0 && Srl_imm < 32 && "bad amount in shift node!");
1925      unsigned Width = 32 - Srl_imm;
1926      int LSB = Srl_imm - Shl_imm;
1927      if (LSB < 0)
1928        return NULL;
1929      SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
1930      SDValue Ops[] = { N->getOperand(0).getOperand(0),
1931                        CurDAG->getTargetConstant(LSB, MVT::i32),
1932                        CurDAG->getTargetConstant(Width, MVT::i32),
1933                        getAL(CurDAG), Reg0 };
1934      return CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops, 5);
1935    }
1936  }
1937  return NULL;
1938}
1939
1940SDNode *ARMDAGToDAGISel::
1941SelectT2CMOVShiftOp(SDNode *N, SDValue FalseVal, SDValue TrueVal,
1942                    ARMCC::CondCodes CCVal, SDValue CCR, SDValue InFlag) {
1943  SDValue CPTmp0;
1944  SDValue CPTmp1;
1945  if (SelectT2ShifterOperandReg(TrueVal, CPTmp0, CPTmp1)) {
1946    unsigned SOVal = cast<ConstantSDNode>(CPTmp1)->getZExtValue();
1947    unsigned SOShOp = ARM_AM::getSORegShOp(SOVal);
1948    unsigned Opc = 0;
1949    switch (SOShOp) {
1950    case ARM_AM::lsl: Opc = ARM::t2MOVCClsl; break;
1951    case ARM_AM::lsr: Opc = ARM::t2MOVCClsr; break;
1952    case ARM_AM::asr: Opc = ARM::t2MOVCCasr; break;
1953    case ARM_AM::ror: Opc = ARM::t2MOVCCror; break;
1954    default:
1955      llvm_unreachable("Unknown so_reg opcode!");
1956      break;
1957    }
1958    SDValue SOShImm =
1959      CurDAG->getTargetConstant(ARM_AM::getSORegOffset(SOVal), MVT::i32);
1960    SDValue CC = CurDAG->getTargetConstant(CCVal, MVT::i32);
1961    SDValue Ops[] = { FalseVal, CPTmp0, SOShImm, CC, CCR, InFlag };
1962    return CurDAG->SelectNodeTo(N, Opc, MVT::i32,Ops, 6);
1963  }
1964  return 0;
1965}
1966
1967SDNode *ARMDAGToDAGISel::
1968SelectARMCMOVShiftOp(SDNode *N, SDValue FalseVal, SDValue TrueVal,
1969                     ARMCC::CondCodes CCVal, SDValue CCR, SDValue InFlag) {
1970  SDValue CPTmp0;
1971  SDValue CPTmp1;
1972  SDValue CPTmp2;
1973  if (SelectShifterOperandReg(TrueVal, CPTmp0, CPTmp1, CPTmp2)) {
1974    SDValue CC = CurDAG->getTargetConstant(CCVal, MVT::i32);
1975    SDValue Ops[] = { FalseVal, CPTmp0, CPTmp1, CPTmp2, CC, CCR, InFlag };
1976    return CurDAG->SelectNodeTo(N, ARM::MOVCCs, MVT::i32, Ops, 7);
1977  }
1978  return 0;
1979}
1980
1981SDNode *ARMDAGToDAGISel::
1982SelectT2CMOVImmOp(SDNode *N, SDValue FalseVal, SDValue TrueVal,
1983                  ARMCC::CondCodes CCVal, SDValue CCR, SDValue InFlag) {
1984  ConstantSDNode *T = dyn_cast<ConstantSDNode>(TrueVal);
1985  if (!T)
1986    return 0;
1987
1988  unsigned Opc = 0;
1989  unsigned TrueImm = T->getZExtValue();
1990  if (is_t2_so_imm(TrueImm)) {
1991    Opc = ARM::t2MOVCCi;
1992  } else if (TrueImm <= 0xffff) {
1993    Opc = ARM::t2MOVCCi16;
1994  } else if (is_t2_so_imm_not(TrueImm)) {
1995    TrueImm = ~TrueImm;
1996    Opc = ARM::t2MVNCCi;
1997  } else if (TrueVal.getNode()->hasOneUse() && Subtarget->hasV6T2Ops()) {
1998    // Large immediate.
1999    Opc = ARM::t2MOVCCi32imm;
2000  }
2001
2002  if (Opc) {
2003    SDValue True = CurDAG->getTargetConstant(TrueImm, MVT::i32);
2004    SDValue CC = CurDAG->getTargetConstant(CCVal, MVT::i32);
2005    SDValue Ops[] = { FalseVal, True, CC, CCR, InFlag };
2006    return CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops, 5);
2007  }
2008
2009  return 0;
2010}
2011
2012SDNode *ARMDAGToDAGISel::
2013SelectARMCMOVImmOp(SDNode *N, SDValue FalseVal, SDValue TrueVal,
2014                   ARMCC::CondCodes CCVal, SDValue CCR, SDValue InFlag) {
2015  ConstantSDNode *T = dyn_cast<ConstantSDNode>(TrueVal);
2016  if (!T)
2017    return 0;
2018
2019  unsigned Opc = 0;
2020  unsigned TrueImm = T->getZExtValue();
2021  bool isSoImm = is_so_imm(TrueImm);
2022  if (isSoImm) {
2023    Opc = ARM::MOVCCi;
2024  } else if (Subtarget->hasV6T2Ops() && TrueImm <= 0xffff) {
2025    Opc = ARM::MOVCCi16;
2026  } else if (is_so_imm_not(TrueImm)) {
2027    TrueImm = ~TrueImm;
2028    Opc = ARM::MVNCCi;
2029  } else if (TrueVal.getNode()->hasOneUse() &&
2030             (Subtarget->hasV6T2Ops() || ARM_AM::isSOImmTwoPartVal(TrueImm))) {
2031    // Large immediate.
2032    Opc = ARM::MOVCCi32imm;
2033  }
2034
2035  if (Opc) {
2036    SDValue True = CurDAG->getTargetConstant(TrueImm, MVT::i32);
2037    SDValue CC = CurDAG->getTargetConstant(CCVal, MVT::i32);
2038    SDValue Ops[] = { FalseVal, True, CC, CCR, InFlag };
2039    return CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops, 5);
2040  }
2041
2042  return 0;
2043}
2044
2045SDNode *ARMDAGToDAGISel::SelectCMOVOp(SDNode *N) {
2046  EVT VT = N->getValueType(0);
2047  SDValue FalseVal = N->getOperand(0);
2048  SDValue TrueVal  = N->getOperand(1);
2049  SDValue CC = N->getOperand(2);
2050  SDValue CCR = N->getOperand(3);
2051  SDValue InFlag = N->getOperand(4);
2052  assert(CC.getOpcode() == ISD::Constant);
2053  assert(CCR.getOpcode() == ISD::Register);
2054  ARMCC::CondCodes CCVal =
2055    (ARMCC::CondCodes)cast<ConstantSDNode>(CC)->getZExtValue();
2056
2057  if (!Subtarget->isThumb1Only() && VT == MVT::i32) {
2058    // Pattern: (ARMcmov:i32 GPR:i32:$false, so_reg:i32:$true, (imm:i32):$cc)
2059    // Emits: (MOVCCs:i32 GPR:i32:$false, so_reg:i32:$true, (imm:i32):$cc)
2060    // Pattern complexity = 18  cost = 1  size = 0
2061    SDValue CPTmp0;
2062    SDValue CPTmp1;
2063    SDValue CPTmp2;
2064    if (Subtarget->isThumb()) {
2065      SDNode *Res = SelectT2CMOVShiftOp(N, FalseVal, TrueVal,
2066                                        CCVal, CCR, InFlag);
2067      if (!Res)
2068        Res = SelectT2CMOVShiftOp(N, TrueVal, FalseVal,
2069                               ARMCC::getOppositeCondition(CCVal), CCR, InFlag);
2070      if (Res)
2071        return Res;
2072    } else {
2073      SDNode *Res = SelectARMCMOVShiftOp(N, FalseVal, TrueVal,
2074                                         CCVal, CCR, InFlag);
2075      if (!Res)
2076        Res = SelectARMCMOVShiftOp(N, TrueVal, FalseVal,
2077                               ARMCC::getOppositeCondition(CCVal), CCR, InFlag);
2078      if (Res)
2079        return Res;
2080    }
2081
2082    // Pattern: (ARMcmov:i32 GPR:i32:$false,
2083    //             (imm:i32)<<P:Pred_so_imm>>:$true,
2084    //             (imm:i32):$cc)
2085    // Emits: (MOVCCi:i32 GPR:i32:$false,
2086    //           (so_imm:i32 (imm:i32):$true), (imm:i32):$cc)
2087    // Pattern complexity = 10  cost = 1  size = 0
2088    if (Subtarget->isThumb()) {
2089      SDNode *Res = SelectT2CMOVImmOp(N, FalseVal, TrueVal,
2090                                        CCVal, CCR, InFlag);
2091      if (!Res)
2092        Res = SelectT2CMOVImmOp(N, TrueVal, FalseVal,
2093                               ARMCC::getOppositeCondition(CCVal), CCR, InFlag);
2094      if (Res)
2095        return Res;
2096    } else {
2097      SDNode *Res = SelectARMCMOVImmOp(N, FalseVal, TrueVal,
2098                                         CCVal, CCR, InFlag);
2099      if (!Res)
2100        Res = SelectARMCMOVImmOp(N, TrueVal, FalseVal,
2101                               ARMCC::getOppositeCondition(CCVal), CCR, InFlag);
2102      if (Res)
2103        return Res;
2104    }
2105  }
2106
2107  // Pattern: (ARMcmov:i32 GPR:i32:$false, GPR:i32:$true, (imm:i32):$cc)
2108  // Emits: (MOVCCr:i32 GPR:i32:$false, GPR:i32:$true, (imm:i32):$cc)
2109  // Pattern complexity = 6  cost = 1  size = 0
2110  //
2111  // Pattern: (ARMcmov:i32 GPR:i32:$false, GPR:i32:$true, (imm:i32):$cc)
2112  // Emits: (tMOVCCr:i32 GPR:i32:$false, GPR:i32:$true, (imm:i32):$cc)
2113  // Pattern complexity = 6  cost = 11  size = 0
2114  //
2115  // Also FCPYScc and FCPYDcc.
2116  SDValue Tmp2 = CurDAG->getTargetConstant(CCVal, MVT::i32);
2117  SDValue Ops[] = { FalseVal, TrueVal, Tmp2, CCR, InFlag };
2118  unsigned Opc = 0;
2119  switch (VT.getSimpleVT().SimpleTy) {
2120  default: assert(false && "Illegal conditional move type!");
2121    break;
2122  case MVT::i32:
2123    Opc = Subtarget->isThumb()
2124      ? (Subtarget->hasThumb2() ? ARM::t2MOVCCr : ARM::tMOVCCr_pseudo)
2125      : ARM::MOVCCr;
2126    break;
2127  case MVT::f32:
2128    Opc = ARM::VMOVScc;
2129    break;
2130  case MVT::f64:
2131    Opc = ARM::VMOVDcc;
2132    break;
2133  }
2134  return CurDAG->SelectNodeTo(N, Opc, VT, Ops, 5);
2135}
2136
2137SDNode *ARMDAGToDAGISel::SelectConcatVector(SDNode *N) {
2138  // The only time a CONCAT_VECTORS operation can have legal types is when
2139  // two 64-bit vectors are concatenated to a 128-bit vector.
2140  EVT VT = N->getValueType(0);
2141  if (!VT.is128BitVector() || N->getNumOperands() != 2)
2142    llvm_unreachable("unexpected CONCAT_VECTORS");
2143  return PairDRegs(VT, N->getOperand(0), N->getOperand(1));
2144}
2145
2146SDNode *ARMDAGToDAGISel::Select(SDNode *N) {
2147  DebugLoc dl = N->getDebugLoc();
2148
2149  if (N->isMachineOpcode())
2150    return NULL;   // Already selected.
2151
2152  switch (N->getOpcode()) {
2153  default: break;
2154  case ISD::Constant: {
2155    unsigned Val = cast<ConstantSDNode>(N)->getZExtValue();
2156    bool UseCP = true;
2157    if (Subtarget->hasThumb2())
2158      // Thumb2-aware targets have the MOVT instruction, so all immediates can
2159      // be done with MOV + MOVT, at worst.
2160      UseCP = 0;
2161    else {
2162      if (Subtarget->isThumb()) {
2163        UseCP = (Val > 255 &&                          // MOV
2164                 ~Val > 255 &&                         // MOV + MVN
2165                 !ARM_AM::isThumbImmShiftedVal(Val));  // MOV + LSL
2166      } else
2167        UseCP = (ARM_AM::getSOImmVal(Val) == -1 &&     // MOV
2168                 ARM_AM::getSOImmVal(~Val) == -1 &&    // MVN
2169                 !ARM_AM::isSOImmTwoPartVal(Val));     // two instrs.
2170    }
2171
2172    if (UseCP) {
2173      SDValue CPIdx =
2174        CurDAG->getTargetConstantPool(ConstantInt::get(
2175                                  Type::getInt32Ty(*CurDAG->getContext()), Val),
2176                                      TLI.getPointerTy());
2177
2178      SDNode *ResNode;
2179      if (Subtarget->isThumb1Only()) {
2180        SDValue Pred = getAL(CurDAG);
2181        SDValue PredReg = CurDAG->getRegister(0, MVT::i32);
2182        SDValue Ops[] = { CPIdx, Pred, PredReg, CurDAG->getEntryNode() };
2183        ResNode = CurDAG->getMachineNode(ARM::tLDRpci, dl, MVT::i32, MVT::Other,
2184                                         Ops, 4);
2185      } else {
2186        SDValue Ops[] = {
2187          CPIdx,
2188          CurDAG->getTargetConstant(0, MVT::i32),
2189          getAL(CurDAG),
2190          CurDAG->getRegister(0, MVT::i32),
2191          CurDAG->getEntryNode()
2192        };
2193        ResNode=CurDAG->getMachineNode(ARM::LDRcp, dl, MVT::i32, MVT::Other,
2194                                       Ops, 5);
2195      }
2196      ReplaceUses(SDValue(N, 0), SDValue(ResNode, 0));
2197      return NULL;
2198    }
2199
2200    // Other cases are autogenerated.
2201    break;
2202  }
2203  case ISD::FrameIndex: {
2204    // Selects to ADDri FI, 0 which in turn will become ADDri SP, imm.
2205    int FI = cast<FrameIndexSDNode>(N)->getIndex();
2206    SDValue TFI = CurDAG->getTargetFrameIndex(FI, TLI.getPointerTy());
2207    if (Subtarget->isThumb1Only()) {
2208      return CurDAG->SelectNodeTo(N, ARM::tADDrSPi, MVT::i32, TFI,
2209                                  CurDAG->getTargetConstant(0, MVT::i32));
2210    } else {
2211      unsigned Opc = ((Subtarget->isThumb() && Subtarget->hasThumb2()) ?
2212                      ARM::t2ADDri : ARM::ADDri);
2213      SDValue Ops[] = { TFI, CurDAG->getTargetConstant(0, MVT::i32),
2214                        getAL(CurDAG), CurDAG->getRegister(0, MVT::i32),
2215                        CurDAG->getRegister(0, MVT::i32) };
2216      return CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops, 5);
2217    }
2218  }
2219  case ISD::SRL:
2220    if (SDNode *I = SelectV6T2BitfieldExtractOp(N, false))
2221      return I;
2222    break;
2223  case ISD::SRA:
2224    if (SDNode *I = SelectV6T2BitfieldExtractOp(N, true))
2225      return I;
2226    break;
2227  case ISD::MUL:
2228    if (Subtarget->isThumb1Only())
2229      break;
2230    if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(N->getOperand(1))) {
2231      unsigned RHSV = C->getZExtValue();
2232      if (!RHSV) break;
2233      if (isPowerOf2_32(RHSV-1)) {  // 2^n+1?
2234        unsigned ShImm = Log2_32(RHSV-1);
2235        if (ShImm >= 32)
2236          break;
2237        SDValue V = N->getOperand(0);
2238        ShImm = ARM_AM::getSORegOpc(ARM_AM::lsl, ShImm);
2239        SDValue ShImmOp = CurDAG->getTargetConstant(ShImm, MVT::i32);
2240        SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
2241        if (Subtarget->isThumb()) {
2242          SDValue Ops[] = { V, V, ShImmOp, getAL(CurDAG), Reg0, Reg0 };
2243          return CurDAG->SelectNodeTo(N, ARM::t2ADDrs, MVT::i32, Ops, 6);
2244        } else {
2245          SDValue Ops[] = { V, V, Reg0, ShImmOp, getAL(CurDAG), Reg0, Reg0 };
2246          return CurDAG->SelectNodeTo(N, ARM::ADDrs, MVT::i32, Ops, 7);
2247        }
2248      }
2249      if (isPowerOf2_32(RHSV+1)) {  // 2^n-1?
2250        unsigned ShImm = Log2_32(RHSV+1);
2251        if (ShImm >= 32)
2252          break;
2253        SDValue V = N->getOperand(0);
2254        ShImm = ARM_AM::getSORegOpc(ARM_AM::lsl, ShImm);
2255        SDValue ShImmOp = CurDAG->getTargetConstant(ShImm, MVT::i32);
2256        SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
2257        if (Subtarget->isThumb()) {
2258          SDValue Ops[] = { V, V, ShImmOp, getAL(CurDAG), Reg0, Reg0 };
2259          return CurDAG->SelectNodeTo(N, ARM::t2RSBrs, MVT::i32, Ops, 6);
2260        } else {
2261          SDValue Ops[] = { V, V, Reg0, ShImmOp, getAL(CurDAG), Reg0, Reg0 };
2262          return CurDAG->SelectNodeTo(N, ARM::RSBrs, MVT::i32, Ops, 7);
2263        }
2264      }
2265    }
2266    break;
2267  case ISD::AND: {
2268    // Check for unsigned bitfield extract
2269    if (SDNode *I = SelectV6T2BitfieldExtractOp(N, false))
2270      return I;
2271
2272    // (and (or x, c2), c1) and top 16-bits of c1 and c2 match, lower 16-bits
2273    // of c1 are 0xffff, and lower 16-bit of c2 are 0. That is, the top 16-bits
2274    // are entirely contributed by c2 and lower 16-bits are entirely contributed
2275    // by x. That's equal to (or (and x, 0xffff), (and c1, 0xffff0000)).
2276    // Select it to: "movt x, ((c1 & 0xffff) >> 16)
2277    EVT VT = N->getValueType(0);
2278    if (VT != MVT::i32)
2279      break;
2280    unsigned Opc = (Subtarget->isThumb() && Subtarget->hasThumb2())
2281      ? ARM::t2MOVTi16
2282      : (Subtarget->hasV6T2Ops() ? ARM::MOVTi16 : 0);
2283    if (!Opc)
2284      break;
2285    SDValue N0 = N->getOperand(0), N1 = N->getOperand(1);
2286    ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
2287    if (!N1C)
2288      break;
2289    if (N0.getOpcode() == ISD::OR && N0.getNode()->hasOneUse()) {
2290      SDValue N2 = N0.getOperand(1);
2291      ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(N2);
2292      if (!N2C)
2293        break;
2294      unsigned N1CVal = N1C->getZExtValue();
2295      unsigned N2CVal = N2C->getZExtValue();
2296      if ((N1CVal & 0xffff0000U) == (N2CVal & 0xffff0000U) &&
2297          (N1CVal & 0xffffU) == 0xffffU &&
2298          (N2CVal & 0xffffU) == 0x0U) {
2299        SDValue Imm16 = CurDAG->getTargetConstant((N2CVal & 0xFFFF0000U) >> 16,
2300                                                  MVT::i32);
2301        SDValue Ops[] = { N0.getOperand(0), Imm16,
2302                          getAL(CurDAG), CurDAG->getRegister(0, MVT::i32) };
2303        return CurDAG->getMachineNode(Opc, dl, VT, Ops, 4);
2304      }
2305    }
2306    break;
2307  }
2308  case ARMISD::VMOVRRD:
2309    return CurDAG->getMachineNode(ARM::VMOVRRD, dl, MVT::i32, MVT::i32,
2310                                  N->getOperand(0), getAL(CurDAG),
2311                                  CurDAG->getRegister(0, MVT::i32));
2312  case ISD::UMUL_LOHI: {
2313    if (Subtarget->isThumb1Only())
2314      break;
2315    if (Subtarget->isThumb()) {
2316      SDValue Ops[] = { N->getOperand(0), N->getOperand(1),
2317                        getAL(CurDAG), CurDAG->getRegister(0, MVT::i32),
2318                        CurDAG->getRegister(0, MVT::i32) };
2319      return CurDAG->getMachineNode(ARM::t2UMULL, dl, MVT::i32, MVT::i32,Ops,4);
2320    } else {
2321      SDValue Ops[] = { N->getOperand(0), N->getOperand(1),
2322                        getAL(CurDAG), CurDAG->getRegister(0, MVT::i32),
2323                        CurDAG->getRegister(0, MVT::i32) };
2324      return CurDAG->getMachineNode(Subtarget->hasV6Ops() ?
2325                                    ARM::UMULL : ARM::UMULLv5,
2326                                    dl, MVT::i32, MVT::i32, Ops, 5);
2327    }
2328  }
2329  case ISD::SMUL_LOHI: {
2330    if (Subtarget->isThumb1Only())
2331      break;
2332    if (Subtarget->isThumb()) {
2333      SDValue Ops[] = { N->getOperand(0), N->getOperand(1),
2334                        getAL(CurDAG), CurDAG->getRegister(0, MVT::i32) };
2335      return CurDAG->getMachineNode(ARM::t2SMULL, dl, MVT::i32, MVT::i32,Ops,4);
2336    } else {
2337      SDValue Ops[] = { N->getOperand(0), N->getOperand(1),
2338                        getAL(CurDAG), CurDAG->getRegister(0, MVT::i32),
2339                        CurDAG->getRegister(0, MVT::i32) };
2340      return CurDAG->getMachineNode(Subtarget->hasV6Ops() ?
2341                                    ARM::SMULL : ARM::SMULLv5,
2342                                    dl, MVT::i32, MVT::i32, Ops, 5);
2343    }
2344  }
2345  case ISD::LOAD: {
2346    SDNode *ResNode = 0;
2347    if (Subtarget->isThumb() && Subtarget->hasThumb2())
2348      ResNode = SelectT2IndexedLoad(N);
2349    else
2350      ResNode = SelectARMIndexedLoad(N);
2351    if (ResNode)
2352      return ResNode;
2353    // Other cases are autogenerated.
2354    break;
2355  }
2356  case ARMISD::BRCOND: {
2357    // Pattern: (ARMbrcond:void (bb:Other):$dst, (imm:i32):$cc)
2358    // Emits: (Bcc:void (bb:Other):$dst, (imm:i32):$cc)
2359    // Pattern complexity = 6  cost = 1  size = 0
2360
2361    // Pattern: (ARMbrcond:void (bb:Other):$dst, (imm:i32):$cc)
2362    // Emits: (tBcc:void (bb:Other):$dst, (imm:i32):$cc)
2363    // Pattern complexity = 6  cost = 1  size = 0
2364
2365    // Pattern: (ARMbrcond:void (bb:Other):$dst, (imm:i32):$cc)
2366    // Emits: (t2Bcc:void (bb:Other):$dst, (imm:i32):$cc)
2367    // Pattern complexity = 6  cost = 1  size = 0
2368
2369    unsigned Opc = Subtarget->isThumb() ?
2370      ((Subtarget->hasThumb2()) ? ARM::t2Bcc : ARM::tBcc) : ARM::Bcc;
2371    SDValue Chain = N->getOperand(0);
2372    SDValue N1 = N->getOperand(1);
2373    SDValue N2 = N->getOperand(2);
2374    SDValue N3 = N->getOperand(3);
2375    SDValue InFlag = N->getOperand(4);
2376    assert(N1.getOpcode() == ISD::BasicBlock);
2377    assert(N2.getOpcode() == ISD::Constant);
2378    assert(N3.getOpcode() == ISD::Register);
2379
2380    SDValue Tmp2 = CurDAG->getTargetConstant(((unsigned)
2381                               cast<ConstantSDNode>(N2)->getZExtValue()),
2382                               MVT::i32);
2383    SDValue Ops[] = { N1, Tmp2, N3, Chain, InFlag };
2384    SDNode *ResNode = CurDAG->getMachineNode(Opc, dl, MVT::Other,
2385                                             MVT::Glue, Ops, 5);
2386    Chain = SDValue(ResNode, 0);
2387    if (N->getNumValues() == 2) {
2388      InFlag = SDValue(ResNode, 1);
2389      ReplaceUses(SDValue(N, 1), InFlag);
2390    }
2391    ReplaceUses(SDValue(N, 0),
2392                SDValue(Chain.getNode(), Chain.getResNo()));
2393    return NULL;
2394  }
2395  case ARMISD::CMOV:
2396    return SelectCMOVOp(N);
2397  case ARMISD::CNEG: {
2398    EVT VT = N->getValueType(0);
2399    SDValue N0 = N->getOperand(0);
2400    SDValue N1 = N->getOperand(1);
2401    SDValue N2 = N->getOperand(2);
2402    SDValue N3 = N->getOperand(3);
2403    SDValue InFlag = N->getOperand(4);
2404    assert(N2.getOpcode() == ISD::Constant);
2405    assert(N3.getOpcode() == ISD::Register);
2406
2407    SDValue Tmp2 = CurDAG->getTargetConstant(((unsigned)
2408                               cast<ConstantSDNode>(N2)->getZExtValue()),
2409                               MVT::i32);
2410    SDValue Ops[] = { N0, N1, Tmp2, N3, InFlag };
2411    unsigned Opc = 0;
2412    switch (VT.getSimpleVT().SimpleTy) {
2413    default: assert(false && "Illegal conditional move type!");
2414      break;
2415    case MVT::f32:
2416      Opc = ARM::VNEGScc;
2417      break;
2418    case MVT::f64:
2419      Opc = ARM::VNEGDcc;
2420      break;
2421    }
2422    return CurDAG->SelectNodeTo(N, Opc, VT, Ops, 5);
2423  }
2424
2425  case ARMISD::VZIP: {
2426    unsigned Opc = 0;
2427    EVT VT = N->getValueType(0);
2428    switch (VT.getSimpleVT().SimpleTy) {
2429    default: return NULL;
2430    case MVT::v8i8:  Opc = ARM::VZIPd8; break;
2431    case MVT::v4i16: Opc = ARM::VZIPd16; break;
2432    case MVT::v2f32:
2433    case MVT::v2i32: Opc = ARM::VZIPd32; break;
2434    case MVT::v16i8: Opc = ARM::VZIPq8; break;
2435    case MVT::v8i16: Opc = ARM::VZIPq16; break;
2436    case MVT::v4f32:
2437    case MVT::v4i32: Opc = ARM::VZIPq32; break;
2438    }
2439    SDValue Pred = getAL(CurDAG);
2440    SDValue PredReg = CurDAG->getRegister(0, MVT::i32);
2441    SDValue Ops[] = { N->getOperand(0), N->getOperand(1), Pred, PredReg };
2442    return CurDAG->getMachineNode(Opc, dl, VT, VT, Ops, 4);
2443  }
2444  case ARMISD::VUZP: {
2445    unsigned Opc = 0;
2446    EVT VT = N->getValueType(0);
2447    switch (VT.getSimpleVT().SimpleTy) {
2448    default: return NULL;
2449    case MVT::v8i8:  Opc = ARM::VUZPd8; break;
2450    case MVT::v4i16: Opc = ARM::VUZPd16; break;
2451    case MVT::v2f32:
2452    case MVT::v2i32: Opc = ARM::VUZPd32; break;
2453    case MVT::v16i8: Opc = ARM::VUZPq8; break;
2454    case MVT::v8i16: Opc = ARM::VUZPq16; break;
2455    case MVT::v4f32:
2456    case MVT::v4i32: Opc = ARM::VUZPq32; break;
2457    }
2458    SDValue Pred = getAL(CurDAG);
2459    SDValue PredReg = CurDAG->getRegister(0, MVT::i32);
2460    SDValue Ops[] = { N->getOperand(0), N->getOperand(1), Pred, PredReg };
2461    return CurDAG->getMachineNode(Opc, dl, VT, VT, Ops, 4);
2462  }
2463  case ARMISD::VTRN: {
2464    unsigned Opc = 0;
2465    EVT VT = N->getValueType(0);
2466    switch (VT.getSimpleVT().SimpleTy) {
2467    default: return NULL;
2468    case MVT::v8i8:  Opc = ARM::VTRNd8; break;
2469    case MVT::v4i16: Opc = ARM::VTRNd16; break;
2470    case MVT::v2f32:
2471    case MVT::v2i32: Opc = ARM::VTRNd32; break;
2472    case MVT::v16i8: Opc = ARM::VTRNq8; break;
2473    case MVT::v8i16: Opc = ARM::VTRNq16; break;
2474    case MVT::v4f32:
2475    case MVT::v4i32: Opc = ARM::VTRNq32; break;
2476    }
2477    SDValue Pred = getAL(CurDAG);
2478    SDValue PredReg = CurDAG->getRegister(0, MVT::i32);
2479    SDValue Ops[] = { N->getOperand(0), N->getOperand(1), Pred, PredReg };
2480    return CurDAG->getMachineNode(Opc, dl, VT, VT, Ops, 4);
2481  }
2482  case ARMISD::BUILD_VECTOR: {
2483    EVT VecVT = N->getValueType(0);
2484    EVT EltVT = VecVT.getVectorElementType();
2485    unsigned NumElts = VecVT.getVectorNumElements();
2486    if (EltVT == MVT::f64) {
2487      assert(NumElts == 2 && "unexpected type for BUILD_VECTOR");
2488      return PairDRegs(VecVT, N->getOperand(0), N->getOperand(1));
2489    }
2490    assert(EltVT == MVT::f32 && "unexpected type for BUILD_VECTOR");
2491    if (NumElts == 2)
2492      return PairSRegs(VecVT, N->getOperand(0), N->getOperand(1));
2493    assert(NumElts == 4 && "unexpected type for BUILD_VECTOR");
2494    return QuadSRegs(VecVT, N->getOperand(0), N->getOperand(1),
2495                     N->getOperand(2), N->getOperand(3));
2496  }
2497
2498  case ARMISD::VLD2DUP: {
2499    unsigned Opcodes[] = { ARM::VLD2DUPd8Pseudo, ARM::VLD2DUPd16Pseudo,
2500                           ARM::VLD2DUPd32Pseudo };
2501    return SelectVLDDup(N, 2, Opcodes);
2502  }
2503
2504  case ARMISD::VLD3DUP: {
2505    unsigned Opcodes[] = { ARM::VLD3DUPd8Pseudo, ARM::VLD3DUPd16Pseudo,
2506                           ARM::VLD3DUPd32Pseudo };
2507    return SelectVLDDup(N, 3, Opcodes);
2508  }
2509
2510  case ARMISD::VLD4DUP: {
2511    unsigned Opcodes[] = { ARM::VLD4DUPd8Pseudo, ARM::VLD4DUPd16Pseudo,
2512                           ARM::VLD4DUPd32Pseudo };
2513    return SelectVLDDup(N, 4, Opcodes);
2514  }
2515
2516  case ISD::INTRINSIC_VOID:
2517  case ISD::INTRINSIC_W_CHAIN: {
2518    unsigned IntNo = cast<ConstantSDNode>(N->getOperand(1))->getZExtValue();
2519    switch (IntNo) {
2520    default:
2521      break;
2522
2523    case Intrinsic::arm_neon_vld1: {
2524      unsigned DOpcodes[] = { ARM::VLD1d8, ARM::VLD1d16,
2525                              ARM::VLD1d32, ARM::VLD1d64 };
2526      unsigned QOpcodes[] = { ARM::VLD1q8Pseudo, ARM::VLD1q16Pseudo,
2527                              ARM::VLD1q32Pseudo, ARM::VLD1q64Pseudo };
2528      return SelectVLD(N, 1, DOpcodes, QOpcodes, 0);
2529    }
2530
2531    case Intrinsic::arm_neon_vld2: {
2532      unsigned DOpcodes[] = { ARM::VLD2d8Pseudo, ARM::VLD2d16Pseudo,
2533                              ARM::VLD2d32Pseudo, ARM::VLD1q64Pseudo };
2534      unsigned QOpcodes[] = { ARM::VLD2q8Pseudo, ARM::VLD2q16Pseudo,
2535                              ARM::VLD2q32Pseudo };
2536      return SelectVLD(N, 2, DOpcodes, QOpcodes, 0);
2537    }
2538
2539    case Intrinsic::arm_neon_vld3: {
2540      unsigned DOpcodes[] = { ARM::VLD3d8Pseudo, ARM::VLD3d16Pseudo,
2541                              ARM::VLD3d32Pseudo, ARM::VLD1d64TPseudo };
2542      unsigned QOpcodes0[] = { ARM::VLD3q8Pseudo_UPD,
2543                               ARM::VLD3q16Pseudo_UPD,
2544                               ARM::VLD3q32Pseudo_UPD };
2545      unsigned QOpcodes1[] = { ARM::VLD3q8oddPseudo_UPD,
2546                               ARM::VLD3q16oddPseudo_UPD,
2547                               ARM::VLD3q32oddPseudo_UPD };
2548      return SelectVLD(N, 3, DOpcodes, QOpcodes0, QOpcodes1);
2549    }
2550
2551    case Intrinsic::arm_neon_vld4: {
2552      unsigned DOpcodes[] = { ARM::VLD4d8Pseudo, ARM::VLD4d16Pseudo,
2553                              ARM::VLD4d32Pseudo, ARM::VLD1d64QPseudo };
2554      unsigned QOpcodes0[] = { ARM::VLD4q8Pseudo_UPD,
2555                               ARM::VLD4q16Pseudo_UPD,
2556                               ARM::VLD4q32Pseudo_UPD };
2557      unsigned QOpcodes1[] = { ARM::VLD4q8oddPseudo_UPD,
2558                               ARM::VLD4q16oddPseudo_UPD,
2559                               ARM::VLD4q32oddPseudo_UPD };
2560      return SelectVLD(N, 4, DOpcodes, QOpcodes0, QOpcodes1);
2561    }
2562
2563    case Intrinsic::arm_neon_vld2lane: {
2564      unsigned DOpcodes[] = { ARM::VLD2LNd8Pseudo, ARM::VLD2LNd16Pseudo,
2565                              ARM::VLD2LNd32Pseudo };
2566      unsigned QOpcodes[] = { ARM::VLD2LNq16Pseudo, ARM::VLD2LNq32Pseudo };
2567      return SelectVLDSTLane(N, true, 2, DOpcodes, QOpcodes);
2568    }
2569
2570    case Intrinsic::arm_neon_vld3lane: {
2571      unsigned DOpcodes[] = { ARM::VLD3LNd8Pseudo, ARM::VLD3LNd16Pseudo,
2572                              ARM::VLD3LNd32Pseudo };
2573      unsigned QOpcodes[] = { ARM::VLD3LNq16Pseudo, ARM::VLD3LNq32Pseudo };
2574      return SelectVLDSTLane(N, true, 3, DOpcodes, QOpcodes);
2575    }
2576
2577    case Intrinsic::arm_neon_vld4lane: {
2578      unsigned DOpcodes[] = { ARM::VLD4LNd8Pseudo, ARM::VLD4LNd16Pseudo,
2579                              ARM::VLD4LNd32Pseudo };
2580      unsigned QOpcodes[] = { ARM::VLD4LNq16Pseudo, ARM::VLD4LNq32Pseudo };
2581      return SelectVLDSTLane(N, true, 4, DOpcodes, QOpcodes);
2582    }
2583
2584    case Intrinsic::arm_neon_vst1: {
2585      unsigned DOpcodes[] = { ARM::VST1d8, ARM::VST1d16,
2586                              ARM::VST1d32, ARM::VST1d64 };
2587      unsigned QOpcodes[] = { ARM::VST1q8Pseudo, ARM::VST1q16Pseudo,
2588                              ARM::VST1q32Pseudo, ARM::VST1q64Pseudo };
2589      return SelectVST(N, 1, DOpcodes, QOpcodes, 0);
2590    }
2591
2592    case Intrinsic::arm_neon_vst2: {
2593      unsigned DOpcodes[] = { ARM::VST2d8Pseudo, ARM::VST2d16Pseudo,
2594                              ARM::VST2d32Pseudo, ARM::VST1q64Pseudo };
2595      unsigned QOpcodes[] = { ARM::VST2q8Pseudo, ARM::VST2q16Pseudo,
2596                              ARM::VST2q32Pseudo };
2597      return SelectVST(N, 2, DOpcodes, QOpcodes, 0);
2598    }
2599
2600    case Intrinsic::arm_neon_vst3: {
2601      unsigned DOpcodes[] = { ARM::VST3d8Pseudo, ARM::VST3d16Pseudo,
2602                              ARM::VST3d32Pseudo, ARM::VST1d64TPseudo };
2603      unsigned QOpcodes0[] = { ARM::VST3q8Pseudo_UPD,
2604                               ARM::VST3q16Pseudo_UPD,
2605                               ARM::VST3q32Pseudo_UPD };
2606      unsigned QOpcodes1[] = { ARM::VST3q8oddPseudo_UPD,
2607                               ARM::VST3q16oddPseudo_UPD,
2608                               ARM::VST3q32oddPseudo_UPD };
2609      return SelectVST(N, 3, DOpcodes, QOpcodes0, QOpcodes1);
2610    }
2611
2612    case Intrinsic::arm_neon_vst4: {
2613      unsigned DOpcodes[] = { ARM::VST4d8Pseudo, ARM::VST4d16Pseudo,
2614                              ARM::VST4d32Pseudo, ARM::VST1d64QPseudo };
2615      unsigned QOpcodes0[] = { ARM::VST4q8Pseudo_UPD,
2616                               ARM::VST4q16Pseudo_UPD,
2617                               ARM::VST4q32Pseudo_UPD };
2618      unsigned QOpcodes1[] = { ARM::VST4q8oddPseudo_UPD,
2619                               ARM::VST4q16oddPseudo_UPD,
2620                               ARM::VST4q32oddPseudo_UPD };
2621      return SelectVST(N, 4, DOpcodes, QOpcodes0, QOpcodes1);
2622    }
2623
2624    case Intrinsic::arm_neon_vst2lane: {
2625      unsigned DOpcodes[] = { ARM::VST2LNd8Pseudo, ARM::VST2LNd16Pseudo,
2626                              ARM::VST2LNd32Pseudo };
2627      unsigned QOpcodes[] = { ARM::VST2LNq16Pseudo, ARM::VST2LNq32Pseudo };
2628      return SelectVLDSTLane(N, false, 2, DOpcodes, QOpcodes);
2629    }
2630
2631    case Intrinsic::arm_neon_vst3lane: {
2632      unsigned DOpcodes[] = { ARM::VST3LNd8Pseudo, ARM::VST3LNd16Pseudo,
2633                              ARM::VST3LNd32Pseudo };
2634      unsigned QOpcodes[] = { ARM::VST3LNq16Pseudo, ARM::VST3LNq32Pseudo };
2635      return SelectVLDSTLane(N, false, 3, DOpcodes, QOpcodes);
2636    }
2637
2638    case Intrinsic::arm_neon_vst4lane: {
2639      unsigned DOpcodes[] = { ARM::VST4LNd8Pseudo, ARM::VST4LNd16Pseudo,
2640                              ARM::VST4LNd32Pseudo };
2641      unsigned QOpcodes[] = { ARM::VST4LNq16Pseudo, ARM::VST4LNq32Pseudo };
2642      return SelectVLDSTLane(N, false, 4, DOpcodes, QOpcodes);
2643    }
2644    }
2645    break;
2646  }
2647
2648  case ISD::INTRINSIC_WO_CHAIN: {
2649    unsigned IntNo = cast<ConstantSDNode>(N->getOperand(0))->getZExtValue();
2650    switch (IntNo) {
2651    default:
2652      break;
2653
2654    case Intrinsic::arm_neon_vtbl2:
2655      return SelectVTBL(N, false, 2, ARM::VTBL2Pseudo);
2656    case Intrinsic::arm_neon_vtbl3:
2657      return SelectVTBL(N, false, 3, ARM::VTBL3Pseudo);
2658    case Intrinsic::arm_neon_vtbl4:
2659      return SelectVTBL(N, false, 4, ARM::VTBL4Pseudo);
2660
2661    case Intrinsic::arm_neon_vtbx2:
2662      return SelectVTBL(N, true, 2, ARM::VTBX2Pseudo);
2663    case Intrinsic::arm_neon_vtbx3:
2664      return SelectVTBL(N, true, 3, ARM::VTBX3Pseudo);
2665    case Intrinsic::arm_neon_vtbx4:
2666      return SelectVTBL(N, true, 4, ARM::VTBX4Pseudo);
2667    }
2668    break;
2669  }
2670
2671  case ISD::CONCAT_VECTORS:
2672    return SelectConcatVector(N);
2673  }
2674
2675  return SelectCode(N);
2676}
2677
2678bool ARMDAGToDAGISel::
2679SelectInlineAsmMemoryOperand(const SDValue &Op, char ConstraintCode,
2680                             std::vector<SDValue> &OutOps) {
2681  assert(ConstraintCode == 'm' && "unexpected asm memory constraint");
2682  // Require the address to be in a register.  That is safe for all ARM
2683  // variants and it is hard to do anything much smarter without knowing
2684  // how the operand is used.
2685  OutOps.push_back(Op);
2686  return false;
2687}
2688
2689/// createARMISelDag - This pass converts a legalized DAG into a
2690/// ARM-specific DAG, ready for instruction scheduling.
2691///
2692FunctionPass *llvm::createARMISelDag(ARMBaseTargetMachine &TM,
2693                                     CodeGenOpt::Level OptLevel) {
2694  return new ARMDAGToDAGISel(TM, OptLevel);
2695}
2696