MipsSEISelLowering.cpp revision aee7825762830536956b9e634fd7ffd59396984d 
1//===-- MipsSEISelLowering.cpp - MipsSE DAG Lowering Interface --*- C++ -*-===//
2//
3//                     The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10// Subclass of MipsTargetLowering specialized for mips32/64.
11//
12//===----------------------------------------------------------------------===//
13#define DEBUG_TYPE "mips-isel"
14#include "MipsSEISelLowering.h"
15#include "MipsRegisterInfo.h"
16#include "MipsTargetMachine.h"
17#include "llvm/CodeGen/MachineInstrBuilder.h"
18#include "llvm/CodeGen/MachineRegisterInfo.h"
19#include "llvm/IR/Intrinsics.h"
20#include "llvm/Support/CommandLine.h"
21#include "llvm/Support/Debug.h"
22#include "llvm/Support/raw_ostream.h"
23#include "llvm/Target/TargetInstrInfo.h"
24
25using namespace llvm;
26
27static cl::opt<bool>
28EnableMipsTailCalls("enable-mips-tail-calls", cl::Hidden,
29                    cl::desc("MIPS: Enable tail calls."), cl::init(false));
30
31static cl::opt<bool> NoDPLoadStore("mno-ldc1-sdc1", cl::init(false),
32                                   cl::desc("Expand double precision loads and "
33                                            "stores to their single precision "
34                                            "counterparts"));
35
36MipsSETargetLowering::MipsSETargetLowering(MipsTargetMachine &TM)
37  : MipsTargetLowering(TM) {
38  // Set up the register classes
39  addRegisterClass(MVT::i32, &Mips::GPR32RegClass);
40
41  if (HasMips64)
42    addRegisterClass(MVT::i64, &Mips::GPR64RegClass);
43
44  if (Subtarget->hasDSP() || Subtarget->hasMSA()) {
45    // Expand all truncating stores and extending loads.
46    unsigned FirstVT = (unsigned)MVT::FIRST_VECTOR_VALUETYPE;
47    unsigned LastVT = (unsigned)MVT::LAST_VECTOR_VALUETYPE;
48
49    for (unsigned VT0 = FirstVT; VT0 <= LastVT; ++VT0) {
50      for (unsigned VT1 = FirstVT; VT1 <= LastVT; ++VT1)
51        setTruncStoreAction((MVT::SimpleValueType)VT0,
52                            (MVT::SimpleValueType)VT1, Expand);
53
54      setLoadExtAction(ISD::SEXTLOAD, (MVT::SimpleValueType)VT0, Expand);
55      setLoadExtAction(ISD::ZEXTLOAD, (MVT::SimpleValueType)VT0, Expand);
56      setLoadExtAction(ISD::EXTLOAD, (MVT::SimpleValueType)VT0, Expand);
57    }
58  }
59
60  if (Subtarget->hasDSP()) {
61    MVT::SimpleValueType VecTys[2] = {MVT::v2i16, MVT::v4i8};
62
63    for (unsigned i = 0; i < array_lengthof(VecTys); ++i) {
64      addRegisterClass(VecTys[i], &Mips::DSPRRegClass);
65
66      // Expand all builtin opcodes.
67      for (unsigned Opc = 0; Opc < ISD::BUILTIN_OP_END; ++Opc)
68        setOperationAction(Opc, VecTys[i], Expand);
69
70      setOperationAction(ISD::ADD, VecTys[i], Legal);
71      setOperationAction(ISD::SUB, VecTys[i], Legal);
72      setOperationAction(ISD::LOAD, VecTys[i], Legal);
73      setOperationAction(ISD::STORE, VecTys[i], Legal);
74      setOperationAction(ISD::BITCAST, VecTys[i], Legal);
75    }
76
77    setTargetDAGCombine(ISD::SHL);
78    setTargetDAGCombine(ISD::SRA);
79    setTargetDAGCombine(ISD::SRL);
80    setTargetDAGCombine(ISD::SETCC);
81    setTargetDAGCombine(ISD::VSELECT);
82  }
83
84  if (Subtarget->hasDSPR2())
85    setOperationAction(ISD::MUL, MVT::v2i16, Legal);
86
87  if (Subtarget->hasMSA()) {
88    addMSAIntType(MVT::v16i8, &Mips::MSA128BRegClass);
89    addMSAIntType(MVT::v8i16, &Mips::MSA128HRegClass);
90    addMSAIntType(MVT::v4i32, &Mips::MSA128WRegClass);
91    addMSAIntType(MVT::v2i64, &Mips::MSA128DRegClass);
92    addMSAFloatType(MVT::v8f16, &Mips::MSA128HRegClass);
93    addMSAFloatType(MVT::v4f32, &Mips::MSA128WRegClass);
94    addMSAFloatType(MVT::v2f64, &Mips::MSA128DRegClass);
95
96    setTargetDAGCombine(ISD::AND);
97    setTargetDAGCombine(ISD::OR);
98    setTargetDAGCombine(ISD::SRA);
99    setTargetDAGCombine(ISD::VSELECT);
100    setTargetDAGCombine(ISD::XOR);
101  }
102
103  if (!Subtarget->mipsSEUsesSoftFloat()) {
104    addRegisterClass(MVT::f32, &Mips::FGR32RegClass);
105
106    // When dealing with single precision only, use libcalls
107    if (!Subtarget->isSingleFloat()) {
108      if (Subtarget->isFP64bit())
109        addRegisterClass(MVT::f64, &Mips::FGR64RegClass);
110      else
111        addRegisterClass(MVT::f64, &Mips::AFGR64RegClass);
112    }
113  }
114
115  setOperationAction(ISD::SMUL_LOHI,          MVT::i32, Custom);
116  setOperationAction(ISD::UMUL_LOHI,          MVT::i32, Custom);
117  setOperationAction(ISD::MULHS,              MVT::i32, Custom);
118  setOperationAction(ISD::MULHU,              MVT::i32, Custom);
119
120  if (HasMips64) {
121    setOperationAction(ISD::MULHS,            MVT::i64, Custom);
122    setOperationAction(ISD::MULHU,            MVT::i64, Custom);
123    setOperationAction(ISD::MUL,              MVT::i64, Custom);
124  }
125
126  setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::i64, Custom);
127  setOperationAction(ISD::INTRINSIC_W_CHAIN,  MVT::i64, Custom);
128
129  setOperationAction(ISD::SDIVREM, MVT::i32, Custom);
130  setOperationAction(ISD::UDIVREM, MVT::i32, Custom);
131  setOperationAction(ISD::SDIVREM, MVT::i64, Custom);
132  setOperationAction(ISD::UDIVREM, MVT::i64, Custom);
133  setOperationAction(ISD::ATOMIC_FENCE,       MVT::Other, Custom);
134  setOperationAction(ISD::LOAD,               MVT::i32, Custom);
135  setOperationAction(ISD::STORE,              MVT::i32, Custom);
136
137  setTargetDAGCombine(ISD::ADDE);
138  setTargetDAGCombine(ISD::SUBE);
139  setTargetDAGCombine(ISD::MUL);
140
141  setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::Other, Custom);
142  setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::Other, Custom);
143  setOperationAction(ISD::INTRINSIC_VOID, MVT::Other, Custom);
144
145  if (NoDPLoadStore) {
146    setOperationAction(ISD::LOAD, MVT::f64, Custom);
147    setOperationAction(ISD::STORE, MVT::f64, Custom);
148  }
149
150  computeRegisterProperties();
151}
152
153const MipsTargetLowering *
154llvm::createMipsSETargetLowering(MipsTargetMachine &TM) {
155  return new MipsSETargetLowering(TM);
156}
157
158// Enable MSA support for the given integer type and Register class.
159void MipsSETargetLowering::
160addMSAIntType(MVT::SimpleValueType Ty, const TargetRegisterClass *RC) {
161  addRegisterClass(Ty, RC);
162
163  // Expand all builtin opcodes.
164  for (unsigned Opc = 0; Opc < ISD::BUILTIN_OP_END; ++Opc)
165    setOperationAction(Opc, Ty, Expand);
166
167  setOperationAction(ISD::BITCAST, Ty, Legal);
168  setOperationAction(ISD::LOAD, Ty, Legal);
169  setOperationAction(ISD::STORE, Ty, Legal);
170  setOperationAction(ISD::EXTRACT_VECTOR_ELT, Ty, Custom);
171  setOperationAction(ISD::INSERT_VECTOR_ELT, Ty, Legal);
172  setOperationAction(ISD::BUILD_VECTOR, Ty, Custom);
173
174  setOperationAction(ISD::ADD, Ty, Legal);
175  setOperationAction(ISD::AND, Ty, Legal);
176  setOperationAction(ISD::CTLZ, Ty, Legal);
177  setOperationAction(ISD::CTPOP, Ty, Legal);
178  setOperationAction(ISD::MUL, Ty, Legal);
179  setOperationAction(ISD::OR, Ty, Legal);
180  setOperationAction(ISD::SDIV, Ty, Legal);
181  setOperationAction(ISD::SREM, Ty, Legal);
182  setOperationAction(ISD::SHL, Ty, Legal);
183  setOperationAction(ISD::SRA, Ty, Legal);
184  setOperationAction(ISD::SRL, Ty, Legal);
185  setOperationAction(ISD::SUB, Ty, Legal);
186  setOperationAction(ISD::UDIV, Ty, Legal);
187  setOperationAction(ISD::UREM, Ty, Legal);
188  setOperationAction(ISD::VECTOR_SHUFFLE, Ty, Custom);
189  setOperationAction(ISD::VSELECT, Ty, Legal);
190  setOperationAction(ISD::XOR, Ty, Legal);
191
192  if (Ty == MVT::v4i32 || Ty == MVT::v2i64) {
193    setOperationAction(ISD::FP_TO_SINT, Ty, Legal);
194    setOperationAction(ISD::FP_TO_UINT, Ty, Legal);
195    setOperationAction(ISD::SINT_TO_FP, Ty, Legal);
196    setOperationAction(ISD::UINT_TO_FP, Ty, Legal);
197  }
198
199  setOperationAction(ISD::SETCC, Ty, Legal);
200  setCondCodeAction(ISD::SETNE, Ty, Expand);
201  setCondCodeAction(ISD::SETGE, Ty, Expand);
202  setCondCodeAction(ISD::SETGT, Ty, Expand);
203  setCondCodeAction(ISD::SETUGE, Ty, Expand);
204  setCondCodeAction(ISD::SETUGT, Ty, Expand);
205}
206
207// Enable MSA support for the given floating-point type and Register class.
208void MipsSETargetLowering::
209addMSAFloatType(MVT::SimpleValueType Ty, const TargetRegisterClass *RC) {
210  addRegisterClass(Ty, RC);
211
212  // Expand all builtin opcodes.
213  for (unsigned Opc = 0; Opc < ISD::BUILTIN_OP_END; ++Opc)
214    setOperationAction(Opc, Ty, Expand);
215
216  setOperationAction(ISD::LOAD, Ty, Legal);
217  setOperationAction(ISD::STORE, Ty, Legal);
218  setOperationAction(ISD::BITCAST, Ty, Legal);
219  setOperationAction(ISD::EXTRACT_VECTOR_ELT, Ty, Legal);
220  setOperationAction(ISD::INSERT_VECTOR_ELT, Ty, Legal);
221  setOperationAction(ISD::BUILD_VECTOR, Ty, Custom);
222
223  if (Ty != MVT::v8f16) {
224    setOperationAction(ISD::FABS,  Ty, Legal);
225    setOperationAction(ISD::FADD,  Ty, Legal);
226    setOperationAction(ISD::FDIV,  Ty, Legal);
227    setOperationAction(ISD::FEXP2, Ty, Legal);
228    setOperationAction(ISD::FLOG2, Ty, Legal);
229    setOperationAction(ISD::FMA,   Ty, Legal);
230    setOperationAction(ISD::FMUL,  Ty, Legal);
231    setOperationAction(ISD::FRINT, Ty, Legal);
232    setOperationAction(ISD::FSQRT, Ty, Legal);
233    setOperationAction(ISD::FSUB,  Ty, Legal);
234    setOperationAction(ISD::VSELECT, Ty, Legal);
235
236    setOperationAction(ISD::SETCC, Ty, Legal);
237    setCondCodeAction(ISD::SETOGE, Ty, Expand);
238    setCondCodeAction(ISD::SETOGT, Ty, Expand);
239    setCondCodeAction(ISD::SETUGE, Ty, Expand);
240    setCondCodeAction(ISD::SETUGT, Ty, Expand);
241    setCondCodeAction(ISD::SETGE,  Ty, Expand);
242    setCondCodeAction(ISD::SETGT,  Ty, Expand);
243  }
244}
245
246bool
247MipsSETargetLowering::allowsUnalignedMemoryAccesses(EVT VT, bool *Fast) const {
248  MVT::SimpleValueType SVT = VT.getSimpleVT().SimpleTy;
249
250  switch (SVT) {
251  case MVT::i64:
252  case MVT::i32:
253    if (Fast)
254      *Fast = true;
255    return true;
256  default:
257    return false;
258  }
259}
260
261SDValue MipsSETargetLowering::LowerOperation(SDValue Op,
262                                             SelectionDAG &DAG) const {
263  switch(Op.getOpcode()) {
264  case ISD::LOAD:  return lowerLOAD(Op, DAG);
265  case ISD::STORE: return lowerSTORE(Op, DAG);
266  case ISD::SMUL_LOHI: return lowerMulDiv(Op, MipsISD::Mult, true, true, DAG);
267  case ISD::UMUL_LOHI: return lowerMulDiv(Op, MipsISD::Multu, true, true, DAG);
268  case ISD::MULHS:     return lowerMulDiv(Op, MipsISD::Mult, false, true, DAG);
269  case ISD::MULHU:     return lowerMulDiv(Op, MipsISD::Multu, false, true, DAG);
270  case ISD::MUL:       return lowerMulDiv(Op, MipsISD::Mult, true, false, DAG);
271  case ISD::SDIVREM:   return lowerMulDiv(Op, MipsISD::DivRem, true, true, DAG);
272  case ISD::UDIVREM:   return lowerMulDiv(Op, MipsISD::DivRemU, true, true,
273                                          DAG);
274  case ISD::INTRINSIC_WO_CHAIN: return lowerINTRINSIC_WO_CHAIN(Op, DAG);
275  case ISD::INTRINSIC_W_CHAIN:  return lowerINTRINSIC_W_CHAIN(Op, DAG);
276  case ISD::INTRINSIC_VOID:     return lowerINTRINSIC_VOID(Op, DAG);
277  case ISD::EXTRACT_VECTOR_ELT: return lowerEXTRACT_VECTOR_ELT(Op, DAG);
278  case ISD::BUILD_VECTOR:       return lowerBUILD_VECTOR(Op, DAG);
279  case ISD::VECTOR_SHUFFLE:     return lowerVECTOR_SHUFFLE(Op, DAG);
280  }
281
282  return MipsTargetLowering::LowerOperation(Op, DAG);
283}
284
285// selectMADD -
286// Transforms a subgraph in CurDAG if the following pattern is found:
287//  (addc multLo, Lo0), (adde multHi, Hi0),
288// where,
289//  multHi/Lo: product of multiplication
290//  Lo0: initial value of Lo register
291//  Hi0: initial value of Hi register
292// Return true if pattern matching was successful.
293static bool selectMADD(SDNode *ADDENode, SelectionDAG *CurDAG) {
294  // ADDENode's second operand must be a flag output of an ADDC node in order
295  // for the matching to be successful.
296  SDNode *ADDCNode = ADDENode->getOperand(2).getNode();
297
298  if (ADDCNode->getOpcode() != ISD::ADDC)
299    return false;
300
301  SDValue MultHi = ADDENode->getOperand(0);
302  SDValue MultLo = ADDCNode->getOperand(0);
303  SDNode *MultNode = MultHi.getNode();
304  unsigned MultOpc = MultHi.getOpcode();
305
306  // MultHi and MultLo must be generated by the same node,
307  if (MultLo.getNode() != MultNode)
308    return false;
309
310  // and it must be a multiplication.
311  if (MultOpc != ISD::SMUL_LOHI && MultOpc != ISD::UMUL_LOHI)
312    return false;
313
314  // MultLo amd MultHi must be the first and second output of MultNode
315  // respectively.
316  if (MultHi.getResNo() != 1 || MultLo.getResNo() != 0)
317    return false;
318
319  // Transform this to a MADD only if ADDENode and ADDCNode are the only users
320  // of the values of MultNode, in which case MultNode will be removed in later
321  // phases.
322  // If there exist users other than ADDENode or ADDCNode, this function returns
323  // here, which will result in MultNode being mapped to a single MULT
324  // instruction node rather than a pair of MULT and MADD instructions being
325  // produced.
326  if (!MultHi.hasOneUse() || !MultLo.hasOneUse())
327    return false;
328
329  SDLoc DL(ADDENode);
330
331  // Initialize accumulator.
332  SDValue ACCIn = CurDAG->getNode(MipsISD::MTLOHI, DL, MVT::Untyped,
333                                  ADDCNode->getOperand(1),
334                                  ADDENode->getOperand(1));
335
336  // create MipsMAdd(u) node
337  MultOpc = MultOpc == ISD::UMUL_LOHI ? MipsISD::MAddu : MipsISD::MAdd;
338
339  SDValue MAdd = CurDAG->getNode(MultOpc, DL, MVT::Untyped,
340                                 MultNode->getOperand(0),// Factor 0
341                                 MultNode->getOperand(1),// Factor 1
342                                 ACCIn);
343
344  // replace uses of adde and addc here
345  if (!SDValue(ADDCNode, 0).use_empty()) {
346    SDValue LoOut = CurDAG->getNode(MipsISD::MFLO, DL, MVT::i32, MAdd);
347    CurDAG->ReplaceAllUsesOfValueWith(SDValue(ADDCNode, 0), LoOut);
348  }
349  if (!SDValue(ADDENode, 0).use_empty()) {
350    SDValue HiOut = CurDAG->getNode(MipsISD::MFHI, DL, MVT::i32, MAdd);
351    CurDAG->ReplaceAllUsesOfValueWith(SDValue(ADDENode, 0), HiOut);
352  }
353
354  return true;
355}
356
357// selectMSUB -
358// Transforms a subgraph in CurDAG if the following pattern is found:
359//  (addc Lo0, multLo), (sube Hi0, multHi),
360// where,
361//  multHi/Lo: product of multiplication
362//  Lo0: initial value of Lo register
363//  Hi0: initial value of Hi register
364// Return true if pattern matching was successful.
365static bool selectMSUB(SDNode *SUBENode, SelectionDAG *CurDAG) {
366  // SUBENode's second operand must be a flag output of an SUBC node in order
367  // for the matching to be successful.
368  SDNode *SUBCNode = SUBENode->getOperand(2).getNode();
369
370  if (SUBCNode->getOpcode() != ISD::SUBC)
371    return false;
372
373  SDValue MultHi = SUBENode->getOperand(1);
374  SDValue MultLo = SUBCNode->getOperand(1);
375  SDNode *MultNode = MultHi.getNode();
376  unsigned MultOpc = MultHi.getOpcode();
377
378  // MultHi and MultLo must be generated by the same node,
379  if (MultLo.getNode() != MultNode)
380    return false;
381
382  // and it must be a multiplication.
383  if (MultOpc != ISD::SMUL_LOHI && MultOpc != ISD::UMUL_LOHI)
384    return false;
385
386  // MultLo amd MultHi must be the first and second output of MultNode
387  // respectively.
388  if (MultHi.getResNo() != 1 || MultLo.getResNo() != 0)
389    return false;
390
391  // Transform this to a MSUB only if SUBENode and SUBCNode are the only users
392  // of the values of MultNode, in which case MultNode will be removed in later
393  // phases.
394  // If there exist users other than SUBENode or SUBCNode, this function returns
395  // here, which will result in MultNode being mapped to a single MULT
396  // instruction node rather than a pair of MULT and MSUB instructions being
397  // produced.
398  if (!MultHi.hasOneUse() || !MultLo.hasOneUse())
399    return false;
400
401  SDLoc DL(SUBENode);
402
403  // Initialize accumulator.
404  SDValue ACCIn = CurDAG->getNode(MipsISD::MTLOHI, DL, MVT::Untyped,
405                                  SUBCNode->getOperand(0),
406                                  SUBENode->getOperand(0));
407
408  // create MipsSub(u) node
409  MultOpc = MultOpc == ISD::UMUL_LOHI ? MipsISD::MSubu : MipsISD::MSub;
410
411  SDValue MSub = CurDAG->getNode(MultOpc, DL, MVT::Glue,
412                                 MultNode->getOperand(0),// Factor 0
413                                 MultNode->getOperand(1),// Factor 1
414                                 ACCIn);
415
416  // replace uses of sube and subc here
417  if (!SDValue(SUBCNode, 0).use_empty()) {
418    SDValue LoOut = CurDAG->getNode(MipsISD::MFLO, DL, MVT::i32, MSub);
419    CurDAG->ReplaceAllUsesOfValueWith(SDValue(SUBCNode, 0), LoOut);
420  }
421  if (!SDValue(SUBENode, 0).use_empty()) {
422    SDValue HiOut = CurDAG->getNode(MipsISD::MFHI, DL, MVT::i32, MSub);
423    CurDAG->ReplaceAllUsesOfValueWith(SDValue(SUBENode, 0), HiOut);
424  }
425
426  return true;
427}
428
429static SDValue performADDECombine(SDNode *N, SelectionDAG &DAG,
430                                  TargetLowering::DAGCombinerInfo &DCI,
431                                  const MipsSubtarget *Subtarget) {
432  if (DCI.isBeforeLegalize())
433    return SDValue();
434
435  if (Subtarget->hasMips32() && N->getValueType(0) == MVT::i32 &&
436      selectMADD(N, &DAG))
437    return SDValue(N, 0);
438
439  return SDValue();
440}
441
442// Fold zero extensions into MipsISD::VEXTRACT_[SZ]EXT_ELT
443//
444// Performs the following transformations:
445// - Changes MipsISD::VEXTRACT_[SZ]EXT_ELT to zero extension if its
446//   sign/zero-extension is completely overwritten by the new one performed by
447//   the ISD::AND.
448// - Removes redundant zero extensions performed by an ISD::AND.
449static SDValue performANDCombine(SDNode *N, SelectionDAG &DAG,
450                                 TargetLowering::DAGCombinerInfo &DCI,
451                                 const MipsSubtarget *Subtarget) {
452  if (!Subtarget->hasMSA())
453    return SDValue();
454
455  SDValue Op0 = N->getOperand(0);
456  SDValue Op1 = N->getOperand(1);
457  unsigned Op0Opcode = Op0->getOpcode();
458
459  // (and (MipsVExtract[SZ]Ext $a, $b, $c), imm:$d)
460  // where $d + 1 == 2^n and n == 32
461  // or    $d + 1 == 2^n and n <= 32 and ZExt
462  // -> (MipsVExtractZExt $a, $b, $c)
463  if (Op0Opcode == MipsISD::VEXTRACT_SEXT_ELT ||
464      Op0Opcode == MipsISD::VEXTRACT_ZEXT_ELT) {
465    ConstantSDNode *Mask = dyn_cast<ConstantSDNode>(Op1);
466
467    if (!Mask)
468      return SDValue();
469
470    int32_t Log2IfPositive = (Mask->getAPIntValue() + 1).exactLogBase2();
471
472    if (Log2IfPositive <= 0)
473      return SDValue(); // Mask+1 is not a power of 2
474
475    SDValue Op0Op2 = Op0->getOperand(2);
476    EVT ExtendTy = cast<VTSDNode>(Op0Op2)->getVT();
477    unsigned ExtendTySize = ExtendTy.getSizeInBits();
478    unsigned Log2 = Log2IfPositive;
479
480    if ((Op0Opcode == MipsISD::VEXTRACT_ZEXT_ELT && Log2 >= ExtendTySize) ||
481        Log2 == ExtendTySize) {
482      SDValue Ops[] = { Op0->getOperand(0), Op0->getOperand(1), Op0Op2 };
483      DAG.MorphNodeTo(Op0.getNode(), MipsISD::VEXTRACT_ZEXT_ELT,
484                      Op0->getVTList(), Ops, Op0->getNumOperands());
485      return Op0;
486    }
487  }
488
489  return SDValue();
490}
491
492// Determine if the specified node is a constant vector splat.
493//
494// Returns true and sets Imm if:
495// * N is a ISD::BUILD_VECTOR representing a constant splat
496//
497// This function is quite similar to MipsSEDAGToDAGISel::selectVSplat. The
498// differences are that it assumes the MSA has already been checked and the
499// arbitrary requirement for a maximum of 32-bit integers isn't applied (and
500// must not be in order for binsri.d to be selectable).
501static bool isVSplat(SDValue N, APInt &Imm, bool IsLittleEndian) {
502  BuildVectorSDNode *Node = dyn_cast<BuildVectorSDNode>(N.getNode());
503
504  if (Node == NULL)
505    return false;
506
507  APInt SplatValue, SplatUndef;
508  unsigned SplatBitSize;
509  bool HasAnyUndefs;
510
511  if (!Node->isConstantSplat(SplatValue, SplatUndef, SplatBitSize, HasAnyUndefs,
512                             8, !IsLittleEndian))
513    return false;
514
515  Imm = SplatValue;
516
517  return true;
518}
519
520// Test whether the given node is an all-ones build_vector.
521static bool isVectorAllOnes(SDValue N) {
522  // Look through bitcasts. Endianness doesn't matter because we are looking
523  // for an all-ones value.
524  if (N->getOpcode() == ISD::BITCAST)
525    N = N->getOperand(0);
526
527  BuildVectorSDNode *BVN = dyn_cast<BuildVectorSDNode>(N);
528
529  if (!BVN)
530    return false;
531
532  APInt SplatValue, SplatUndef;
533  unsigned SplatBitSize;
534  bool HasAnyUndefs;
535
536  // Endianness doesn't matter in this context because we are looking for
537  // an all-ones value.
538  if (BVN->isConstantSplat(SplatValue, SplatUndef, SplatBitSize, HasAnyUndefs))
539    return SplatValue.isAllOnesValue();
540
541  return false;
542}
543
544// Test whether N is the bitwise inverse of OfNode.
545static bool isBitwiseInverse(SDValue N, SDValue OfNode) {
546  if (N->getOpcode() != ISD::XOR)
547    return false;
548
549  if (isVectorAllOnes(N->getOperand(0)))
550    return N->getOperand(1) == OfNode;
551
552  if (isVectorAllOnes(N->getOperand(1)))
553    return N->getOperand(0) == OfNode;
554
555  return false;
556}
557
558// Perform combines where ISD::OR is the root node.
559//
560// Performs the following transformations:
561// - (or (and $a, $mask), (and $b, $inv_mask)) => (vselect $mask, $a, $b)
562//   where $inv_mask is the bitwise inverse of $mask and the 'or' has a 128-bit
563//   vector type.
564static SDValue performORCombine(SDNode *N, SelectionDAG &DAG,
565                                TargetLowering::DAGCombinerInfo &DCI,
566                                const MipsSubtarget *Subtarget) {
567  if (!Subtarget->hasMSA())
568    return SDValue();
569
570  EVT Ty = N->getValueType(0);
571
572  if (!Ty.is128BitVector())
573    return SDValue();
574
575  SDValue Op0 = N->getOperand(0);
576  SDValue Op1 = N->getOperand(1);
577
578  if (Op0->getOpcode() == ISD::AND && Op1->getOpcode() == ISD::AND) {
579    SDValue Op0Op0 = Op0->getOperand(0);
580    SDValue Op0Op1 = Op0->getOperand(1);
581    SDValue Op1Op0 = Op1->getOperand(0);
582    SDValue Op1Op1 = Op1->getOperand(1);
583    bool IsLittleEndian = !Subtarget->isLittle();
584
585    SDValue IfSet, IfClr, Cond;
586    bool IsConstantMask = false;
587    APInt Mask, InvMask;
588
589    // If Op0Op0 is an appropriate mask, try to find it's inverse in either
590    // Op1Op0, or Op1Op1. Keep track of the Cond, IfSet, and IfClr nodes, while
591    // looking.
592    // IfClr will be set if we find a valid match.
593    if (isVSplat(Op0Op0, Mask, IsLittleEndian)) {
594      Cond = Op0Op0;
595      IfSet = Op0Op1;
596
597      if (isVSplat(Op1Op0, InvMask, IsLittleEndian) && Mask == ~InvMask)
598        IfClr = Op1Op1;
599      else if (isVSplat(Op1Op1, InvMask, IsLittleEndian) && Mask == ~InvMask)
600        IfClr = Op1Op0;
601
602      IsConstantMask = true;
603    }
604
605    // If IfClr is not yet set, and Op0Op1 is an appropriate mask, try the same
606    // thing again using this mask.
607    // IfClr will be set if we find a valid match.
608    if (!IfClr.getNode() && isVSplat(Op0Op1, Mask, IsLittleEndian)) {
609      Cond = Op0Op1;
610      IfSet = Op0Op0;
611
612      if (isVSplat(Op1Op0, InvMask, IsLittleEndian) && Mask == ~InvMask)
613        IfClr = Op1Op1;
614      else if (isVSplat(Op1Op1, InvMask, IsLittleEndian) && Mask == ~InvMask)
615        IfClr = Op1Op0;
616
617      IsConstantMask = true;
618    }
619
620    // If IfClr is not yet set, try looking for a non-constant match.
621    // IfClr will be set if we find a valid match amongst the eight
622    // possibilities.
623    if (!IfClr.getNode()) {
624      if (isBitwiseInverse(Op0Op0, Op1Op0)) {
625        Cond = Op1Op0;
626        IfSet = Op1Op1;
627        IfClr = Op0Op1;
628      } else if (isBitwiseInverse(Op0Op1, Op1Op0)) {
629        Cond = Op1Op0;
630        IfSet = Op1Op1;
631        IfClr = Op0Op0;
632      } else if (isBitwiseInverse(Op0Op0, Op1Op1)) {
633        Cond = Op1Op1;
634        IfSet = Op1Op0;
635        IfClr = Op0Op1;
636      } else if (isBitwiseInverse(Op0Op1, Op1Op1)) {
637        Cond = Op1Op1;
638        IfSet = Op1Op0;
639        IfClr = Op0Op0;
640      } else if (isBitwiseInverse(Op1Op0, Op0Op0)) {
641        Cond = Op0Op0;
642        IfSet = Op0Op1;
643        IfClr = Op1Op1;
644      } else if (isBitwiseInverse(Op1Op1, Op0Op0)) {
645        Cond = Op0Op0;
646        IfSet = Op0Op1;
647        IfClr = Op1Op0;
648      } else if (isBitwiseInverse(Op1Op0, Op0Op1)) {
649        Cond = Op0Op1;
650        IfSet = Op0Op0;
651        IfClr = Op1Op1;
652      } else if (isBitwiseInverse(Op1Op1, Op0Op1)) {
653        Cond = Op0Op1;
654        IfSet = Op0Op0;
655        IfClr = Op1Op0;
656      }
657    }
658
659    // At this point, IfClr will be set if we have a valid match.
660    if (!IfClr.getNode())
661      return SDValue();
662
663    assert(Cond.getNode() && IfSet.getNode());
664
665    // Fold degenerate cases.
666    if (IsConstantMask) {
667      if (Mask.isAllOnesValue())
668        return IfSet;
669      else if (Mask == 0)
670        return IfClr;
671    }
672
673    // Transform the DAG into an equivalent VSELECT.
674    return DAG.getNode(ISD::VSELECT, SDLoc(N), Ty, Cond, IfClr, IfSet);
675  }
676
677  return SDValue();
678}
679
680static SDValue performSUBECombine(SDNode *N, SelectionDAG &DAG,
681                                  TargetLowering::DAGCombinerInfo &DCI,
682                                  const MipsSubtarget *Subtarget) {
683  if (DCI.isBeforeLegalize())
684    return SDValue();
685
686  if (Subtarget->hasMips32() && N->getValueType(0) == MVT::i32 &&
687      selectMSUB(N, &DAG))
688    return SDValue(N, 0);
689
690  return SDValue();
691}
692
693static SDValue genConstMult(SDValue X, uint64_t C, SDLoc DL, EVT VT,
694                            EVT ShiftTy, SelectionDAG &DAG) {
695  // Clear the upper (64 - VT.sizeInBits) bits.
696  C &= ((uint64_t)-1) >> (64 - VT.getSizeInBits());
697
698  // Return 0.
699  if (C == 0)
700    return DAG.getConstant(0, VT);
701
702  // Return x.
703  if (C == 1)
704    return X;
705
706  // If c is power of 2, return (shl x, log2(c)).
707  if (isPowerOf2_64(C))
708    return DAG.getNode(ISD::SHL, DL, VT, X,
709                       DAG.getConstant(Log2_64(C), ShiftTy));
710
711  unsigned Log2Ceil = Log2_64_Ceil(C);
712  uint64_t Floor = 1LL << Log2_64(C);
713  uint64_t Ceil = Log2Ceil == 64 ? 0LL : 1LL << Log2Ceil;
714
715  // If |c - floor_c| <= |c - ceil_c|,
716  // where floor_c = pow(2, floor(log2(c))) and ceil_c = pow(2, ceil(log2(c))),
717  // return (add constMult(x, floor_c), constMult(x, c - floor_c)).
718  if (C - Floor <= Ceil - C) {
719    SDValue Op0 = genConstMult(X, Floor, DL, VT, ShiftTy, DAG);
720    SDValue Op1 = genConstMult(X, C - Floor, DL, VT, ShiftTy, DAG);
721    return DAG.getNode(ISD::ADD, DL, VT, Op0, Op1);
722  }
723
724  // If |c - floor_c| > |c - ceil_c|,
725  // return (sub constMult(x, ceil_c), constMult(x, ceil_c - c)).
726  SDValue Op0 = genConstMult(X, Ceil, DL, VT, ShiftTy, DAG);
727  SDValue Op1 = genConstMult(X, Ceil - C, DL, VT, ShiftTy, DAG);
728  return DAG.getNode(ISD::SUB, DL, VT, Op0, Op1);
729}
730
731static SDValue performMULCombine(SDNode *N, SelectionDAG &DAG,
732                                 const TargetLowering::DAGCombinerInfo &DCI,
733                                 const MipsSETargetLowering *TL) {
734  EVT VT = N->getValueType(0);
735
736  if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(N->getOperand(1)))
737    if (!VT.isVector())
738      return genConstMult(N->getOperand(0), C->getZExtValue(), SDLoc(N),
739                          VT, TL->getScalarShiftAmountTy(VT), DAG);
740
741  return SDValue(N, 0);
742}
743
744static SDValue performDSPShiftCombine(unsigned Opc, SDNode *N, EVT Ty,
745                                      SelectionDAG &DAG,
746                                      const MipsSubtarget *Subtarget) {
747  // See if this is a vector splat immediate node.
748  APInt SplatValue, SplatUndef;
749  unsigned SplatBitSize;
750  bool HasAnyUndefs;
751  unsigned EltSize = Ty.getVectorElementType().getSizeInBits();
752  BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(N->getOperand(1));
753
754  if (!BV ||
755      !BV->isConstantSplat(SplatValue, SplatUndef, SplatBitSize, HasAnyUndefs,
756                           EltSize, !Subtarget->isLittle()) ||
757      (SplatBitSize != EltSize) ||
758      (SplatValue.getZExtValue() >= EltSize))
759    return SDValue();
760
761  return DAG.getNode(Opc, SDLoc(N), Ty, N->getOperand(0),
762                     DAG.getConstant(SplatValue.getZExtValue(), MVT::i32));
763}
764
765static SDValue performSHLCombine(SDNode *N, SelectionDAG &DAG,
766                                 TargetLowering::DAGCombinerInfo &DCI,
767                                 const MipsSubtarget *Subtarget) {
768  EVT Ty = N->getValueType(0);
769
770  if ((Ty != MVT::v2i16) && (Ty != MVT::v4i8))
771    return SDValue();
772
773  return performDSPShiftCombine(MipsISD::SHLL_DSP, N, Ty, DAG, Subtarget);
774}
775
776// Fold sign-extensions into MipsISD::VEXTRACT_[SZ]EXT_ELT for MSA and fold
777// constant splats into MipsISD::SHRA_DSP for DSPr2.
778//
779// Performs the following transformations:
780// - Changes MipsISD::VEXTRACT_[SZ]EXT_ELT to sign extension if its
781//   sign/zero-extension is completely overwritten by the new one performed by
782//   the ISD::SRA and ISD::SHL nodes.
783// - Removes redundant sign extensions performed by an ISD::SRA and ISD::SHL
784//   sequence.
785//
786// See performDSPShiftCombine for more information about the transformation
787// used for DSPr2.
788static SDValue performSRACombine(SDNode *N, SelectionDAG &DAG,
789                                 TargetLowering::DAGCombinerInfo &DCI,
790                                 const MipsSubtarget *Subtarget) {
791  EVT Ty = N->getValueType(0);
792
793  if (Subtarget->hasMSA()) {
794    SDValue Op0 = N->getOperand(0);
795    SDValue Op1 = N->getOperand(1);
796
797    // (sra (shl (MipsVExtract[SZ]Ext $a, $b, $c), imm:$d), imm:$d)
798    // where $d + sizeof($c) == 32
799    // or    $d + sizeof($c) <= 32 and SExt
800    // -> (MipsVExtractSExt $a, $b, $c)
801    if (Op0->getOpcode() == ISD::SHL && Op1 == Op0->getOperand(1)) {
802      SDValue Op0Op0 = Op0->getOperand(0);
803      ConstantSDNode *ShAmount = dyn_cast<ConstantSDNode>(Op1);
804
805      if (!ShAmount)
806        return SDValue();
807
808      if (Op0Op0->getOpcode() != MipsISD::VEXTRACT_SEXT_ELT &&
809          Op0Op0->getOpcode() != MipsISD::VEXTRACT_ZEXT_ELT)
810        return SDValue();
811
812      EVT ExtendTy = cast<VTSDNode>(Op0Op0->getOperand(2))->getVT();
813      unsigned TotalBits = ShAmount->getZExtValue() + ExtendTy.getSizeInBits();
814
815      if (TotalBits == 32 ||
816          (Op0Op0->getOpcode() == MipsISD::VEXTRACT_SEXT_ELT &&
817           TotalBits <= 32)) {
818        SDValue Ops[] = { Op0Op0->getOperand(0), Op0Op0->getOperand(1),
819                          Op0Op0->getOperand(2) };
820        DAG.MorphNodeTo(Op0Op0.getNode(), MipsISD::VEXTRACT_SEXT_ELT,
821                        Op0Op0->getVTList(), Ops, Op0Op0->getNumOperands());
822        return Op0Op0;
823      }
824    }
825  }
826
827  if ((Ty != MVT::v2i16) && ((Ty != MVT::v4i8) || !Subtarget->hasDSPR2()))
828    return SDValue();
829
830  return performDSPShiftCombine(MipsISD::SHRA_DSP, N, Ty, DAG, Subtarget);
831}
832
833
834static SDValue performSRLCombine(SDNode *N, SelectionDAG &DAG,
835                                 TargetLowering::DAGCombinerInfo &DCI,
836                                 const MipsSubtarget *Subtarget) {
837  EVT Ty = N->getValueType(0);
838
839  if (((Ty != MVT::v2i16) || !Subtarget->hasDSPR2()) && (Ty != MVT::v4i8))
840    return SDValue();
841
842  return performDSPShiftCombine(MipsISD::SHRL_DSP, N, Ty, DAG, Subtarget);
843}
844
845static bool isLegalDSPCondCode(EVT Ty, ISD::CondCode CC) {
846  bool IsV216 = (Ty == MVT::v2i16);
847
848  switch (CC) {
849  case ISD::SETEQ:
850  case ISD::SETNE:  return true;
851  case ISD::SETLT:
852  case ISD::SETLE:
853  case ISD::SETGT:
854  case ISD::SETGE:  return IsV216;
855  case ISD::SETULT:
856  case ISD::SETULE:
857  case ISD::SETUGT:
858  case ISD::SETUGE: return !IsV216;
859  default:          return false;
860  }
861}
862
863static SDValue performSETCCCombine(SDNode *N, SelectionDAG &DAG) {
864  EVT Ty = N->getValueType(0);
865
866  if ((Ty != MVT::v2i16) && (Ty != MVT::v4i8))
867    return SDValue();
868
869  if (!isLegalDSPCondCode(Ty, cast<CondCodeSDNode>(N->getOperand(2))->get()))
870    return SDValue();
871
872  return DAG.getNode(MipsISD::SETCC_DSP, SDLoc(N), Ty, N->getOperand(0),
873                     N->getOperand(1), N->getOperand(2));
874}
875
876static SDValue performVSELECTCombine(SDNode *N, SelectionDAG &DAG) {
877  EVT Ty = N->getValueType(0);
878
879  if (Ty.is128BitVector() && Ty.isInteger()) {
880    // Try the following combines:
881    //   (vselect (setcc $a, $b, SETLT), $b, $a)) -> (vsmax $a, $b)
882    //   (vselect (setcc $a, $b, SETLE), $b, $a)) -> (vsmax $a, $b)
883    //   (vselect (setcc $a, $b, SETLT), $a, $b)) -> (vsmin $a, $b)
884    //   (vselect (setcc $a, $b, SETLE), $a, $b)) -> (vsmin $a, $b)
885    //   (vselect (setcc $a, $b, SETULT), $b, $a)) -> (vumax $a, $b)
886    //   (vselect (setcc $a, $b, SETULE), $b, $a)) -> (vumax $a, $b)
887    //   (vselect (setcc $a, $b, SETULT), $a, $b)) -> (vumin $a, $b)
888    //   (vselect (setcc $a, $b, SETULE), $a, $b)) -> (vumin $a, $b)
889    // SETGT/SETGE/SETUGT/SETUGE variants of these will show up initially but
890    // will be expanded to equivalent SETLT/SETLE/SETULT/SETULE versions by the
891    // legalizer.
892    SDValue Op0 = N->getOperand(0);
893
894    if (Op0->getOpcode() != ISD::SETCC)
895      return SDValue();
896
897    ISD::CondCode CondCode = cast<CondCodeSDNode>(Op0->getOperand(2))->get();
898    bool Signed;
899
900    if (CondCode == ISD::SETLT  || CondCode == ISD::SETLE)
901      Signed = true;
902    else if (CondCode == ISD::SETULT || CondCode == ISD::SETULE)
903      Signed = false;
904    else
905      return SDValue();
906
907    SDValue Op1 = N->getOperand(1);
908    SDValue Op2 = N->getOperand(2);
909    SDValue Op0Op0 = Op0->getOperand(0);
910    SDValue Op0Op1 = Op0->getOperand(1);
911
912    if (Op1 == Op0Op0 && Op2 == Op0Op1)
913      return DAG.getNode(Signed ? MipsISD::VSMIN : MipsISD::VUMIN, SDLoc(N),
914                         Ty, Op1, Op2);
915    else if (Op1 == Op0Op1 && Op2 == Op0Op0)
916      return DAG.getNode(Signed ? MipsISD::VSMAX : MipsISD::VUMAX, SDLoc(N),
917                         Ty, Op1, Op2);
918  } else if ((Ty == MVT::v2i16) || (Ty == MVT::v4i8)) {
919    SDValue SetCC = N->getOperand(0);
920
921    if (SetCC.getOpcode() != MipsISD::SETCC_DSP)
922      return SDValue();
923
924    return DAG.getNode(MipsISD::SELECT_CC_DSP, SDLoc(N), Ty,
925                       SetCC.getOperand(0), SetCC.getOperand(1),
926                       N->getOperand(1), N->getOperand(2), SetCC.getOperand(2));
927  }
928
929  return SDValue();
930}
931
932static SDValue performXORCombine(SDNode *N, SelectionDAG &DAG,
933                                 const MipsSubtarget *Subtarget) {
934  EVT Ty = N->getValueType(0);
935
936  if (Subtarget->hasMSA() && Ty.is128BitVector() && Ty.isInteger()) {
937    // Try the following combines:
938    //   (xor (or $a, $b), (build_vector allones))
939    //   (xor (or $a, $b), (bitcast (build_vector allones)))
940    SDValue Op0 = N->getOperand(0);
941    SDValue Op1 = N->getOperand(1);
942    SDValue NotOp;
943
944    if (ISD::isBuildVectorAllOnes(Op0.getNode()))
945      NotOp = Op1;
946    else if (ISD::isBuildVectorAllOnes(Op1.getNode()))
947      NotOp = Op0;
948    else
949      return SDValue();
950
951    if (NotOp->getOpcode() == ISD::OR)
952      return DAG.getNode(MipsISD::VNOR, SDLoc(N), Ty, NotOp->getOperand(0),
953                         NotOp->getOperand(1));
954  }
955
956  return SDValue();
957}
958
959SDValue
960MipsSETargetLowering::PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const {
961  SelectionDAG &DAG = DCI.DAG;
962  SDValue Val;
963
964  switch (N->getOpcode()) {
965  case ISD::ADDE:
966    return performADDECombine(N, DAG, DCI, Subtarget);
967  case ISD::AND:
968    Val = performANDCombine(N, DAG, DCI, Subtarget);
969    break;
970  case ISD::OR:
971    Val = performORCombine(N, DAG, DCI, Subtarget);
972    break;
973  case ISD::SUBE:
974    return performSUBECombine(N, DAG, DCI, Subtarget);
975  case ISD::MUL:
976    return performMULCombine(N, DAG, DCI, this);
977  case ISD::SHL:
978    return performSHLCombine(N, DAG, DCI, Subtarget);
979  case ISD::SRA:
980    return performSRACombine(N, DAG, DCI, Subtarget);
981  case ISD::SRL:
982    return performSRLCombine(N, DAG, DCI, Subtarget);
983  case ISD::VSELECT:
984    return performVSELECTCombine(N, DAG);
985  case ISD::XOR:
986    Val = performXORCombine(N, DAG, Subtarget);
987    break;
988  case ISD::SETCC:
989    Val = performSETCCCombine(N, DAG);
990    break;
991  }
992
993  if (Val.getNode()) {
994    DEBUG(dbgs() << "\nMipsSE DAG Combine:\n";
995          N->printrWithDepth(dbgs(), &DAG);
996          dbgs() << "\n=> \n";
997          Val.getNode()->printrWithDepth(dbgs(), &DAG);
998          dbgs() << "\n");
999    return Val;
1000  }
1001
1002  return MipsTargetLowering::PerformDAGCombine(N, DCI);
1003}
1004
1005MachineBasicBlock *
1006MipsSETargetLowering::EmitInstrWithCustomInserter(MachineInstr *MI,
1007                                                  MachineBasicBlock *BB) const {
1008  switch (MI->getOpcode()) {
1009  default:
1010    return MipsTargetLowering::EmitInstrWithCustomInserter(MI, BB);
1011  case Mips::BPOSGE32_PSEUDO:
1012    return emitBPOSGE32(MI, BB);
1013  case Mips::SNZ_B_PSEUDO:
1014    return emitMSACBranchPseudo(MI, BB, Mips::BNZ_B);
1015  case Mips::SNZ_H_PSEUDO:
1016    return emitMSACBranchPseudo(MI, BB, Mips::BNZ_H);
1017  case Mips::SNZ_W_PSEUDO:
1018    return emitMSACBranchPseudo(MI, BB, Mips::BNZ_W);
1019  case Mips::SNZ_D_PSEUDO:
1020    return emitMSACBranchPseudo(MI, BB, Mips::BNZ_D);
1021  case Mips::SNZ_V_PSEUDO:
1022    return emitMSACBranchPseudo(MI, BB, Mips::BNZ_V);
1023  case Mips::SZ_B_PSEUDO:
1024    return emitMSACBranchPseudo(MI, BB, Mips::BZ_B);
1025  case Mips::SZ_H_PSEUDO:
1026    return emitMSACBranchPseudo(MI, BB, Mips::BZ_H);
1027  case Mips::SZ_W_PSEUDO:
1028    return emitMSACBranchPseudo(MI, BB, Mips::BZ_W);
1029  case Mips::SZ_D_PSEUDO:
1030    return emitMSACBranchPseudo(MI, BB, Mips::BZ_D);
1031  case Mips::SZ_V_PSEUDO:
1032    return emitMSACBranchPseudo(MI, BB, Mips::BZ_V);
1033  case Mips::COPY_FW_PSEUDO:
1034    return emitCOPY_FW(MI, BB);
1035  case Mips::COPY_FD_PSEUDO:
1036    return emitCOPY_FD(MI, BB);
1037  case Mips::INSERT_FW_PSEUDO:
1038    return emitINSERT_FW(MI, BB);
1039  case Mips::INSERT_FD_PSEUDO:
1040    return emitINSERT_FD(MI, BB);
1041  case Mips::FILL_FW_PSEUDO:
1042    return emitFILL_FW(MI, BB);
1043  case Mips::FILL_FD_PSEUDO:
1044    return emitFILL_FD(MI, BB);
1045  case Mips::FEXP2_W_1_PSEUDO:
1046    return emitFEXP2_W_1(MI, BB);
1047  case Mips::FEXP2_D_1_PSEUDO:
1048    return emitFEXP2_D_1(MI, BB);
1049  }
1050}
1051
1052bool MipsSETargetLowering::
1053isEligibleForTailCallOptimization(const MipsCC &MipsCCInfo,
1054                                  unsigned NextStackOffset,
1055                                  const MipsFunctionInfo& FI) const {
1056  if (!EnableMipsTailCalls)
1057    return false;
1058
1059  // Return false if either the callee or caller has a byval argument.
1060  if (MipsCCInfo.hasByValArg() || FI.hasByvalArg())
1061    return false;
1062
1063  // Return true if the callee's argument area is no larger than the
1064  // caller's.
1065  return NextStackOffset <= FI.getIncomingArgSize();
1066}
1067
1068void MipsSETargetLowering::
1069getOpndList(SmallVectorImpl<SDValue> &Ops,
1070            std::deque< std::pair<unsigned, SDValue> > &RegsToPass,
1071            bool IsPICCall, bool GlobalOrExternal, bool InternalLinkage,
1072            CallLoweringInfo &CLI, SDValue Callee, SDValue Chain) const {
1073  // T9 should contain the address of the callee function if
1074  // -reloction-model=pic or it is an indirect call.
1075  if (IsPICCall || !GlobalOrExternal) {
1076    unsigned T9Reg = IsN64 ? Mips::T9_64 : Mips::T9;
1077    RegsToPass.push_front(std::make_pair(T9Reg, Callee));
1078  } else
1079    Ops.push_back(Callee);
1080
1081  MipsTargetLowering::getOpndList(Ops, RegsToPass, IsPICCall, GlobalOrExternal,
1082                                  InternalLinkage, CLI, Callee, Chain);
1083}
1084
1085SDValue MipsSETargetLowering::lowerLOAD(SDValue Op, SelectionDAG &DAG) const {
1086  LoadSDNode &Nd = *cast<LoadSDNode>(Op);
1087
1088  if (Nd.getMemoryVT() != MVT::f64 || !NoDPLoadStore)
1089    return MipsTargetLowering::lowerLOAD(Op, DAG);
1090
1091  // Replace a double precision load with two i32 loads and a buildpair64.
1092  SDLoc DL(Op);
1093  SDValue Ptr = Nd.getBasePtr(), Chain = Nd.getChain();
1094  EVT PtrVT = Ptr.getValueType();
1095
1096  // i32 load from lower address.
1097  SDValue Lo = DAG.getLoad(MVT::i32, DL, Chain, Ptr,
1098                           MachinePointerInfo(), Nd.isVolatile(),
1099                           Nd.isNonTemporal(), Nd.isInvariant(),
1100                           Nd.getAlignment());
1101
1102  // i32 load from higher address.
1103  Ptr = DAG.getNode(ISD::ADD, DL, PtrVT, Ptr, DAG.getConstant(4, PtrVT));
1104  SDValue Hi = DAG.getLoad(MVT::i32, DL, Lo.getValue(1), Ptr,
1105                           MachinePointerInfo(), Nd.isVolatile(),
1106                           Nd.isNonTemporal(), Nd.isInvariant(),
1107                           std::min(Nd.getAlignment(), 4U));
1108
1109  if (!Subtarget->isLittle())
1110    std::swap(Lo, Hi);
1111
1112  SDValue BP = DAG.getNode(MipsISD::BuildPairF64, DL, MVT::f64, Lo, Hi);
1113  SDValue Ops[2] = {BP, Hi.getValue(1)};
1114  return DAG.getMergeValues(Ops, 2, DL);
1115}
1116
1117SDValue MipsSETargetLowering::lowerSTORE(SDValue Op, SelectionDAG &DAG) const {
1118  StoreSDNode &Nd = *cast<StoreSDNode>(Op);
1119
1120  if (Nd.getMemoryVT() != MVT::f64 || !NoDPLoadStore)
1121    return MipsTargetLowering::lowerSTORE(Op, DAG);
1122
1123  // Replace a double precision store with two extractelement64s and i32 stores.
1124  SDLoc DL(Op);
1125  SDValue Val = Nd.getValue(), Ptr = Nd.getBasePtr(), Chain = Nd.getChain();
1126  EVT PtrVT = Ptr.getValueType();
1127  SDValue Lo = DAG.getNode(MipsISD::ExtractElementF64, DL, MVT::i32,
1128                           Val, DAG.getConstant(0, MVT::i32));
1129  SDValue Hi = DAG.getNode(MipsISD::ExtractElementF64, DL, MVT::i32,
1130                           Val, DAG.getConstant(1, MVT::i32));
1131
1132  if (!Subtarget->isLittle())
1133    std::swap(Lo, Hi);
1134
1135  // i32 store to lower address.
1136  Chain = DAG.getStore(Chain, DL, Lo, Ptr, MachinePointerInfo(),
1137                       Nd.isVolatile(), Nd.isNonTemporal(), Nd.getAlignment(),
1138                       Nd.getTBAAInfo());
1139
1140  // i32 store to higher address.
1141  Ptr = DAG.getNode(ISD::ADD, DL, PtrVT, Ptr, DAG.getConstant(4, PtrVT));
1142  return DAG.getStore(Chain, DL, Hi, Ptr, MachinePointerInfo(),
1143                      Nd.isVolatile(), Nd.isNonTemporal(),
1144                      std::min(Nd.getAlignment(), 4U), Nd.getTBAAInfo());
1145}
1146
1147SDValue MipsSETargetLowering::lowerMulDiv(SDValue Op, unsigned NewOpc,
1148                                          bool HasLo, bool HasHi,
1149                                          SelectionDAG &DAG) const {
1150  EVT Ty = Op.getOperand(0).getValueType();
1151  SDLoc DL(Op);
1152  SDValue Mult = DAG.getNode(NewOpc, DL, MVT::Untyped,
1153                             Op.getOperand(0), Op.getOperand(1));
1154  SDValue Lo, Hi;
1155
1156  if (HasLo)
1157    Lo = DAG.getNode(MipsISD::MFLO, DL, Ty, Mult);
1158  if (HasHi)
1159    Hi = DAG.getNode(MipsISD::MFHI, DL, Ty, Mult);
1160
1161  if (!HasLo || !HasHi)
1162    return HasLo ? Lo : Hi;
1163
1164  SDValue Vals[] = { Lo, Hi };
1165  return DAG.getMergeValues(Vals, 2, DL);
1166}
1167
1168
1169static SDValue initAccumulator(SDValue In, SDLoc DL, SelectionDAG &DAG) {
1170  SDValue InLo = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i32, In,
1171                             DAG.getConstant(0, MVT::i32));
1172  SDValue InHi = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i32, In,
1173                             DAG.getConstant(1, MVT::i32));
1174  return DAG.getNode(MipsISD::MTLOHI, DL, MVT::Untyped, InLo, InHi);
1175}
1176
1177static SDValue extractLOHI(SDValue Op, SDLoc DL, SelectionDAG &DAG) {
1178  SDValue Lo = DAG.getNode(MipsISD::MFLO, DL, MVT::i32, Op);
1179  SDValue Hi = DAG.getNode(MipsISD::MFHI, DL, MVT::i32, Op);
1180  return DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i64, Lo, Hi);
1181}
1182
1183// This function expands mips intrinsic nodes which have 64-bit input operands
1184// or output values.
1185//
1186// out64 = intrinsic-node in64
1187// =>
1188// lo = copy (extract-element (in64, 0))
1189// hi = copy (extract-element (in64, 1))
1190// mips-specific-node
1191// v0 = copy lo
1192// v1 = copy hi
1193// out64 = merge-values (v0, v1)
1194//
1195static SDValue lowerDSPIntr(SDValue Op, SelectionDAG &DAG, unsigned Opc) {
1196  SDLoc DL(Op);
1197  bool HasChainIn = Op->getOperand(0).getValueType() == MVT::Other;
1198  SmallVector<SDValue, 3> Ops;
1199  unsigned OpNo = 0;
1200
1201  // See if Op has a chain input.
1202  if (HasChainIn)
1203    Ops.push_back(Op->getOperand(OpNo++));
1204
1205  // The next operand is the intrinsic opcode.
1206  assert(Op->getOperand(OpNo).getOpcode() == ISD::TargetConstant);
1207
1208  // See if the next operand has type i64.
1209  SDValue Opnd = Op->getOperand(++OpNo), In64;
1210
1211  if (Opnd.getValueType() == MVT::i64)
1212    In64 = initAccumulator(Opnd, DL, DAG);
1213  else
1214    Ops.push_back(Opnd);
1215
1216  // Push the remaining operands.
1217  for (++OpNo ; OpNo < Op->getNumOperands(); ++OpNo)
1218    Ops.push_back(Op->getOperand(OpNo));
1219
1220  // Add In64 to the end of the list.
1221  if (In64.getNode())
1222    Ops.push_back(In64);
1223
1224  // Scan output.
1225  SmallVector<EVT, 2> ResTys;
1226
1227  for (SDNode::value_iterator I = Op->value_begin(), E = Op->value_end();
1228       I != E; ++I)
1229    ResTys.push_back((*I == MVT::i64) ? MVT::Untyped : *I);
1230
1231  // Create node.
1232  SDValue Val = DAG.getNode(Opc, DL, ResTys, &Ops[0], Ops.size());
1233  SDValue Out = (ResTys[0] == MVT::Untyped) ? extractLOHI(Val, DL, DAG) : Val;
1234
1235  if (!HasChainIn)
1236    return Out;
1237
1238  assert(Val->getValueType(1) == MVT::Other);
1239  SDValue Vals[] = { Out, SDValue(Val.getNode(), 1) };
1240  return DAG.getMergeValues(Vals, 2, DL);
1241}
1242
1243// Lower an MSA copy intrinsic into the specified SelectionDAG node
1244static SDValue lowerMSACopyIntr(SDValue Op, SelectionDAG &DAG, unsigned Opc) {
1245  SDLoc DL(Op);
1246  SDValue Vec = Op->getOperand(1);
1247  SDValue Idx = Op->getOperand(2);
1248  EVT ResTy = Op->getValueType(0);
1249  EVT EltTy = Vec->getValueType(0).getVectorElementType();
1250
1251  SDValue Result = DAG.getNode(Opc, DL, ResTy, Vec, Idx,
1252                               DAG.getValueType(EltTy));
1253
1254  return Result;
1255}
1256
1257static SDValue
1258lowerMSASplatImm(SDLoc DL, EVT ResTy, SDValue ImmOp, SelectionDAG &DAG) {
1259  EVT ViaVecTy = ResTy;
1260  SmallVector<SDValue, 16> Ops;
1261  SDValue ImmHiOp;
1262
1263  if (ViaVecTy == MVT::v2i64) {
1264    ImmHiOp = DAG.getNode(ISD::SRA, DL, MVT::i32, ImmOp,
1265                          DAG.getConstant(31, MVT::i32));
1266    for (unsigned i = 0; i < ViaVecTy.getVectorNumElements(); ++i) {
1267      Ops.push_back(ImmHiOp);
1268      Ops.push_back(ImmOp);
1269    }
1270    ViaVecTy = MVT::v4i32;
1271  } else {
1272    for (unsigned i = 0; i < ResTy.getVectorNumElements(); ++i)
1273      Ops.push_back(ImmOp);
1274  }
1275
1276  SDValue Result = DAG.getNode(ISD::BUILD_VECTOR, DL, ViaVecTy, &Ops[0],
1277                               Ops.size());
1278
1279  if (ResTy != ViaVecTy)
1280    Result = DAG.getNode(ISD::BITCAST, DL, ResTy, Result);
1281
1282  return Result;
1283}
1284
1285static SDValue
1286lowerMSASplatImm(SDValue Op, unsigned ImmOp, SelectionDAG &DAG) {
1287  return lowerMSASplatImm(SDLoc(Op), Op->getValueType(0),
1288                          Op->getOperand(ImmOp), DAG);
1289}
1290
1291static SDValue lowerMSABinaryBitImmIntr(SDValue Op, SelectionDAG &DAG,
1292                                        unsigned Opc, SDValue Imm,
1293                                        bool BigEndian) {
1294  EVT VecTy = Op->getValueType(0);
1295  SDValue Exp2Imm;
1296  SDLoc DL(Op);
1297
1298  // The DAG Combiner can't constant fold bitcasted vectors so we must do it
1299  // here.
1300  if (VecTy == MVT::v2i64) {
1301    if (ConstantSDNode *CImm = dyn_cast<ConstantSDNode>(Imm)) {
1302      APInt BitImm = APInt(64, 1) << CImm->getAPIntValue();
1303
1304      SDValue BitImmHiOp = DAG.getConstant(BitImm.lshr(32).trunc(32), MVT::i32);
1305      SDValue BitImmOp = DAG.getConstant(BitImm.trunc(32), MVT::i32);
1306      Exp2Imm = DAG.getNode(ISD::BITCAST, DL, MVT::v2i64,
1307                            DAG.getNode(ISD::BUILD_VECTOR, DL, MVT::v4i32,
1308                                        BitImmHiOp, BitImmOp,
1309                                        BitImmHiOp, BitImmOp));
1310    }
1311  }
1312
1313  if (Exp2Imm.getNode() == NULL) {
1314    // We couldnt constant fold, do a vector shift instead
1315    SDValue One = lowerMSASplatImm(DL, VecTy, DAG.getConstant(1, MVT::i32),
1316                                   DAG);
1317    Exp2Imm = lowerMSASplatImm(DL, VecTy, Imm, DAG);
1318    Exp2Imm = DAG.getNode(ISD::SHL, DL, VecTy, One, Exp2Imm);
1319  }
1320
1321  return DAG.getNode(Opc, DL, VecTy, Op->getOperand(1), Exp2Imm);
1322}
1323
1324SDValue MipsSETargetLowering::lowerINTRINSIC_WO_CHAIN(SDValue Op,
1325                                                      SelectionDAG &DAG) const {
1326  SDLoc DL(Op);
1327
1328  switch (cast<ConstantSDNode>(Op->getOperand(0))->getZExtValue()) {
1329  default:
1330    return SDValue();
1331  case Intrinsic::mips_shilo:
1332    return lowerDSPIntr(Op, DAG, MipsISD::SHILO);
1333  case Intrinsic::mips_dpau_h_qbl:
1334    return lowerDSPIntr(Op, DAG, MipsISD::DPAU_H_QBL);
1335  case Intrinsic::mips_dpau_h_qbr:
1336    return lowerDSPIntr(Op, DAG, MipsISD::DPAU_H_QBR);
1337  case Intrinsic::mips_dpsu_h_qbl:
1338    return lowerDSPIntr(Op, DAG, MipsISD::DPSU_H_QBL);
1339  case Intrinsic::mips_dpsu_h_qbr:
1340    return lowerDSPIntr(Op, DAG, MipsISD::DPSU_H_QBR);
1341  case Intrinsic::mips_dpa_w_ph:
1342    return lowerDSPIntr(Op, DAG, MipsISD::DPA_W_PH);
1343  case Intrinsic::mips_dps_w_ph:
1344    return lowerDSPIntr(Op, DAG, MipsISD::DPS_W_PH);
1345  case Intrinsic::mips_dpax_w_ph:
1346    return lowerDSPIntr(Op, DAG, MipsISD::DPAX_W_PH);
1347  case Intrinsic::mips_dpsx_w_ph:
1348    return lowerDSPIntr(Op, DAG, MipsISD::DPSX_W_PH);
1349  case Intrinsic::mips_mulsa_w_ph:
1350    return lowerDSPIntr(Op, DAG, MipsISD::MULSA_W_PH);
1351  case Intrinsic::mips_mult:
1352    return lowerDSPIntr(Op, DAG, MipsISD::Mult);
1353  case Intrinsic::mips_multu:
1354    return lowerDSPIntr(Op, DAG, MipsISD::Multu);
1355  case Intrinsic::mips_madd:
1356    return lowerDSPIntr(Op, DAG, MipsISD::MAdd);
1357  case Intrinsic::mips_maddu:
1358    return lowerDSPIntr(Op, DAG, MipsISD::MAddu);
1359  case Intrinsic::mips_msub:
1360    return lowerDSPIntr(Op, DAG, MipsISD::MSub);
1361  case Intrinsic::mips_msubu:
1362    return lowerDSPIntr(Op, DAG, MipsISD::MSubu);
1363  case Intrinsic::mips_addv_b:
1364  case Intrinsic::mips_addv_h:
1365  case Intrinsic::mips_addv_w:
1366  case Intrinsic::mips_addv_d:
1367    return DAG.getNode(ISD::ADD, DL, Op->getValueType(0), Op->getOperand(1),
1368                       Op->getOperand(2));
1369  case Intrinsic::mips_addvi_b:
1370  case Intrinsic::mips_addvi_h:
1371  case Intrinsic::mips_addvi_w:
1372  case Intrinsic::mips_addvi_d:
1373    return DAG.getNode(ISD::ADD, DL, Op->getValueType(0), Op->getOperand(1),
1374                       lowerMSASplatImm(Op, 2, DAG));
1375  case Intrinsic::mips_and_v:
1376    return DAG.getNode(ISD::AND, DL, Op->getValueType(0), Op->getOperand(1),
1377                       Op->getOperand(2));
1378  case Intrinsic::mips_andi_b:
1379    return DAG.getNode(ISD::AND, DL, Op->getValueType(0), Op->getOperand(1),
1380                       lowerMSASplatImm(Op, 2, DAG));
1381  case Intrinsic::mips_binsli_b:
1382  case Intrinsic::mips_binsli_h:
1383  case Intrinsic::mips_binsli_w:
1384  case Intrinsic::mips_binsli_d: {
1385    EVT VecTy = Op->getValueType(0);
1386    EVT EltTy = VecTy.getVectorElementType();
1387    APInt Mask = APInt::getHighBitsSet(EltTy.getSizeInBits(),
1388                                       Op->getConstantOperandVal(3));
1389    return DAG.getNode(ISD::VSELECT, DL, VecTy,
1390                       DAG.getConstant(Mask, VecTy, true), Op->getOperand(1),
1391                       Op->getOperand(2));
1392  }
1393  case Intrinsic::mips_binsri_b:
1394  case Intrinsic::mips_binsri_h:
1395  case Intrinsic::mips_binsri_w:
1396  case Intrinsic::mips_binsri_d: {
1397    EVT VecTy = Op->getValueType(0);
1398    EVT EltTy = VecTy.getVectorElementType();
1399    APInt Mask = APInt::getLowBitsSet(EltTy.getSizeInBits(),
1400                                      Op->getConstantOperandVal(3));
1401    return DAG.getNode(ISD::VSELECT, DL, VecTy,
1402                       DAG.getConstant(Mask, VecTy, true), Op->getOperand(1),
1403                       Op->getOperand(2));
1404  }
1405  case Intrinsic::mips_bmnz_v:
1406    return DAG.getNode(ISD::VSELECT, DL, Op->getValueType(0), Op->getOperand(3),
1407                       Op->getOperand(2), Op->getOperand(1));
1408  case Intrinsic::mips_bmnzi_b:
1409    return DAG.getNode(ISD::VSELECT, DL, Op->getValueType(0),
1410                       lowerMSASplatImm(Op, 3, DAG), Op->getOperand(2),
1411                       Op->getOperand(1));
1412  case Intrinsic::mips_bmz_v:
1413    return DAG.getNode(ISD::VSELECT, DL, Op->getValueType(0), Op->getOperand(3),
1414                       Op->getOperand(1), Op->getOperand(2));
1415  case Intrinsic::mips_bmzi_b:
1416    return DAG.getNode(ISD::VSELECT, DL, Op->getValueType(0),
1417                       lowerMSASplatImm(Op, 3, DAG), Op->getOperand(1),
1418                       Op->getOperand(2));
1419  case Intrinsic::mips_bneg_b:
1420  case Intrinsic::mips_bneg_h:
1421  case Intrinsic::mips_bneg_w:
1422  case Intrinsic::mips_bneg_d: {
1423    EVT VecTy = Op->getValueType(0);
1424    SDValue One = lowerMSASplatImm(DL, VecTy, DAG.getConstant(1, MVT::i32),
1425                                   DAG);
1426
1427    return DAG.getNode(ISD::XOR, DL, VecTy, Op->getOperand(1),
1428                       DAG.getNode(ISD::SHL, DL, VecTy, One,
1429                                   Op->getOperand(2)));
1430  }
1431  case Intrinsic::mips_bnegi_b:
1432  case Intrinsic::mips_bnegi_h:
1433  case Intrinsic::mips_bnegi_w:
1434  case Intrinsic::mips_bnegi_d:
1435    return lowerMSABinaryBitImmIntr(Op, DAG, ISD::XOR, Op->getOperand(2),
1436                                    !Subtarget->isLittle());
1437  case Intrinsic::mips_bnz_b:
1438  case Intrinsic::mips_bnz_h:
1439  case Intrinsic::mips_bnz_w:
1440  case Intrinsic::mips_bnz_d:
1441    return DAG.getNode(MipsISD::VALL_NONZERO, DL, Op->getValueType(0),
1442                       Op->getOperand(1));
1443  case Intrinsic::mips_bnz_v:
1444    return DAG.getNode(MipsISD::VANY_NONZERO, DL, Op->getValueType(0),
1445                       Op->getOperand(1));
1446  case Intrinsic::mips_bsel_v:
1447    return DAG.getNode(ISD::VSELECT, DL, Op->getValueType(0),
1448                       Op->getOperand(1), Op->getOperand(2),
1449                       Op->getOperand(3));
1450  case Intrinsic::mips_bseli_b:
1451    return DAG.getNode(ISD::VSELECT, DL, Op->getValueType(0),
1452                       Op->getOperand(1), Op->getOperand(2),
1453                       lowerMSASplatImm(Op, 3, DAG));
1454  case Intrinsic::mips_bset_b:
1455  case Intrinsic::mips_bset_h:
1456  case Intrinsic::mips_bset_w:
1457  case Intrinsic::mips_bset_d: {
1458    EVT VecTy = Op->getValueType(0);
1459    SDValue One = lowerMSASplatImm(DL, VecTy, DAG.getConstant(1, MVT::i32),
1460                                   DAG);
1461
1462    return DAG.getNode(ISD::OR, DL, VecTy, Op->getOperand(1),
1463                       DAG.getNode(ISD::SHL, DL, VecTy, One,
1464                                   Op->getOperand(2)));
1465  }
1466  case Intrinsic::mips_bseti_b:
1467  case Intrinsic::mips_bseti_h:
1468  case Intrinsic::mips_bseti_w:
1469  case Intrinsic::mips_bseti_d:
1470    return lowerMSABinaryBitImmIntr(Op, DAG, ISD::OR, Op->getOperand(2),
1471                                    !Subtarget->isLittle());
1472  case Intrinsic::mips_bz_b:
1473  case Intrinsic::mips_bz_h:
1474  case Intrinsic::mips_bz_w:
1475  case Intrinsic::mips_bz_d:
1476    return DAG.getNode(MipsISD::VALL_ZERO, DL, Op->getValueType(0),
1477                       Op->getOperand(1));
1478  case Intrinsic::mips_bz_v:
1479    return DAG.getNode(MipsISD::VANY_ZERO, DL, Op->getValueType(0),
1480                       Op->getOperand(1));
1481  case Intrinsic::mips_ceq_b:
1482  case Intrinsic::mips_ceq_h:
1483  case Intrinsic::mips_ceq_w:
1484  case Intrinsic::mips_ceq_d:
1485    return DAG.getSetCC(DL, Op->getValueType(0), Op->getOperand(1),
1486                        Op->getOperand(2), ISD::SETEQ);
1487  case Intrinsic::mips_ceqi_b:
1488  case Intrinsic::mips_ceqi_h:
1489  case Intrinsic::mips_ceqi_w:
1490  case Intrinsic::mips_ceqi_d:
1491    return DAG.getSetCC(DL, Op->getValueType(0), Op->getOperand(1),
1492                        lowerMSASplatImm(Op, 2, DAG), ISD::SETEQ);
1493  case Intrinsic::mips_cle_s_b:
1494  case Intrinsic::mips_cle_s_h:
1495  case Intrinsic::mips_cle_s_w:
1496  case Intrinsic::mips_cle_s_d:
1497    return DAG.getSetCC(DL, Op->getValueType(0), Op->getOperand(1),
1498                        Op->getOperand(2), ISD::SETLE);
1499  case Intrinsic::mips_clei_s_b:
1500  case Intrinsic::mips_clei_s_h:
1501  case Intrinsic::mips_clei_s_w:
1502  case Intrinsic::mips_clei_s_d:
1503    return DAG.getSetCC(DL, Op->getValueType(0), Op->getOperand(1),
1504                        lowerMSASplatImm(Op, 2, DAG), ISD::SETLE);
1505  case Intrinsic::mips_cle_u_b:
1506  case Intrinsic::mips_cle_u_h:
1507  case Intrinsic::mips_cle_u_w:
1508  case Intrinsic::mips_cle_u_d:
1509    return DAG.getSetCC(DL, Op->getValueType(0), Op->getOperand(1),
1510                        Op->getOperand(2), ISD::SETULE);
1511  case Intrinsic::mips_clei_u_b:
1512  case Intrinsic::mips_clei_u_h:
1513  case Intrinsic::mips_clei_u_w:
1514  case Intrinsic::mips_clei_u_d:
1515    return DAG.getSetCC(DL, Op->getValueType(0), Op->getOperand(1),
1516                        lowerMSASplatImm(Op, 2, DAG), ISD::SETULE);
1517  case Intrinsic::mips_clt_s_b:
1518  case Intrinsic::mips_clt_s_h:
1519  case Intrinsic::mips_clt_s_w:
1520  case Intrinsic::mips_clt_s_d:
1521    return DAG.getSetCC(DL, Op->getValueType(0), Op->getOperand(1),
1522                        Op->getOperand(2), ISD::SETLT);
1523  case Intrinsic::mips_clti_s_b:
1524  case Intrinsic::mips_clti_s_h:
1525  case Intrinsic::mips_clti_s_w:
1526  case Intrinsic::mips_clti_s_d:
1527    return DAG.getSetCC(DL, Op->getValueType(0), Op->getOperand(1),
1528                        lowerMSASplatImm(Op, 2, DAG), ISD::SETLT);
1529  case Intrinsic::mips_clt_u_b:
1530  case Intrinsic::mips_clt_u_h:
1531  case Intrinsic::mips_clt_u_w:
1532  case Intrinsic::mips_clt_u_d:
1533    return DAG.getSetCC(DL, Op->getValueType(0), Op->getOperand(1),
1534                        Op->getOperand(2), ISD::SETULT);
1535  case Intrinsic::mips_clti_u_b:
1536  case Intrinsic::mips_clti_u_h:
1537  case Intrinsic::mips_clti_u_w:
1538  case Intrinsic::mips_clti_u_d:
1539    return DAG.getSetCC(DL, Op->getValueType(0), Op->getOperand(1),
1540                        lowerMSASplatImm(Op, 2, DAG), ISD::SETULT);
1541  case Intrinsic::mips_copy_s_b:
1542  case Intrinsic::mips_copy_s_h:
1543  case Intrinsic::mips_copy_s_w:
1544    return lowerMSACopyIntr(Op, DAG, MipsISD::VEXTRACT_SEXT_ELT);
1545  case Intrinsic::mips_copy_s_d:
1546    // Don't lower directly into VEXTRACT_SEXT_ELT since i64 might be illegal.
1547    // Instead lower to the generic EXTRACT_VECTOR_ELT node and let the type
1548    // legalizer and EXTRACT_VECTOR_ELT lowering sort it out.
1549    return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(Op), Op->getValueType(0),
1550                       Op->getOperand(1), Op->getOperand(2));
1551  case Intrinsic::mips_copy_u_b:
1552  case Intrinsic::mips_copy_u_h:
1553  case Intrinsic::mips_copy_u_w:
1554    return lowerMSACopyIntr(Op, DAG, MipsISD::VEXTRACT_ZEXT_ELT);
1555  case Intrinsic::mips_copy_u_d:
1556    // Don't lower directly into VEXTRACT_ZEXT_ELT since i64 might be illegal.
1557    // Instead lower to the generic EXTRACT_VECTOR_ELT node and let the type
1558    // legalizer and EXTRACT_VECTOR_ELT lowering sort it out.
1559    //
1560    // Note: When i64 is illegal, this results in copy_s.w instructions instead
1561    // of copy_u.w instructions. This makes no difference to the behaviour
1562    // since i64 is only illegal when the register file is 32-bit.
1563    return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(Op), Op->getValueType(0),
1564                       Op->getOperand(1), Op->getOperand(2));
1565  case Intrinsic::mips_div_s_b:
1566  case Intrinsic::mips_div_s_h:
1567  case Intrinsic::mips_div_s_w:
1568  case Intrinsic::mips_div_s_d:
1569    return DAG.getNode(ISD::SDIV, DL, Op->getValueType(0), Op->getOperand(1),
1570                       Op->getOperand(2));
1571  case Intrinsic::mips_div_u_b:
1572  case Intrinsic::mips_div_u_h:
1573  case Intrinsic::mips_div_u_w:
1574  case Intrinsic::mips_div_u_d:
1575    return DAG.getNode(ISD::UDIV, DL, Op->getValueType(0), Op->getOperand(1),
1576                       Op->getOperand(2));
1577  case Intrinsic::mips_fadd_w:
1578  case Intrinsic::mips_fadd_d:
1579    return DAG.getNode(ISD::FADD, DL, Op->getValueType(0), Op->getOperand(1),
1580                       Op->getOperand(2));
1581  // Don't lower mips_fcaf_[wd] since LLVM folds SETFALSE condcodes away
1582  case Intrinsic::mips_fceq_w:
1583  case Intrinsic::mips_fceq_d:
1584    return DAG.getSetCC(DL, Op->getValueType(0), Op->getOperand(1),
1585                        Op->getOperand(2), ISD::SETOEQ);
1586  case Intrinsic::mips_fcle_w:
1587  case Intrinsic::mips_fcle_d:
1588    return DAG.getSetCC(DL, Op->getValueType(0), Op->getOperand(1),
1589                        Op->getOperand(2), ISD::SETOLE);
1590  case Intrinsic::mips_fclt_w:
1591  case Intrinsic::mips_fclt_d:
1592    return DAG.getSetCC(DL, Op->getValueType(0), Op->getOperand(1),
1593                        Op->getOperand(2), ISD::SETOLT);
1594  case Intrinsic::mips_fcne_w:
1595  case Intrinsic::mips_fcne_d:
1596    return DAG.getSetCC(DL, Op->getValueType(0), Op->getOperand(1),
1597                        Op->getOperand(2), ISD::SETONE);
1598  case Intrinsic::mips_fcor_w:
1599  case Intrinsic::mips_fcor_d:
1600    return DAG.getSetCC(DL, Op->getValueType(0), Op->getOperand(1),
1601                        Op->getOperand(2), ISD::SETO);
1602  case Intrinsic::mips_fcueq_w:
1603  case Intrinsic::mips_fcueq_d:
1604    return DAG.getSetCC(DL, Op->getValueType(0), Op->getOperand(1),
1605                        Op->getOperand(2), ISD::SETUEQ);
1606  case Intrinsic::mips_fcule_w:
1607  case Intrinsic::mips_fcule_d:
1608    return DAG.getSetCC(DL, Op->getValueType(0), Op->getOperand(1),
1609                        Op->getOperand(2), ISD::SETULE);
1610  case Intrinsic::mips_fcult_w:
1611  case Intrinsic::mips_fcult_d:
1612    return DAG.getSetCC(DL, Op->getValueType(0), Op->getOperand(1),
1613                        Op->getOperand(2), ISD::SETULT);
1614  case Intrinsic::mips_fcun_w:
1615  case Intrinsic::mips_fcun_d:
1616    return DAG.getSetCC(DL, Op->getValueType(0), Op->getOperand(1),
1617                        Op->getOperand(2), ISD::SETUO);
1618  case Intrinsic::mips_fcune_w:
1619  case Intrinsic::mips_fcune_d:
1620    return DAG.getSetCC(DL, Op->getValueType(0), Op->getOperand(1),
1621                        Op->getOperand(2), ISD::SETUNE);
1622  case Intrinsic::mips_fdiv_w:
1623  case Intrinsic::mips_fdiv_d:
1624    return DAG.getNode(ISD::FDIV, DL, Op->getValueType(0), Op->getOperand(1),
1625                       Op->getOperand(2));
1626  case Intrinsic::mips_ffint_u_w:
1627  case Intrinsic::mips_ffint_u_d:
1628    return DAG.getNode(ISD::UINT_TO_FP, DL, Op->getValueType(0),
1629                       Op->getOperand(1));
1630  case Intrinsic::mips_ffint_s_w:
1631  case Intrinsic::mips_ffint_s_d:
1632    return DAG.getNode(ISD::SINT_TO_FP, DL, Op->getValueType(0),
1633                       Op->getOperand(1));
1634  case Intrinsic::mips_fill_b:
1635  case Intrinsic::mips_fill_h:
1636  case Intrinsic::mips_fill_w:
1637  case Intrinsic::mips_fill_d: {
1638    SmallVector<SDValue, 16> Ops;
1639    EVT ResTy = Op->getValueType(0);
1640
1641    for (unsigned i = 0; i < ResTy.getVectorNumElements(); ++i)
1642      Ops.push_back(Op->getOperand(1));
1643
1644    // If ResTy is v2i64 then the type legalizer will break this node down into
1645    // an equivalent v4i32.
1646    return DAG.getNode(ISD::BUILD_VECTOR, DL, ResTy, &Ops[0], Ops.size());
1647  }
1648  case Intrinsic::mips_fexp2_w:
1649  case Intrinsic::mips_fexp2_d: {
1650    EVT ResTy = Op->getValueType(0);
1651    return DAG.getNode(
1652        ISD::FMUL, SDLoc(Op), ResTy, Op->getOperand(1),
1653        DAG.getNode(ISD::FEXP2, SDLoc(Op), ResTy, Op->getOperand(2)));
1654  }
1655  case Intrinsic::mips_flog2_w:
1656  case Intrinsic::mips_flog2_d:
1657    return DAG.getNode(ISD::FLOG2, DL, Op->getValueType(0), Op->getOperand(1));
1658  case Intrinsic::mips_fmadd_w:
1659  case Intrinsic::mips_fmadd_d:
1660    return DAG.getNode(ISD::FMA, SDLoc(Op), Op->getValueType(0),
1661                       Op->getOperand(1), Op->getOperand(2), Op->getOperand(3));
1662  case Intrinsic::mips_fmul_w:
1663  case Intrinsic::mips_fmul_d:
1664    return DAG.getNode(ISD::FMUL, DL, Op->getValueType(0), Op->getOperand(1),
1665                       Op->getOperand(2));
1666  case Intrinsic::mips_fmsub_w:
1667  case Intrinsic::mips_fmsub_d: {
1668    EVT ResTy = Op->getValueType(0);
1669    return DAG.getNode(ISD::FSUB, SDLoc(Op), ResTy, Op->getOperand(1),
1670                       DAG.getNode(ISD::FMUL, SDLoc(Op), ResTy,
1671                                   Op->getOperand(2), Op->getOperand(3)));
1672  }
1673  case Intrinsic::mips_frint_w:
1674  case Intrinsic::mips_frint_d:
1675    return DAG.getNode(ISD::FRINT, DL, Op->getValueType(0), Op->getOperand(1));
1676  case Intrinsic::mips_fsqrt_w:
1677  case Intrinsic::mips_fsqrt_d:
1678    return DAG.getNode(ISD::FSQRT, DL, Op->getValueType(0), Op->getOperand(1));
1679  case Intrinsic::mips_fsub_w:
1680  case Intrinsic::mips_fsub_d:
1681    return DAG.getNode(ISD::FSUB, DL, Op->getValueType(0), Op->getOperand(1),
1682                       Op->getOperand(2));
1683  case Intrinsic::mips_ftrunc_u_w:
1684  case Intrinsic::mips_ftrunc_u_d:
1685    return DAG.getNode(ISD::FP_TO_UINT, DL, Op->getValueType(0),
1686                       Op->getOperand(1));
1687  case Intrinsic::mips_ftrunc_s_w:
1688  case Intrinsic::mips_ftrunc_s_d:
1689    return DAG.getNode(ISD::FP_TO_SINT, DL, Op->getValueType(0),
1690                       Op->getOperand(1));
1691  case Intrinsic::mips_ilvev_b:
1692  case Intrinsic::mips_ilvev_h:
1693  case Intrinsic::mips_ilvev_w:
1694  case Intrinsic::mips_ilvev_d:
1695    return DAG.getNode(MipsISD::ILVEV, DL, Op->getValueType(0),
1696                       Op->getOperand(1), Op->getOperand(2));
1697  case Intrinsic::mips_ilvl_b:
1698  case Intrinsic::mips_ilvl_h:
1699  case Intrinsic::mips_ilvl_w:
1700  case Intrinsic::mips_ilvl_d:
1701    return DAG.getNode(MipsISD::ILVL, DL, Op->getValueType(0),
1702                       Op->getOperand(1), Op->getOperand(2));
1703  case Intrinsic::mips_ilvod_b:
1704  case Intrinsic::mips_ilvod_h:
1705  case Intrinsic::mips_ilvod_w:
1706  case Intrinsic::mips_ilvod_d:
1707    return DAG.getNode(MipsISD::ILVOD, DL, Op->getValueType(0),
1708                       Op->getOperand(1), Op->getOperand(2));
1709  case Intrinsic::mips_ilvr_b:
1710  case Intrinsic::mips_ilvr_h:
1711  case Intrinsic::mips_ilvr_w:
1712  case Intrinsic::mips_ilvr_d:
1713    return DAG.getNode(MipsISD::ILVR, DL, Op->getValueType(0),
1714                       Op->getOperand(1), Op->getOperand(2));
1715  case Intrinsic::mips_insert_b:
1716  case Intrinsic::mips_insert_h:
1717  case Intrinsic::mips_insert_w:
1718  case Intrinsic::mips_insert_d:
1719    return DAG.getNode(ISD::INSERT_VECTOR_ELT, SDLoc(Op), Op->getValueType(0),
1720                       Op->getOperand(1), Op->getOperand(3), Op->getOperand(2));
1721  case Intrinsic::mips_ldi_b:
1722  case Intrinsic::mips_ldi_h:
1723  case Intrinsic::mips_ldi_w:
1724  case Intrinsic::mips_ldi_d:
1725    return lowerMSASplatImm(Op, 1, DAG);
1726  case Intrinsic::mips_lsa: {
1727    EVT ResTy = Op->getValueType(0);
1728    return DAG.getNode(ISD::ADD, SDLoc(Op), ResTy, Op->getOperand(1),
1729                       DAG.getNode(ISD::SHL, SDLoc(Op), ResTy,
1730                                   Op->getOperand(2), Op->getOperand(3)));
1731  }
1732  case Intrinsic::mips_maddv_b:
1733  case Intrinsic::mips_maddv_h:
1734  case Intrinsic::mips_maddv_w:
1735  case Intrinsic::mips_maddv_d: {
1736    EVT ResTy = Op->getValueType(0);
1737    return DAG.getNode(ISD::ADD, SDLoc(Op), ResTy, Op->getOperand(1),
1738                       DAG.getNode(ISD::MUL, SDLoc(Op), ResTy,
1739                                   Op->getOperand(2), Op->getOperand(3)));
1740  }
1741  case Intrinsic::mips_max_s_b:
1742  case Intrinsic::mips_max_s_h:
1743  case Intrinsic::mips_max_s_w:
1744  case Intrinsic::mips_max_s_d:
1745    return DAG.getNode(MipsISD::VSMAX, DL, Op->getValueType(0),
1746                       Op->getOperand(1), Op->getOperand(2));
1747  case Intrinsic::mips_max_u_b:
1748  case Intrinsic::mips_max_u_h:
1749  case Intrinsic::mips_max_u_w:
1750  case Intrinsic::mips_max_u_d:
1751    return DAG.getNode(MipsISD::VUMAX, DL, Op->getValueType(0),
1752                       Op->getOperand(1), Op->getOperand(2));
1753  case Intrinsic::mips_maxi_s_b:
1754  case Intrinsic::mips_maxi_s_h:
1755  case Intrinsic::mips_maxi_s_w:
1756  case Intrinsic::mips_maxi_s_d:
1757    return DAG.getNode(MipsISD::VSMAX, DL, Op->getValueType(0),
1758                       Op->getOperand(1), lowerMSASplatImm(Op, 2, DAG));
1759  case Intrinsic::mips_maxi_u_b:
1760  case Intrinsic::mips_maxi_u_h:
1761  case Intrinsic::mips_maxi_u_w:
1762  case Intrinsic::mips_maxi_u_d:
1763    return DAG.getNode(MipsISD::VUMAX, DL, Op->getValueType(0),
1764                       Op->getOperand(1), lowerMSASplatImm(Op, 2, DAG));
1765  case Intrinsic::mips_min_s_b:
1766  case Intrinsic::mips_min_s_h:
1767  case Intrinsic::mips_min_s_w:
1768  case Intrinsic::mips_min_s_d:
1769    return DAG.getNode(MipsISD::VSMIN, DL, Op->getValueType(0),
1770                       Op->getOperand(1), Op->getOperand(2));
1771  case Intrinsic::mips_min_u_b:
1772  case Intrinsic::mips_min_u_h:
1773  case Intrinsic::mips_min_u_w:
1774  case Intrinsic::mips_min_u_d:
1775    return DAG.getNode(MipsISD::VUMIN, DL, Op->getValueType(0),
1776                       Op->getOperand(1), Op->getOperand(2));
1777  case Intrinsic::mips_mini_s_b:
1778  case Intrinsic::mips_mini_s_h:
1779  case Intrinsic::mips_mini_s_w:
1780  case Intrinsic::mips_mini_s_d:
1781    return DAG.getNode(MipsISD::VSMIN, DL, Op->getValueType(0),
1782                       Op->getOperand(1), lowerMSASplatImm(Op, 2, DAG));
1783  case Intrinsic::mips_mini_u_b:
1784  case Intrinsic::mips_mini_u_h:
1785  case Intrinsic::mips_mini_u_w:
1786  case Intrinsic::mips_mini_u_d:
1787    return DAG.getNode(MipsISD::VUMIN, DL, Op->getValueType(0),
1788                       Op->getOperand(1), lowerMSASplatImm(Op, 2, DAG));
1789  case Intrinsic::mips_mod_s_b:
1790  case Intrinsic::mips_mod_s_h:
1791  case Intrinsic::mips_mod_s_w:
1792  case Intrinsic::mips_mod_s_d:
1793    return DAG.getNode(ISD::SREM, DL, Op->getValueType(0), Op->getOperand(1),
1794                       Op->getOperand(2));
1795  case Intrinsic::mips_mod_u_b:
1796  case Intrinsic::mips_mod_u_h:
1797  case Intrinsic::mips_mod_u_w:
1798  case Intrinsic::mips_mod_u_d:
1799    return DAG.getNode(ISD::UREM, DL, Op->getValueType(0), Op->getOperand(1),
1800                       Op->getOperand(2));
1801  case Intrinsic::mips_mulv_b:
1802  case Intrinsic::mips_mulv_h:
1803  case Intrinsic::mips_mulv_w:
1804  case Intrinsic::mips_mulv_d:
1805    return DAG.getNode(ISD::MUL, DL, Op->getValueType(0), Op->getOperand(1),
1806                       Op->getOperand(2));
1807  case Intrinsic::mips_msubv_b:
1808  case Intrinsic::mips_msubv_h:
1809  case Intrinsic::mips_msubv_w:
1810  case Intrinsic::mips_msubv_d: {
1811    EVT ResTy = Op->getValueType(0);
1812    return DAG.getNode(ISD::SUB, SDLoc(Op), ResTy, Op->getOperand(1),
1813                       DAG.getNode(ISD::MUL, SDLoc(Op), ResTy,
1814                                   Op->getOperand(2), Op->getOperand(3)));
1815  }
1816  case Intrinsic::mips_nlzc_b:
1817  case Intrinsic::mips_nlzc_h:
1818  case Intrinsic::mips_nlzc_w:
1819  case Intrinsic::mips_nlzc_d:
1820    return DAG.getNode(ISD::CTLZ, DL, Op->getValueType(0), Op->getOperand(1));
1821  case Intrinsic::mips_nor_v: {
1822    SDValue Res = DAG.getNode(ISD::OR, DL, Op->getValueType(0),
1823                              Op->getOperand(1), Op->getOperand(2));
1824    return DAG.getNOT(DL, Res, Res->getValueType(0));
1825  }
1826  case Intrinsic::mips_nori_b: {
1827    SDValue Res =  DAG.getNode(ISD::OR, DL, Op->getValueType(0),
1828                               Op->getOperand(1),
1829                               lowerMSASplatImm(Op, 2, DAG));
1830    return DAG.getNOT(DL, Res, Res->getValueType(0));
1831  }
1832  case Intrinsic::mips_or_v:
1833    return DAG.getNode(ISD::OR, DL, Op->getValueType(0), Op->getOperand(1),
1834                       Op->getOperand(2));
1835  case Intrinsic::mips_ori_b:
1836    return DAG.getNode(ISD::OR, DL, Op->getValueType(0),
1837                       Op->getOperand(1), lowerMSASplatImm(Op, 2, DAG));
1838  case Intrinsic::mips_pckev_b:
1839  case Intrinsic::mips_pckev_h:
1840  case Intrinsic::mips_pckev_w:
1841  case Intrinsic::mips_pckev_d:
1842    return DAG.getNode(MipsISD::PCKEV, DL, Op->getValueType(0),
1843                       Op->getOperand(1), Op->getOperand(2));
1844  case Intrinsic::mips_pckod_b:
1845  case Intrinsic::mips_pckod_h:
1846  case Intrinsic::mips_pckod_w:
1847  case Intrinsic::mips_pckod_d:
1848    return DAG.getNode(MipsISD::PCKOD, DL, Op->getValueType(0),
1849                       Op->getOperand(1), Op->getOperand(2));
1850  case Intrinsic::mips_pcnt_b:
1851  case Intrinsic::mips_pcnt_h:
1852  case Intrinsic::mips_pcnt_w:
1853  case Intrinsic::mips_pcnt_d:
1854    return DAG.getNode(ISD::CTPOP, DL, Op->getValueType(0), Op->getOperand(1));
1855  case Intrinsic::mips_shf_b:
1856  case Intrinsic::mips_shf_h:
1857  case Intrinsic::mips_shf_w:
1858    return DAG.getNode(MipsISD::SHF, DL, Op->getValueType(0),
1859                       Op->getOperand(2), Op->getOperand(1));
1860  case Intrinsic::mips_sll_b:
1861  case Intrinsic::mips_sll_h:
1862  case Intrinsic::mips_sll_w:
1863  case Intrinsic::mips_sll_d:
1864    return DAG.getNode(ISD::SHL, DL, Op->getValueType(0), Op->getOperand(1),
1865                       Op->getOperand(2));
1866  case Intrinsic::mips_slli_b:
1867  case Intrinsic::mips_slli_h:
1868  case Intrinsic::mips_slli_w:
1869  case Intrinsic::mips_slli_d:
1870    return DAG.getNode(ISD::SHL, DL, Op->getValueType(0),
1871                       Op->getOperand(1), lowerMSASplatImm(Op, 2, DAG));
1872  case Intrinsic::mips_splat_b:
1873  case Intrinsic::mips_splat_h:
1874  case Intrinsic::mips_splat_w:
1875  case Intrinsic::mips_splat_d:
1876    // We can't lower via VECTOR_SHUFFLE because it requires constant shuffle
1877    // masks, nor can we lower via BUILD_VECTOR & EXTRACT_VECTOR_ELT because
1878    // EXTRACT_VECTOR_ELT can't extract i64's on MIPS32.
1879    // Instead we lower to MipsISD::VSHF and match from there.
1880    return DAG.getNode(MipsISD::VSHF, DL, Op->getValueType(0),
1881                       lowerMSASplatImm(Op, 2, DAG), Op->getOperand(1),
1882                       Op->getOperand(1));
1883  case Intrinsic::mips_splati_b:
1884  case Intrinsic::mips_splati_h:
1885  case Intrinsic::mips_splati_w:
1886  case Intrinsic::mips_splati_d:
1887    return DAG.getNode(MipsISD::VSHF, DL, Op->getValueType(0),
1888                       lowerMSASplatImm(Op, 2, DAG), Op->getOperand(1),
1889                       Op->getOperand(1));
1890  case Intrinsic::mips_sra_b:
1891  case Intrinsic::mips_sra_h:
1892  case Intrinsic::mips_sra_w:
1893  case Intrinsic::mips_sra_d:
1894    return DAG.getNode(ISD::SRA, DL, Op->getValueType(0), Op->getOperand(1),
1895                       Op->getOperand(2));
1896  case Intrinsic::mips_srai_b:
1897  case Intrinsic::mips_srai_h:
1898  case Intrinsic::mips_srai_w:
1899  case Intrinsic::mips_srai_d:
1900    return DAG.getNode(ISD::SRA, DL, Op->getValueType(0),
1901                       Op->getOperand(1), lowerMSASplatImm(Op, 2, DAG));
1902  case Intrinsic::mips_srl_b:
1903  case Intrinsic::mips_srl_h:
1904  case Intrinsic::mips_srl_w:
1905  case Intrinsic::mips_srl_d:
1906    return DAG.getNode(ISD::SRL, DL, Op->getValueType(0), Op->getOperand(1),
1907                       Op->getOperand(2));
1908  case Intrinsic::mips_srli_b:
1909  case Intrinsic::mips_srli_h:
1910  case Intrinsic::mips_srli_w:
1911  case Intrinsic::mips_srli_d:
1912    return DAG.getNode(ISD::SRL, DL, Op->getValueType(0),
1913                       Op->getOperand(1), lowerMSASplatImm(Op, 2, DAG));
1914  case Intrinsic::mips_subv_b:
1915  case Intrinsic::mips_subv_h:
1916  case Intrinsic::mips_subv_w:
1917  case Intrinsic::mips_subv_d:
1918    return DAG.getNode(ISD::SUB, DL, Op->getValueType(0), Op->getOperand(1),
1919                       Op->getOperand(2));
1920  case Intrinsic::mips_subvi_b:
1921  case Intrinsic::mips_subvi_h:
1922  case Intrinsic::mips_subvi_w:
1923  case Intrinsic::mips_subvi_d:
1924    return DAG.getNode(ISD::SUB, DL, Op->getValueType(0),
1925                       Op->getOperand(1), lowerMSASplatImm(Op, 2, DAG));
1926  case Intrinsic::mips_vshf_b:
1927  case Intrinsic::mips_vshf_h:
1928  case Intrinsic::mips_vshf_w:
1929  case Intrinsic::mips_vshf_d:
1930    return DAG.getNode(MipsISD::VSHF, DL, Op->getValueType(0),
1931                       Op->getOperand(1), Op->getOperand(2), Op->getOperand(3));
1932  case Intrinsic::mips_xor_v:
1933    return DAG.getNode(ISD::XOR, DL, Op->getValueType(0), Op->getOperand(1),
1934                       Op->getOperand(2));
1935  case Intrinsic::mips_xori_b:
1936    return DAG.getNode(ISD::XOR, DL, Op->getValueType(0),
1937                       Op->getOperand(1), lowerMSASplatImm(Op, 2, DAG));
1938  }
1939}
1940
1941static SDValue lowerMSALoadIntr(SDValue Op, SelectionDAG &DAG, unsigned Intr) {
1942  SDLoc DL(Op);
1943  SDValue ChainIn = Op->getOperand(0);
1944  SDValue Address = Op->getOperand(2);
1945  SDValue Offset  = Op->getOperand(3);
1946  EVT ResTy = Op->getValueType(0);
1947  EVT PtrTy = Address->getValueType(0);
1948
1949  Address = DAG.getNode(ISD::ADD, DL, PtrTy, Address, Offset);
1950
1951  return DAG.getLoad(ResTy, DL, ChainIn, Address, MachinePointerInfo(), false,
1952                     false, false, 16);
1953}
1954
1955SDValue MipsSETargetLowering::lowerINTRINSIC_W_CHAIN(SDValue Op,
1956                                                     SelectionDAG &DAG) const {
1957  unsigned Intr = cast<ConstantSDNode>(Op->getOperand(1))->getZExtValue();
1958  switch (Intr) {
1959  default:
1960    return SDValue();
1961  case Intrinsic::mips_extp:
1962    return lowerDSPIntr(Op, DAG, MipsISD::EXTP);
1963  case Intrinsic::mips_extpdp:
1964    return lowerDSPIntr(Op, DAG, MipsISD::EXTPDP);
1965  case Intrinsic::mips_extr_w:
1966    return lowerDSPIntr(Op, DAG, MipsISD::EXTR_W);
1967  case Intrinsic::mips_extr_r_w:
1968    return lowerDSPIntr(Op, DAG, MipsISD::EXTR_R_W);
1969  case Intrinsic::mips_extr_rs_w:
1970    return lowerDSPIntr(Op, DAG, MipsISD::EXTR_RS_W);
1971  case Intrinsic::mips_extr_s_h:
1972    return lowerDSPIntr(Op, DAG, MipsISD::EXTR_S_H);
1973  case Intrinsic::mips_mthlip:
1974    return lowerDSPIntr(Op, DAG, MipsISD::MTHLIP);
1975  case Intrinsic::mips_mulsaq_s_w_ph:
1976    return lowerDSPIntr(Op, DAG, MipsISD::MULSAQ_S_W_PH);
1977  case Intrinsic::mips_maq_s_w_phl:
1978    return lowerDSPIntr(Op, DAG, MipsISD::MAQ_S_W_PHL);
1979  case Intrinsic::mips_maq_s_w_phr:
1980    return lowerDSPIntr(Op, DAG, MipsISD::MAQ_S_W_PHR);
1981  case Intrinsic::mips_maq_sa_w_phl:
1982    return lowerDSPIntr(Op, DAG, MipsISD::MAQ_SA_W_PHL);
1983  case Intrinsic::mips_maq_sa_w_phr:
1984    return lowerDSPIntr(Op, DAG, MipsISD::MAQ_SA_W_PHR);
1985  case Intrinsic::mips_dpaq_s_w_ph:
1986    return lowerDSPIntr(Op, DAG, MipsISD::DPAQ_S_W_PH);
1987  case Intrinsic::mips_dpsq_s_w_ph:
1988    return lowerDSPIntr(Op, DAG, MipsISD::DPSQ_S_W_PH);
1989  case Intrinsic::mips_dpaq_sa_l_w:
1990    return lowerDSPIntr(Op, DAG, MipsISD::DPAQ_SA_L_W);
1991  case Intrinsic::mips_dpsq_sa_l_w:
1992    return lowerDSPIntr(Op, DAG, MipsISD::DPSQ_SA_L_W);
1993  case Intrinsic::mips_dpaqx_s_w_ph:
1994    return lowerDSPIntr(Op, DAG, MipsISD::DPAQX_S_W_PH);
1995  case Intrinsic::mips_dpaqx_sa_w_ph:
1996    return lowerDSPIntr(Op, DAG, MipsISD::DPAQX_SA_W_PH);
1997  case Intrinsic::mips_dpsqx_s_w_ph:
1998    return lowerDSPIntr(Op, DAG, MipsISD::DPSQX_S_W_PH);
1999  case Intrinsic::mips_dpsqx_sa_w_ph:
2000    return lowerDSPIntr(Op, DAG, MipsISD::DPSQX_SA_W_PH);
2001  case Intrinsic::mips_ld_b:
2002  case Intrinsic::mips_ld_h:
2003  case Intrinsic::mips_ld_w:
2004  case Intrinsic::mips_ld_d:
2005   return lowerMSALoadIntr(Op, DAG, Intr);
2006  }
2007}
2008
2009static SDValue lowerMSAStoreIntr(SDValue Op, SelectionDAG &DAG, unsigned Intr) {
2010  SDLoc DL(Op);
2011  SDValue ChainIn = Op->getOperand(0);
2012  SDValue Value   = Op->getOperand(2);
2013  SDValue Address = Op->getOperand(3);
2014  SDValue Offset  = Op->getOperand(4);
2015  EVT PtrTy = Address->getValueType(0);
2016
2017  Address = DAG.getNode(ISD::ADD, DL, PtrTy, Address, Offset);
2018
2019  return DAG.getStore(ChainIn, DL, Value, Address, MachinePointerInfo(), false,
2020                      false, 16);
2021}
2022
2023SDValue MipsSETargetLowering::lowerINTRINSIC_VOID(SDValue Op,
2024                                                  SelectionDAG &DAG) const {
2025  unsigned Intr = cast<ConstantSDNode>(Op->getOperand(1))->getZExtValue();
2026  switch (Intr) {
2027  default:
2028    return SDValue();
2029  case Intrinsic::mips_st_b:
2030  case Intrinsic::mips_st_h:
2031  case Intrinsic::mips_st_w:
2032  case Intrinsic::mips_st_d:
2033    return lowerMSAStoreIntr(Op, DAG, Intr);
2034  }
2035}
2036
2037/// \brief Check if the given BuildVectorSDNode is a splat.
2038/// This method currently relies on DAG nodes being reused when equivalent,
2039/// so it's possible for this to return false even when isConstantSplat returns
2040/// true.
2041static bool isSplatVector(const BuildVectorSDNode *N) {
2042  unsigned int nOps = N->getNumOperands();
2043  assert(nOps > 1 && "isSplatVector has 0 or 1 sized build vector");
2044
2045  SDValue Operand0 = N->getOperand(0);
2046
2047  for (unsigned int i = 1; i < nOps; ++i) {
2048    if (N->getOperand(i) != Operand0)
2049      return false;
2050  }
2051
2052  return true;
2053}
2054
2055// Lower ISD::EXTRACT_VECTOR_ELT into MipsISD::VEXTRACT_SEXT_ELT.
2056//
2057// The non-value bits resulting from ISD::EXTRACT_VECTOR_ELT are undefined. We
2058// choose to sign-extend but we could have equally chosen zero-extend. The
2059// DAGCombiner will fold any sign/zero extension of the ISD::EXTRACT_VECTOR_ELT
2060// result into this node later (possibly changing it to a zero-extend in the
2061// process).
2062SDValue MipsSETargetLowering::
2063lowerEXTRACT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG) const {
2064  SDLoc DL(Op);
2065  EVT ResTy = Op->getValueType(0);
2066  SDValue Op0 = Op->getOperand(0);
2067  EVT VecTy = Op0->getValueType(0);
2068
2069  if (!VecTy.is128BitVector())
2070    return SDValue();
2071
2072  if (ResTy.isInteger()) {
2073    SDValue Op1 = Op->getOperand(1);
2074    EVT EltTy = VecTy.getVectorElementType();
2075    return DAG.getNode(MipsISD::VEXTRACT_SEXT_ELT, DL, ResTy, Op0, Op1,
2076                       DAG.getValueType(EltTy));
2077  }
2078
2079  return Op;
2080}
2081
2082static bool isConstantOrUndef(const SDValue Op) {
2083  if (Op->getOpcode() == ISD::UNDEF)
2084    return true;
2085  if (dyn_cast<ConstantSDNode>(Op))
2086    return true;
2087  if (dyn_cast<ConstantFPSDNode>(Op))
2088    return true;
2089  return false;
2090}
2091
2092static bool isConstantOrUndefBUILD_VECTOR(const BuildVectorSDNode *Op) {
2093  for (unsigned i = 0; i < Op->getNumOperands(); ++i)
2094    if (isConstantOrUndef(Op->getOperand(i)))
2095      return true;
2096  return false;
2097}
2098
2099// Lowers ISD::BUILD_VECTOR into appropriate SelectionDAG nodes for the
2100// backend.
2101//
2102// Lowers according to the following rules:
2103// - Constant splats are legal as-is as long as the SplatBitSize is a power of
2104//   2 less than or equal to 64 and the value fits into a signed 10-bit
2105//   immediate
2106// - Constant splats are lowered to bitconverted BUILD_VECTORs if SplatBitSize
2107//   is a power of 2 less than or equal to 64 and the value does not fit into a
2108//   signed 10-bit immediate
2109// - Non-constant splats are legal as-is.
2110// - Non-constant non-splats are lowered to sequences of INSERT_VECTOR_ELT.
2111// - All others are illegal and must be expanded.
2112SDValue MipsSETargetLowering::lowerBUILD_VECTOR(SDValue Op,
2113                                                SelectionDAG &DAG) const {
2114  BuildVectorSDNode *Node = cast<BuildVectorSDNode>(Op);
2115  EVT ResTy = Op->getValueType(0);
2116  SDLoc DL(Op);
2117  APInt SplatValue, SplatUndef;
2118  unsigned SplatBitSize;
2119  bool HasAnyUndefs;
2120
2121  if (!Subtarget->hasMSA() || !ResTy.is128BitVector())
2122    return SDValue();
2123
2124  if (Node->isConstantSplat(SplatValue, SplatUndef, SplatBitSize,
2125                            HasAnyUndefs, 8,
2126                            !Subtarget->isLittle()) && SplatBitSize <= 64) {
2127    // We can only cope with 8, 16, 32, or 64-bit elements
2128    if (SplatBitSize != 8 && SplatBitSize != 16 && SplatBitSize != 32 &&
2129        SplatBitSize != 64)
2130      return SDValue();
2131
2132    // If the value fits into a simm10 then we can use ldi.[bhwd]
2133    if (SplatValue.isSignedIntN(10))
2134      return Op;
2135
2136    EVT ViaVecTy;
2137
2138    switch (SplatBitSize) {
2139    default:
2140      return SDValue();
2141    case 8:
2142      ViaVecTy = MVT::v16i8;
2143      break;
2144    case 16:
2145      ViaVecTy = MVT::v8i16;
2146      break;
2147    case 32:
2148      ViaVecTy = MVT::v4i32;
2149      break;
2150    case 64:
2151      // There's no fill.d to fall back on for 64-bit values
2152      return SDValue();
2153    }
2154
2155    SmallVector<SDValue, 16> Ops;
2156    SDValue Constant = DAG.getConstant(SplatValue.sextOrSelf(32), MVT::i32);
2157
2158    for (unsigned i = 0; i < ViaVecTy.getVectorNumElements(); ++i)
2159      Ops.push_back(Constant);
2160
2161    SDValue Result = DAG.getNode(ISD::BUILD_VECTOR, SDLoc(Node), ViaVecTy,
2162                                 &Ops[0], Ops.size());
2163
2164    if (ViaVecTy != ResTy)
2165      Result = DAG.getNode(ISD::BITCAST, SDLoc(Node), ResTy, Result);
2166
2167    return Result;
2168  } else if (isSplatVector(Node))
2169    return Op;
2170  else if (!isConstantOrUndefBUILD_VECTOR(Node)) {
2171    // Use INSERT_VECTOR_ELT operations rather than expand to stores.
2172    // The resulting code is the same length as the expansion, but it doesn't
2173    // use memory operations
2174    EVT ResTy = Node->getValueType(0);
2175
2176    assert(ResTy.isVector());
2177
2178    unsigned NumElts = ResTy.getVectorNumElements();
2179    SDValue Vector = DAG.getUNDEF(ResTy);
2180    for (unsigned i = 0; i < NumElts; ++i) {
2181      Vector = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, ResTy, Vector,
2182                           Node->getOperand(i),
2183                           DAG.getConstant(i, MVT::i32));
2184    }
2185    return Vector;
2186  }
2187
2188  return SDValue();
2189}
2190
2191// Lower VECTOR_SHUFFLE into SHF (if possible).
2192//
2193// SHF splits the vector into blocks of four elements, then shuffles these
2194// elements according to a <4 x i2> constant (encoded as an integer immediate).
2195//
2196// It is therefore possible to lower into SHF when the mask takes the form:
2197//   <a, b, c, d, a+4, b+4, c+4, d+4, a+8, b+8, c+8, d+8, ...>
2198// When undef's appear they are treated as if they were whatever value is
2199// necessary in order to fit the above form.
2200//
2201// For example:
2202//   %2 = shufflevector <8 x i16> %0, <8 x i16> undef,
2203//                      <8 x i32> <i32 3, i32 2, i32 1, i32 0,
2204//                                 i32 7, i32 6, i32 5, i32 4>
2205// is lowered to:
2206//   (SHF_H $w0, $w1, 27)
2207// where the 27 comes from:
2208//   3 + (2 << 2) + (1 << 4) + (0 << 6)
2209static SDValue lowerVECTOR_SHUFFLE_SHF(SDValue Op, EVT ResTy,
2210                                       SmallVector<int, 16> Indices,
2211                                       SelectionDAG &DAG) {
2212  int SHFIndices[4] = { -1, -1, -1, -1 };
2213
2214  if (Indices.size() < 4)
2215    return SDValue();
2216
2217  for (unsigned i = 0; i < 4; ++i) {
2218    for (unsigned j = i; j < Indices.size(); j += 4) {
2219      int Idx = Indices[j];
2220
2221      // Convert from vector index to 4-element subvector index
2222      // If an index refers to an element outside of the subvector then give up
2223      if (Idx != -1) {
2224        Idx -= 4 * (j / 4);
2225        if (Idx < 0 || Idx >= 4)
2226          return SDValue();
2227      }
2228
2229      // If the mask has an undef, replace it with the current index.
2230      // Note that it might still be undef if the current index is also undef
2231      if (SHFIndices[i] == -1)
2232        SHFIndices[i] = Idx;
2233
2234      // Check that non-undef values are the same as in the mask. If they
2235      // aren't then give up
2236      if (!(Idx == -1 || Idx == SHFIndices[i]))
2237        return SDValue();
2238    }
2239  }
2240
2241  // Calculate the immediate. Replace any remaining undefs with zero
2242  APInt Imm(32, 0);
2243  for (int i = 3; i >= 0; --i) {
2244    int Idx = SHFIndices[i];
2245
2246    if (Idx == -1)
2247      Idx = 0;
2248
2249    Imm <<= 2;
2250    Imm |= Idx & 0x3;
2251  }
2252
2253  return DAG.getNode(MipsISD::SHF, SDLoc(Op), ResTy,
2254                     DAG.getConstant(Imm, MVT::i32), Op->getOperand(0));
2255}
2256
2257// Lower VECTOR_SHUFFLE into ILVEV (if possible).
2258//
2259// ILVEV interleaves the even elements from each vector.
2260//
2261// It is possible to lower into ILVEV when the mask takes the form:
2262//   <0, n, 2, n+2, 4, n+4, ...>
2263// where n is the number of elements in the vector.
2264//
2265// When undef's appear in the mask they are treated as if they were whatever
2266// value is necessary in order to fit the above form.
2267static SDValue lowerVECTOR_SHUFFLE_ILVEV(SDValue Op, EVT ResTy,
2268                                         SmallVector<int, 16> Indices,
2269                                         SelectionDAG &DAG) {
2270  assert ((Indices.size() % 2) == 0);
2271  int WsIdx = 0;
2272  int WtIdx = ResTy.getVectorNumElements();
2273
2274  for (unsigned i = 0; i < Indices.size(); i += 2) {
2275    if (Indices[i] != -1 && Indices[i] != WsIdx)
2276      return SDValue();
2277    if (Indices[i+1] != -1 && Indices[i+1] != WtIdx)
2278      return SDValue();
2279    WsIdx += 2;
2280    WtIdx += 2;
2281  }
2282
2283  return DAG.getNode(MipsISD::ILVEV, SDLoc(Op), ResTy, Op->getOperand(0),
2284                     Op->getOperand(1));
2285}
2286
2287// Lower VECTOR_SHUFFLE into ILVOD (if possible).
2288//
2289// ILVOD interleaves the odd elements from each vector.
2290//
2291// It is possible to lower into ILVOD when the mask takes the form:
2292//   <1, n+1, 3, n+3, 5, n+5, ...>
2293// where n is the number of elements in the vector.
2294//
2295// When undef's appear in the mask they are treated as if they were whatever
2296// value is necessary in order to fit the above form.
2297static SDValue lowerVECTOR_SHUFFLE_ILVOD(SDValue Op, EVT ResTy,
2298                                         SmallVector<int, 16> Indices,
2299                                         SelectionDAG &DAG) {
2300  assert ((Indices.size() % 2) == 0);
2301  int WsIdx = 1;
2302  int WtIdx = ResTy.getVectorNumElements() + 1;
2303
2304  for (unsigned i = 0; i < Indices.size(); i += 2) {
2305    if (Indices[i] != -1 && Indices[i] != WsIdx)
2306      return SDValue();
2307    if (Indices[i+1] != -1 && Indices[i+1] != WtIdx)
2308      return SDValue();
2309    WsIdx += 2;
2310    WtIdx += 2;
2311  }
2312
2313  return DAG.getNode(MipsISD::ILVOD, SDLoc(Op), ResTy, Op->getOperand(0),
2314                     Op->getOperand(1));
2315}
2316
2317// Lower VECTOR_SHUFFLE into ILVL (if possible).
2318//
2319// ILVL interleaves consecutive elements from the left half of each vector.
2320//
2321// It is possible to lower into ILVL when the mask takes the form:
2322//   <0, n, 1, n+1, 2, n+2, ...>
2323// where n is the number of elements in the vector.
2324//
2325// When undef's appear in the mask they are treated as if they were whatever
2326// value is necessary in order to fit the above form.
2327static SDValue lowerVECTOR_SHUFFLE_ILVL(SDValue Op, EVT ResTy,
2328                                        SmallVector<int, 16> Indices,
2329                                        SelectionDAG &DAG) {
2330  assert ((Indices.size() % 2) == 0);
2331  int WsIdx = 0;
2332  int WtIdx = ResTy.getVectorNumElements();
2333
2334  for (unsigned i = 0; i < Indices.size(); i += 2) {
2335    if (Indices[i] != -1 && Indices[i] != WsIdx)
2336      return SDValue();
2337    if (Indices[i+1] != -1 && Indices[i+1] != WtIdx)
2338      return SDValue();
2339    WsIdx ++;
2340    WtIdx ++;
2341  }
2342
2343  return DAG.getNode(MipsISD::ILVL, SDLoc(Op), ResTy, Op->getOperand(0),
2344                     Op->getOperand(1));
2345}
2346
2347// Lower VECTOR_SHUFFLE into ILVR (if possible).
2348//
2349// ILVR interleaves consecutive elements from the right half of each vector.
2350//
2351// It is possible to lower into ILVR when the mask takes the form:
2352//   <x, n+x, x+1, n+x+1, x+2, n+x+2, ...>
2353// where n is the number of elements in the vector and x is half n.
2354//
2355// When undef's appear in the mask they are treated as if they were whatever
2356// value is necessary in order to fit the above form.
2357static SDValue lowerVECTOR_SHUFFLE_ILVR(SDValue Op, EVT ResTy,
2358                                        SmallVector<int, 16> Indices,
2359                                        SelectionDAG &DAG) {
2360  assert ((Indices.size() % 2) == 0);
2361  unsigned NumElts = ResTy.getVectorNumElements();
2362  int WsIdx = NumElts / 2;
2363  int WtIdx = NumElts + NumElts / 2;
2364
2365  for (unsigned i = 0; i < Indices.size(); i += 2) {
2366    if (Indices[i] != -1 && Indices[i] != WsIdx)
2367      return SDValue();
2368    if (Indices[i+1] != -1 && Indices[i+1] != WtIdx)
2369      return SDValue();
2370    WsIdx ++;
2371    WtIdx ++;
2372  }
2373
2374  return DAG.getNode(MipsISD::ILVR, SDLoc(Op), ResTy, Op->getOperand(0),
2375                     Op->getOperand(1));
2376}
2377
2378// Lower VECTOR_SHUFFLE into PCKEV (if possible).
2379//
2380// PCKEV copies the even elements of each vector into the result vector.
2381//
2382// It is possible to lower into PCKEV when the mask takes the form:
2383//   <0, 2, 4, ..., n, n+2, n+4, ...>
2384// where n is the number of elements in the vector.
2385//
2386// When undef's appear in the mask they are treated as if they were whatever
2387// value is necessary in order to fit the above form.
2388static SDValue lowerVECTOR_SHUFFLE_PCKEV(SDValue Op, EVT ResTy,
2389                                         SmallVector<int, 16> Indices,
2390                                         SelectionDAG &DAG) {
2391  assert ((Indices.size() % 2) == 0);
2392  int Idx = 0;
2393
2394  for (unsigned i = 0; i < Indices.size(); ++i) {
2395    if (Indices[i] != -1 && Indices[i] != Idx)
2396      return SDValue();
2397    Idx += 2;
2398  }
2399
2400  return DAG.getNode(MipsISD::PCKEV, SDLoc(Op), ResTy, Op->getOperand(0),
2401                     Op->getOperand(1));
2402}
2403
2404// Lower VECTOR_SHUFFLE into PCKOD (if possible).
2405//
2406// PCKOD copies the odd elements of each vector into the result vector.
2407//
2408// It is possible to lower into PCKOD when the mask takes the form:
2409//   <1, 3, 5, ..., n+1, n+3, n+5, ...>
2410// where n is the number of elements in the vector.
2411//
2412// When undef's appear in the mask they are treated as if they were whatever
2413// value is necessary in order to fit the above form.
2414static SDValue lowerVECTOR_SHUFFLE_PCKOD(SDValue Op, EVT ResTy,
2415                                         SmallVector<int, 16> Indices,
2416                                         SelectionDAG &DAG) {
2417  assert ((Indices.size() % 2) == 0);
2418  int Idx = 1;
2419
2420  for (unsigned i = 0; i < Indices.size(); ++i) {
2421    if (Indices[i] != -1 && Indices[i] != Idx)
2422      return SDValue();
2423    Idx += 2;
2424  }
2425
2426  return DAG.getNode(MipsISD::PCKOD, SDLoc(Op), ResTy, Op->getOperand(0),
2427                     Op->getOperand(1));
2428}
2429
2430// Lower VECTOR_SHUFFLE into VSHF.
2431//
2432// This mostly consists of converting the shuffle indices in Indices into a
2433// BUILD_VECTOR and adding it as an operand to the resulting VSHF. There is
2434// also code to eliminate unused operands of the VECTOR_SHUFFLE. For example,
2435// if the type is v8i16 and all the indices are less than 8 then the second
2436// operand is unused and can be replaced with anything. We choose to replace it
2437// with the used operand since this reduces the number of instructions overall.
2438static SDValue lowerVECTOR_SHUFFLE_VSHF(SDValue Op, EVT ResTy,
2439                                        SmallVector<int, 16> Indices,
2440                                        SelectionDAG &DAG) {
2441  SmallVector<SDValue, 16> Ops;
2442  SDValue Op0;
2443  SDValue Op1;
2444  EVT MaskVecTy = ResTy.changeVectorElementTypeToInteger();
2445  EVT MaskEltTy = MaskVecTy.getVectorElementType();
2446  bool Using1stVec = false;
2447  bool Using2ndVec = false;
2448  SDLoc DL(Op);
2449  int ResTyNumElts = ResTy.getVectorNumElements();
2450
2451  for (int i = 0; i < ResTyNumElts; ++i) {
2452    // Idx == -1 means UNDEF
2453    int Idx = Indices[i];
2454
2455    if (0 <= Idx && Idx < ResTyNumElts)
2456      Using1stVec = true;
2457    if (ResTyNumElts <= Idx && Idx < ResTyNumElts * 2)
2458      Using2ndVec = true;
2459  }
2460
2461  for (SmallVector<int, 16>::iterator I = Indices.begin(); I != Indices.end();
2462       ++I)
2463    Ops.push_back(DAG.getTargetConstant(*I, MaskEltTy));
2464
2465  SDValue MaskVec = DAG.getNode(ISD::BUILD_VECTOR, DL, MaskVecTy, &Ops[0],
2466                                Ops.size());
2467
2468  if (Using1stVec && Using2ndVec) {
2469    Op0 = Op->getOperand(0);
2470    Op1 = Op->getOperand(1);
2471  } else if (Using1stVec)
2472    Op0 = Op1 = Op->getOperand(0);
2473  else if (Using2ndVec)
2474    Op0 = Op1 = Op->getOperand(1);
2475  else
2476    llvm_unreachable("shuffle vector mask references neither vector operand?");
2477
2478  return DAG.getNode(MipsISD::VSHF, DL, ResTy, MaskVec, Op0, Op1);
2479}
2480
2481// Lower VECTOR_SHUFFLE into one of a number of instructions depending on the
2482// indices in the shuffle.
2483SDValue MipsSETargetLowering::lowerVECTOR_SHUFFLE(SDValue Op,
2484                                                  SelectionDAG &DAG) const {
2485  ShuffleVectorSDNode *Node = cast<ShuffleVectorSDNode>(Op);
2486  EVT ResTy = Op->getValueType(0);
2487
2488  if (!ResTy.is128BitVector())
2489    return SDValue();
2490
2491  int ResTyNumElts = ResTy.getVectorNumElements();
2492  SmallVector<int, 16> Indices;
2493
2494  for (int i = 0; i < ResTyNumElts; ++i)
2495    Indices.push_back(Node->getMaskElt(i));
2496
2497  SDValue Result = lowerVECTOR_SHUFFLE_SHF(Op, ResTy, Indices, DAG);
2498  if (Result.getNode())
2499    return Result;
2500  Result = lowerVECTOR_SHUFFLE_ILVEV(Op, ResTy, Indices, DAG);
2501  if (Result.getNode())
2502    return Result;
2503  Result = lowerVECTOR_SHUFFLE_ILVOD(Op, ResTy, Indices, DAG);
2504  if (Result.getNode())
2505    return Result;
2506  Result = lowerVECTOR_SHUFFLE_ILVL(Op, ResTy, Indices, DAG);
2507  if (Result.getNode())
2508    return Result;
2509  Result = lowerVECTOR_SHUFFLE_ILVR(Op, ResTy, Indices, DAG);
2510  if (Result.getNode())
2511    return Result;
2512  Result = lowerVECTOR_SHUFFLE_PCKEV(Op, ResTy, Indices, DAG);
2513  if (Result.getNode())
2514    return Result;
2515  Result = lowerVECTOR_SHUFFLE_PCKOD(Op, ResTy, Indices, DAG);
2516  if (Result.getNode())
2517    return Result;
2518  return lowerVECTOR_SHUFFLE_VSHF(Op, ResTy, Indices, DAG);
2519}
2520
2521MachineBasicBlock * MipsSETargetLowering::
2522emitBPOSGE32(MachineInstr *MI, MachineBasicBlock *BB) const{
2523  // $bb:
2524  //  bposge32_pseudo $vr0
2525  //  =>
2526  // $bb:
2527  //  bposge32 $tbb
2528  // $fbb:
2529  //  li $vr2, 0
2530  //  b $sink
2531  // $tbb:
2532  //  li $vr1, 1
2533  // $sink:
2534  //  $vr0 = phi($vr2, $fbb, $vr1, $tbb)
2535
2536  MachineRegisterInfo &RegInfo = BB->getParent()->getRegInfo();
2537  const TargetInstrInfo *TII = getTargetMachine().getInstrInfo();
2538  const TargetRegisterClass *RC = &Mips::GPR32RegClass;
2539  DebugLoc DL = MI->getDebugLoc();
2540  const BasicBlock *LLVM_BB = BB->getBasicBlock();
2541  MachineFunction::iterator It = llvm::next(MachineFunction::iterator(BB));
2542  MachineFunction *F = BB->getParent();
2543  MachineBasicBlock *FBB = F->CreateMachineBasicBlock(LLVM_BB);
2544  MachineBasicBlock *TBB = F->CreateMachineBasicBlock(LLVM_BB);
2545  MachineBasicBlock *Sink  = F->CreateMachineBasicBlock(LLVM_BB);
2546  F->insert(It, FBB);
2547  F->insert(It, TBB);
2548  F->insert(It, Sink);
2549
2550  // Transfer the remainder of BB and its successor edges to Sink.
2551  Sink->splice(Sink->begin(), BB, llvm::next(MachineBasicBlock::iterator(MI)),
2552               BB->end());
2553  Sink->transferSuccessorsAndUpdatePHIs(BB);
2554
2555  // Add successors.
2556  BB->addSuccessor(FBB);
2557  BB->addSuccessor(TBB);
2558  FBB->addSuccessor(Sink);
2559  TBB->addSuccessor(Sink);
2560
2561  // Insert the real bposge32 instruction to $BB.
2562  BuildMI(BB, DL, TII->get(Mips::BPOSGE32)).addMBB(TBB);
2563
2564  // Fill $FBB.
2565  unsigned VR2 = RegInfo.createVirtualRegister(RC);
2566  BuildMI(*FBB, FBB->end(), DL, TII->get(Mips::ADDiu), VR2)
2567    .addReg(Mips::ZERO).addImm(0);
2568  BuildMI(*FBB, FBB->end(), DL, TII->get(Mips::B)).addMBB(Sink);
2569
2570  // Fill $TBB.
2571  unsigned VR1 = RegInfo.createVirtualRegister(RC);
2572  BuildMI(*TBB, TBB->end(), DL, TII->get(Mips::ADDiu), VR1)
2573    .addReg(Mips::ZERO).addImm(1);
2574
2575  // Insert phi function to $Sink.
2576  BuildMI(*Sink, Sink->begin(), DL, TII->get(Mips::PHI),
2577          MI->getOperand(0).getReg())
2578    .addReg(VR2).addMBB(FBB).addReg(VR1).addMBB(TBB);
2579
2580  MI->eraseFromParent();   // The pseudo instruction is gone now.
2581  return Sink;
2582}
2583
2584MachineBasicBlock * MipsSETargetLowering::
2585emitMSACBranchPseudo(MachineInstr *MI, MachineBasicBlock *BB,
2586                     unsigned BranchOp) const{
2587  // $bb:
2588  //  vany_nonzero $rd, $ws
2589  //  =>
2590  // $bb:
2591  //  bnz.b $ws, $tbb
2592  //  b $fbb
2593  // $fbb:
2594  //  li $rd1, 0
2595  //  b $sink
2596  // $tbb:
2597  //  li $rd2, 1
2598  // $sink:
2599  //  $rd = phi($rd1, $fbb, $rd2, $tbb)
2600
2601  MachineRegisterInfo &RegInfo = BB->getParent()->getRegInfo();
2602  const TargetInstrInfo *TII = getTargetMachine().getInstrInfo();
2603  const TargetRegisterClass *RC = &Mips::GPR32RegClass;
2604  DebugLoc DL = MI->getDebugLoc();
2605  const BasicBlock *LLVM_BB = BB->getBasicBlock();
2606  MachineFunction::iterator It = llvm::next(MachineFunction::iterator(BB));
2607  MachineFunction *F = BB->getParent();
2608  MachineBasicBlock *FBB = F->CreateMachineBasicBlock(LLVM_BB);
2609  MachineBasicBlock *TBB = F->CreateMachineBasicBlock(LLVM_BB);
2610  MachineBasicBlock *Sink  = F->CreateMachineBasicBlock(LLVM_BB);
2611  F->insert(It, FBB);
2612  F->insert(It, TBB);
2613  F->insert(It, Sink);
2614
2615  // Transfer the remainder of BB and its successor edges to Sink.
2616  Sink->splice(Sink->begin(), BB, llvm::next(MachineBasicBlock::iterator(MI)),
2617               BB->end());
2618  Sink->transferSuccessorsAndUpdatePHIs(BB);
2619
2620  // Add successors.
2621  BB->addSuccessor(FBB);
2622  BB->addSuccessor(TBB);
2623  FBB->addSuccessor(Sink);
2624  TBB->addSuccessor(Sink);
2625
2626  // Insert the real bnz.b instruction to $BB.
2627  BuildMI(BB, DL, TII->get(BranchOp))
2628    .addReg(MI->getOperand(1).getReg())
2629    .addMBB(TBB);
2630
2631  // Fill $FBB.
2632  unsigned RD1 = RegInfo.createVirtualRegister(RC);
2633  BuildMI(*FBB, FBB->end(), DL, TII->get(Mips::ADDiu), RD1)
2634    .addReg(Mips::ZERO).addImm(0);
2635  BuildMI(*FBB, FBB->end(), DL, TII->get(Mips::B)).addMBB(Sink);
2636
2637  // Fill $TBB.
2638  unsigned RD2 = RegInfo.createVirtualRegister(RC);
2639  BuildMI(*TBB, TBB->end(), DL, TII->get(Mips::ADDiu), RD2)
2640    .addReg(Mips::ZERO).addImm(1);
2641
2642  // Insert phi function to $Sink.
2643  BuildMI(*Sink, Sink->begin(), DL, TII->get(Mips::PHI),
2644          MI->getOperand(0).getReg())
2645    .addReg(RD1).addMBB(FBB).addReg(RD2).addMBB(TBB);
2646
2647  MI->eraseFromParent();   // The pseudo instruction is gone now.
2648  return Sink;
2649}
2650
2651// Emit the COPY_FW pseudo instruction.
2652//
2653// copy_fw_pseudo $fd, $ws, n
2654// =>
2655// copy_u_w $rt, $ws, $n
2656// mtc1     $rt, $fd
2657//
2658// When n is zero, the equivalent operation can be performed with (potentially)
2659// zero instructions due to register overlaps. This optimization is never valid
2660// for lane 1 because it would require FR=0 mode which isn't supported by MSA.
2661MachineBasicBlock * MipsSETargetLowering::
2662emitCOPY_FW(MachineInstr *MI, MachineBasicBlock *BB) const{
2663  const TargetInstrInfo *TII = getTargetMachine().getInstrInfo();
2664  MachineRegisterInfo &RegInfo = BB->getParent()->getRegInfo();
2665  DebugLoc DL = MI->getDebugLoc();
2666  unsigned Fd = MI->getOperand(0).getReg();
2667  unsigned Ws = MI->getOperand(1).getReg();
2668  unsigned Lane = MI->getOperand(2).getImm();
2669
2670  if (Lane == 0)
2671    BuildMI(*BB, MI, DL, TII->get(Mips::COPY), Fd).addReg(Ws, 0, Mips::sub_lo);
2672  else {
2673    unsigned Wt = RegInfo.createVirtualRegister(&Mips::MSA128WRegClass);
2674
2675    BuildMI(*BB, MI, DL, TII->get(Mips::SPLATI_W), Wt).addReg(Ws).addImm(1);
2676    BuildMI(*BB, MI, DL, TII->get(Mips::COPY), Fd).addReg(Wt, 0, Mips::sub_lo);
2677  }
2678
2679  MI->eraseFromParent();   // The pseudo instruction is gone now.
2680  return BB;
2681}
2682
2683// Emit the COPY_FD pseudo instruction.
2684//
2685// copy_fd_pseudo $fd, $ws, n
2686// =>
2687// splati.d $wt, $ws, $n
2688// copy $fd, $wt:sub_64
2689//
2690// When n is zero, the equivalent operation can be performed with (potentially)
2691// zero instructions due to register overlaps. This optimization is always
2692// valid because FR=1 mode which is the only supported mode in MSA.
2693MachineBasicBlock * MipsSETargetLowering::
2694emitCOPY_FD(MachineInstr *MI, MachineBasicBlock *BB) const{
2695  assert(Subtarget->isFP64bit());
2696
2697  const TargetInstrInfo *TII = getTargetMachine().getInstrInfo();
2698  MachineRegisterInfo &RegInfo = BB->getParent()->getRegInfo();
2699  unsigned Fd  = MI->getOperand(0).getReg();
2700  unsigned Ws  = MI->getOperand(1).getReg();
2701  unsigned Lane = MI->getOperand(2).getImm() * 2;
2702  DebugLoc DL = MI->getDebugLoc();
2703
2704  if (Lane == 0)
2705    BuildMI(*BB, MI, DL, TII->get(Mips::COPY), Fd).addReg(Ws, 0, Mips::sub_64);
2706  else {
2707    unsigned Wt = RegInfo.createVirtualRegister(&Mips::MSA128DRegClass);
2708
2709    BuildMI(*BB, MI, DL, TII->get(Mips::SPLATI_D), Wt).addReg(Ws).addImm(1);
2710    BuildMI(*BB, MI, DL, TII->get(Mips::COPY), Fd).addReg(Wt, 0, Mips::sub_64);
2711  }
2712
2713  MI->eraseFromParent();   // The pseudo instruction is gone now.
2714  return BB;
2715}
2716
2717// Emit the INSERT_FW pseudo instruction.
2718//
2719// insert_fw_pseudo $wd, $wd_in, $n, $fs
2720// =>
2721// subreg_to_reg $wt:sub_lo, $fs
2722// insve_w $wd[$n], $wd_in, $wt[0]
2723MachineBasicBlock *
2724MipsSETargetLowering::emitINSERT_FW(MachineInstr *MI,
2725                                    MachineBasicBlock *BB) const {
2726  const TargetInstrInfo *TII = getTargetMachine().getInstrInfo();
2727  MachineRegisterInfo &RegInfo = BB->getParent()->getRegInfo();
2728  DebugLoc DL = MI->getDebugLoc();
2729  unsigned Wd = MI->getOperand(0).getReg();
2730  unsigned Wd_in = MI->getOperand(1).getReg();
2731  unsigned Lane = MI->getOperand(2).getImm();
2732  unsigned Fs = MI->getOperand(3).getReg();
2733  unsigned Wt = RegInfo.createVirtualRegister(&Mips::MSA128WRegClass);
2734
2735  BuildMI(*BB, MI, DL, TII->get(Mips::SUBREG_TO_REG), Wt)
2736      .addImm(0)
2737      .addReg(Fs)
2738      .addImm(Mips::sub_lo);
2739  BuildMI(*BB, MI, DL, TII->get(Mips::INSVE_W), Wd)
2740      .addReg(Wd_in)
2741      .addImm(Lane)
2742      .addReg(Wt);
2743
2744  MI->eraseFromParent(); // The pseudo instruction is gone now.
2745  return BB;
2746}
2747
2748// Emit the INSERT_FD pseudo instruction.
2749//
2750// insert_fd_pseudo $wd, $fs, n
2751// =>
2752// subreg_to_reg $wt:sub_64, $fs
2753// insve_d $wd[$n], $wd_in, $wt[0]
2754MachineBasicBlock *
2755MipsSETargetLowering::emitINSERT_FD(MachineInstr *MI,
2756                                    MachineBasicBlock *BB) const {
2757  assert(Subtarget->isFP64bit());
2758
2759  const TargetInstrInfo *TII = getTargetMachine().getInstrInfo();
2760  MachineRegisterInfo &RegInfo = BB->getParent()->getRegInfo();
2761  DebugLoc DL = MI->getDebugLoc();
2762  unsigned Wd = MI->getOperand(0).getReg();
2763  unsigned Wd_in = MI->getOperand(1).getReg();
2764  unsigned Lane = MI->getOperand(2).getImm();
2765  unsigned Fs = MI->getOperand(3).getReg();
2766  unsigned Wt = RegInfo.createVirtualRegister(&Mips::MSA128DRegClass);
2767
2768  BuildMI(*BB, MI, DL, TII->get(Mips::SUBREG_TO_REG), Wt)
2769      .addImm(0)
2770      .addReg(Fs)
2771      .addImm(Mips::sub_64);
2772  BuildMI(*BB, MI, DL, TII->get(Mips::INSVE_D), Wd)
2773      .addReg(Wd_in)
2774      .addImm(Lane)
2775      .addReg(Wt);
2776
2777  MI->eraseFromParent(); // The pseudo instruction is gone now.
2778  return BB;
2779}
2780
2781// Emit the FILL_FW pseudo instruction.
2782//
2783// fill_fw_pseudo $wd, $fs
2784// =>
2785// implicit_def $wt1
2786// insert_subreg $wt2:subreg_lo, $wt1, $fs
2787// splati.w $wd, $wt2[0]
2788MachineBasicBlock *
2789MipsSETargetLowering::emitFILL_FW(MachineInstr *MI,
2790                                  MachineBasicBlock *BB) const {
2791  const TargetInstrInfo *TII = getTargetMachine().getInstrInfo();
2792  MachineRegisterInfo &RegInfo = BB->getParent()->getRegInfo();
2793  DebugLoc DL = MI->getDebugLoc();
2794  unsigned Wd = MI->getOperand(0).getReg();
2795  unsigned Fs = MI->getOperand(1).getReg();
2796  unsigned Wt1 = RegInfo.createVirtualRegister(&Mips::MSA128WRegClass);
2797  unsigned Wt2 = RegInfo.createVirtualRegister(&Mips::MSA128WRegClass);
2798
2799  BuildMI(*BB, MI, DL, TII->get(Mips::IMPLICIT_DEF), Wt1);
2800  BuildMI(*BB, MI, DL, TII->get(Mips::INSERT_SUBREG), Wt2)
2801      .addReg(Wt1)
2802      .addReg(Fs)
2803      .addImm(Mips::sub_lo);
2804  BuildMI(*BB, MI, DL, TII->get(Mips::SPLATI_W), Wd).addReg(Wt2).addImm(0);
2805
2806  MI->eraseFromParent(); // The pseudo instruction is gone now.
2807  return BB;
2808}
2809
2810// Emit the FILL_FD pseudo instruction.
2811//
2812// fill_fd_pseudo $wd, $fs
2813// =>
2814// implicit_def $wt1
2815// insert_subreg $wt2:subreg_64, $wt1, $fs
2816// splati.d $wd, $wt2[0]
2817MachineBasicBlock *
2818MipsSETargetLowering::emitFILL_FD(MachineInstr *MI,
2819                                  MachineBasicBlock *BB) const {
2820  assert(Subtarget->isFP64bit());
2821
2822  const TargetInstrInfo *TII = getTargetMachine().getInstrInfo();
2823  MachineRegisterInfo &RegInfo = BB->getParent()->getRegInfo();
2824  DebugLoc DL = MI->getDebugLoc();
2825  unsigned Wd = MI->getOperand(0).getReg();
2826  unsigned Fs = MI->getOperand(1).getReg();
2827  unsigned Wt1 = RegInfo.createVirtualRegister(&Mips::MSA128DRegClass);
2828  unsigned Wt2 = RegInfo.createVirtualRegister(&Mips::MSA128DRegClass);
2829
2830  BuildMI(*BB, MI, DL, TII->get(Mips::IMPLICIT_DEF), Wt1);
2831  BuildMI(*BB, MI, DL, TII->get(Mips::INSERT_SUBREG), Wt2)
2832      .addReg(Wt1)
2833      .addReg(Fs)
2834      .addImm(Mips::sub_64);
2835  BuildMI(*BB, MI, DL, TII->get(Mips::SPLATI_D), Wd).addReg(Wt2).addImm(0);
2836
2837  MI->eraseFromParent();   // The pseudo instruction is gone now.
2838  return BB;
2839}
2840
2841// Emit the FEXP2_W_1 pseudo instructions.
2842//
2843// fexp2_w_1_pseudo $wd, $wt
2844// =>
2845// ldi.w $ws, 1
2846// fexp2.w $wd, $ws, $wt
2847MachineBasicBlock *
2848MipsSETargetLowering::emitFEXP2_W_1(MachineInstr *MI,
2849                                    MachineBasicBlock *BB) const {
2850  const TargetInstrInfo *TII = getTargetMachine().getInstrInfo();
2851  MachineRegisterInfo &RegInfo = BB->getParent()->getRegInfo();
2852  const TargetRegisterClass *RC = &Mips::MSA128WRegClass;
2853  unsigned Ws1 = RegInfo.createVirtualRegister(RC);
2854  unsigned Ws2 = RegInfo.createVirtualRegister(RC);
2855  DebugLoc DL = MI->getDebugLoc();
2856
2857  // Splat 1.0 into a vector
2858  BuildMI(*BB, MI, DL, TII->get(Mips::LDI_W), Ws1).addImm(1);
2859  BuildMI(*BB, MI, DL, TII->get(Mips::FFINT_U_W), Ws2).addReg(Ws1);
2860
2861  // Emit 1.0 * fexp2(Wt)
2862  BuildMI(*BB, MI, DL, TII->get(Mips::FEXP2_W), MI->getOperand(0).getReg())
2863      .addReg(Ws2)
2864      .addReg(MI->getOperand(1).getReg());
2865
2866  MI->eraseFromParent(); // The pseudo instruction is gone now.
2867  return BB;
2868}
2869
2870// Emit the FEXP2_D_1 pseudo instructions.
2871//
2872// fexp2_d_1_pseudo $wd, $wt
2873// =>
2874// ldi.d $ws, 1
2875// fexp2.d $wd, $ws, $wt
2876MachineBasicBlock *
2877MipsSETargetLowering::emitFEXP2_D_1(MachineInstr *MI,
2878                                    MachineBasicBlock *BB) const {
2879  const TargetInstrInfo *TII = getTargetMachine().getInstrInfo();
2880  MachineRegisterInfo &RegInfo = BB->getParent()->getRegInfo();
2881  const TargetRegisterClass *RC = &Mips::MSA128DRegClass;
2882  unsigned Ws1 = RegInfo.createVirtualRegister(RC);
2883  unsigned Ws2 = RegInfo.createVirtualRegister(RC);
2884  DebugLoc DL = MI->getDebugLoc();
2885
2886  // Splat 1.0 into a vector
2887  BuildMI(*BB, MI, DL, TII->get(Mips::LDI_D), Ws1).addImm(1);
2888  BuildMI(*BB, MI, DL, TII->get(Mips::FFINT_U_D), Ws2).addReg(Ws1);
2889
2890  // Emit 1.0 * fexp2(Wt)
2891  BuildMI(*BB, MI, DL, TII->get(Mips::FEXP2_D), MI->getOperand(0).getReg())
2892      .addReg(Ws2)
2893      .addReg(MI->getOperand(1).getReg());
2894
2895  MI->eraseFromParent(); // The pseudo instruction is gone now.
2896  return BB;
2897}
2898