MipsSEISelLowering.cpp revision 75ac8df380e2bad6b7e0798641b8b1805a393339 
1//===-- MipsSEISelLowering.cpp - MipsSE DAG Lowering Interface --*- C++ -*-===//
2//
3//                     The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10// Subclass of MipsTargetLowering specialized for mips32/64.
11//
12//===----------------------------------------------------------------------===//
13#define DEBUG_TYPE "mips-isel"
14#include "MipsSEISelLowering.h"
15#include "MipsRegisterInfo.h"
16#include "MipsTargetMachine.h"
17#include "llvm/CodeGen/MachineInstrBuilder.h"
18#include "llvm/CodeGen/MachineRegisterInfo.h"
19#include "llvm/IR/Intrinsics.h"
20#include "llvm/Support/CommandLine.h"
21#include "llvm/Support/Debug.h"
22#include "llvm/Support/raw_ostream.h"
23#include "llvm/Target/TargetInstrInfo.h"
24
25using namespace llvm;
26
27static cl::opt<bool>
28EnableMipsTailCalls("enable-mips-tail-calls", cl::Hidden,
29                    cl::desc("MIPS: Enable tail calls."), cl::init(false));
30
31static cl::opt<bool> NoDPLoadStore("mno-ldc1-sdc1", cl::init(false),
32                                   cl::desc("Expand double precision loads and "
33                                            "stores to their single precision "
34                                            "counterparts"));
35
36MipsSETargetLowering::MipsSETargetLowering(MipsTargetMachine &TM)
37  : MipsTargetLowering(TM) {
38  // Set up the register classes
39  addRegisterClass(MVT::i32, &Mips::GPR32RegClass);
40
41  if (HasMips64)
42    addRegisterClass(MVT::i64, &Mips::GPR64RegClass);
43
44  if (Subtarget->hasDSP() || Subtarget->hasMSA()) {
45    // Expand all truncating stores and extending loads.
46    unsigned FirstVT = (unsigned)MVT::FIRST_VECTOR_VALUETYPE;
47    unsigned LastVT = (unsigned)MVT::LAST_VECTOR_VALUETYPE;
48
49    for (unsigned VT0 = FirstVT; VT0 <= LastVT; ++VT0) {
50      for (unsigned VT1 = FirstVT; VT1 <= LastVT; ++VT1)
51        setTruncStoreAction((MVT::SimpleValueType)VT0,
52                            (MVT::SimpleValueType)VT1, Expand);
53
54      setLoadExtAction(ISD::SEXTLOAD, (MVT::SimpleValueType)VT0, Expand);
55      setLoadExtAction(ISD::ZEXTLOAD, (MVT::SimpleValueType)VT0, Expand);
56      setLoadExtAction(ISD::EXTLOAD, (MVT::SimpleValueType)VT0, Expand);
57    }
58  }
59
60  if (Subtarget->hasDSP()) {
61    MVT::SimpleValueType VecTys[2] = {MVT::v2i16, MVT::v4i8};
62
63    for (unsigned i = 0; i < array_lengthof(VecTys); ++i) {
64      addRegisterClass(VecTys[i], &Mips::DSPRRegClass);
65
66      // Expand all builtin opcodes.
67      for (unsigned Opc = 0; Opc < ISD::BUILTIN_OP_END; ++Opc)
68        setOperationAction(Opc, VecTys[i], Expand);
69
70      setOperationAction(ISD::ADD, VecTys[i], Legal);
71      setOperationAction(ISD::SUB, VecTys[i], Legal);
72      setOperationAction(ISD::LOAD, VecTys[i], Legal);
73      setOperationAction(ISD::STORE, VecTys[i], Legal);
74      setOperationAction(ISD::BITCAST, VecTys[i], Legal);
75    }
76
77    setTargetDAGCombine(ISD::SHL);
78    setTargetDAGCombine(ISD::SRA);
79    setTargetDAGCombine(ISD::SRL);
80    setTargetDAGCombine(ISD::SETCC);
81    setTargetDAGCombine(ISD::VSELECT);
82  }
83
84  if (Subtarget->hasDSPR2())
85    setOperationAction(ISD::MUL, MVT::v2i16, Legal);
86
87  if (Subtarget->hasMSA()) {
88    addMSAIntType(MVT::v16i8, &Mips::MSA128BRegClass);
89    addMSAIntType(MVT::v8i16, &Mips::MSA128HRegClass);
90    addMSAIntType(MVT::v4i32, &Mips::MSA128WRegClass);
91    addMSAIntType(MVT::v2i64, &Mips::MSA128DRegClass);
92    addMSAFloatType(MVT::v8f16, &Mips::MSA128HRegClass);
93    addMSAFloatType(MVT::v4f32, &Mips::MSA128WRegClass);
94    addMSAFloatType(MVT::v2f64, &Mips::MSA128DRegClass);
95
96    setTargetDAGCombine(ISD::AND);
97    setTargetDAGCombine(ISD::OR);
98    setTargetDAGCombine(ISD::SRA);
99    setTargetDAGCombine(ISD::VSELECT);
100    setTargetDAGCombine(ISD::XOR);
101  }
102
103  if (!Subtarget->mipsSEUsesSoftFloat()) {
104    addRegisterClass(MVT::f32, &Mips::FGR32RegClass);
105
106    // When dealing with single precision only, use libcalls
107    if (!Subtarget->isSingleFloat()) {
108      if (Subtarget->isFP64bit())
109        addRegisterClass(MVT::f64, &Mips::FGR64RegClass);
110      else
111        addRegisterClass(MVT::f64, &Mips::AFGR64RegClass);
112    }
113  }
114
115  setOperationAction(ISD::SMUL_LOHI,          MVT::i32, Custom);
116  setOperationAction(ISD::UMUL_LOHI,          MVT::i32, Custom);
117  setOperationAction(ISD::MULHS,              MVT::i32, Custom);
118  setOperationAction(ISD::MULHU,              MVT::i32, Custom);
119
120  if (HasMips64) {
121    setOperationAction(ISD::MULHS,            MVT::i64, Custom);
122    setOperationAction(ISD::MULHU,            MVT::i64, Custom);
123    setOperationAction(ISD::MUL,              MVT::i64, Custom);
124  }
125
126  setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::i64, Custom);
127  setOperationAction(ISD::INTRINSIC_W_CHAIN,  MVT::i64, Custom);
128
129  setOperationAction(ISD::SDIVREM, MVT::i32, Custom);
130  setOperationAction(ISD::UDIVREM, MVT::i32, Custom);
131  setOperationAction(ISD::SDIVREM, MVT::i64, Custom);
132  setOperationAction(ISD::UDIVREM, MVT::i64, Custom);
133  setOperationAction(ISD::ATOMIC_FENCE,       MVT::Other, Custom);
134  setOperationAction(ISD::LOAD,               MVT::i32, Custom);
135  setOperationAction(ISD::STORE,              MVT::i32, Custom);
136
137  setTargetDAGCombine(ISD::ADDE);
138  setTargetDAGCombine(ISD::SUBE);
139  setTargetDAGCombine(ISD::MUL);
140
141  setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::Other, Custom);
142  setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::Other, Custom);
143  setOperationAction(ISD::INTRINSIC_VOID, MVT::Other, Custom);
144
145  if (NoDPLoadStore) {
146    setOperationAction(ISD::LOAD, MVT::f64, Custom);
147    setOperationAction(ISD::STORE, MVT::f64, Custom);
148  }
149
150  computeRegisterProperties();
151}
152
153const MipsTargetLowering *
154llvm::createMipsSETargetLowering(MipsTargetMachine &TM) {
155  return new MipsSETargetLowering(TM);
156}
157
158// Enable MSA support for the given integer type and Register class.
159void MipsSETargetLowering::
160addMSAIntType(MVT::SimpleValueType Ty, const TargetRegisterClass *RC) {
161  addRegisterClass(Ty, RC);
162
163  // Expand all builtin opcodes.
164  for (unsigned Opc = 0; Opc < ISD::BUILTIN_OP_END; ++Opc)
165    setOperationAction(Opc, Ty, Expand);
166
167  setOperationAction(ISD::BITCAST, Ty, Legal);
168  setOperationAction(ISD::LOAD, Ty, Legal);
169  setOperationAction(ISD::STORE, Ty, Legal);
170  setOperationAction(ISD::EXTRACT_VECTOR_ELT, Ty, Custom);
171  setOperationAction(ISD::INSERT_VECTOR_ELT, Ty, Legal);
172  setOperationAction(ISD::BUILD_VECTOR, Ty, Custom);
173
174  setOperationAction(ISD::ADD, Ty, Legal);
175  setOperationAction(ISD::AND, Ty, Legal);
176  setOperationAction(ISD::CTLZ, Ty, Legal);
177  setOperationAction(ISD::CTPOP, Ty, Legal);
178  setOperationAction(ISD::MUL, Ty, Legal);
179  setOperationAction(ISD::OR, Ty, Legal);
180  setOperationAction(ISD::SDIV, Ty, Legal);
181  setOperationAction(ISD::SREM, Ty, Legal);
182  setOperationAction(ISD::SHL, Ty, Legal);
183  setOperationAction(ISD::SRA, Ty, Legal);
184  setOperationAction(ISD::SRL, Ty, Legal);
185  setOperationAction(ISD::SUB, Ty, Legal);
186  setOperationAction(ISD::UDIV, Ty, Legal);
187  setOperationAction(ISD::UREM, Ty, Legal);
188  setOperationAction(ISD::VECTOR_SHUFFLE, Ty, Custom);
189  setOperationAction(ISD::VSELECT, Ty, Legal);
190  setOperationAction(ISD::XOR, Ty, Legal);
191
192  if (Ty == MVT::v4i32 || Ty == MVT::v2i64) {
193    setOperationAction(ISD::FP_TO_SINT, Ty, Legal);
194    setOperationAction(ISD::FP_TO_UINT, Ty, Legal);
195    setOperationAction(ISD::SINT_TO_FP, Ty, Legal);
196    setOperationAction(ISD::UINT_TO_FP, Ty, Legal);
197  }
198
199  setOperationAction(ISD::SETCC, Ty, Legal);
200  setCondCodeAction(ISD::SETNE, Ty, Expand);
201  setCondCodeAction(ISD::SETGE, Ty, Expand);
202  setCondCodeAction(ISD::SETGT, Ty, Expand);
203  setCondCodeAction(ISD::SETUGE, Ty, Expand);
204  setCondCodeAction(ISD::SETUGT, Ty, Expand);
205}
206
207// Enable MSA support for the given floating-point type and Register class.
208void MipsSETargetLowering::
209addMSAFloatType(MVT::SimpleValueType Ty, const TargetRegisterClass *RC) {
210  addRegisterClass(Ty, RC);
211
212  // Expand all builtin opcodes.
213  for (unsigned Opc = 0; Opc < ISD::BUILTIN_OP_END; ++Opc)
214    setOperationAction(Opc, Ty, Expand);
215
216  setOperationAction(ISD::LOAD, Ty, Legal);
217  setOperationAction(ISD::STORE, Ty, Legal);
218  setOperationAction(ISD::BITCAST, Ty, Legal);
219  setOperationAction(ISD::EXTRACT_VECTOR_ELT, Ty, Legal);
220  setOperationAction(ISD::INSERT_VECTOR_ELT, Ty, Legal);
221  setOperationAction(ISD::BUILD_VECTOR, Ty, Custom);
222
223  if (Ty != MVT::v8f16) {
224    setOperationAction(ISD::FABS,  Ty, Legal);
225    setOperationAction(ISD::FADD,  Ty, Legal);
226    setOperationAction(ISD::FDIV,  Ty, Legal);
227    setOperationAction(ISD::FEXP2, Ty, Legal);
228    setOperationAction(ISD::FLOG2, Ty, Legal);
229    setOperationAction(ISD::FMA,   Ty, Legal);
230    setOperationAction(ISD::FMUL,  Ty, Legal);
231    setOperationAction(ISD::FRINT, Ty, Legal);
232    setOperationAction(ISD::FSQRT, Ty, Legal);
233    setOperationAction(ISD::FSUB,  Ty, Legal);
234    setOperationAction(ISD::VSELECT, Ty, Legal);
235
236    setOperationAction(ISD::SETCC, Ty, Legal);
237    setCondCodeAction(ISD::SETOGE, Ty, Expand);
238    setCondCodeAction(ISD::SETOGT, Ty, Expand);
239    setCondCodeAction(ISD::SETUGE, Ty, Expand);
240    setCondCodeAction(ISD::SETUGT, Ty, Expand);
241    setCondCodeAction(ISD::SETGE,  Ty, Expand);
242    setCondCodeAction(ISD::SETGT,  Ty, Expand);
243  }
244}
245
246bool
247MipsSETargetLowering::allowsUnalignedMemoryAccesses(EVT VT, bool *Fast) const {
248  MVT::SimpleValueType SVT = VT.getSimpleVT().SimpleTy;
249
250  switch (SVT) {
251  case MVT::i64:
252  case MVT::i32:
253    if (Fast)
254      *Fast = true;
255    return true;
256  default:
257    return false;
258  }
259}
260
261SDValue MipsSETargetLowering::LowerOperation(SDValue Op,
262                                             SelectionDAG &DAG) const {
263  switch(Op.getOpcode()) {
264  case ISD::LOAD:  return lowerLOAD(Op, DAG);
265  case ISD::STORE: return lowerSTORE(Op, DAG);
266  case ISD::SMUL_LOHI: return lowerMulDiv(Op, MipsISD::Mult, true, true, DAG);
267  case ISD::UMUL_LOHI: return lowerMulDiv(Op, MipsISD::Multu, true, true, DAG);
268  case ISD::MULHS:     return lowerMulDiv(Op, MipsISD::Mult, false, true, DAG);
269  case ISD::MULHU:     return lowerMulDiv(Op, MipsISD::Multu, false, true, DAG);
270  case ISD::MUL:       return lowerMulDiv(Op, MipsISD::Mult, true, false, DAG);
271  case ISD::SDIVREM:   return lowerMulDiv(Op, MipsISD::DivRem, true, true, DAG);
272  case ISD::UDIVREM:   return lowerMulDiv(Op, MipsISD::DivRemU, true, true,
273                                          DAG);
274  case ISD::INTRINSIC_WO_CHAIN: return lowerINTRINSIC_WO_CHAIN(Op, DAG);
275  case ISD::INTRINSIC_W_CHAIN:  return lowerINTRINSIC_W_CHAIN(Op, DAG);
276  case ISD::INTRINSIC_VOID:     return lowerINTRINSIC_VOID(Op, DAG);
277  case ISD::EXTRACT_VECTOR_ELT: return lowerEXTRACT_VECTOR_ELT(Op, DAG);
278  case ISD::BUILD_VECTOR:       return lowerBUILD_VECTOR(Op, DAG);
279  case ISD::VECTOR_SHUFFLE:     return lowerVECTOR_SHUFFLE(Op, DAG);
280  }
281
282  return MipsTargetLowering::LowerOperation(Op, DAG);
283}
284
285// selectMADD -
286// Transforms a subgraph in CurDAG if the following pattern is found:
287//  (addc multLo, Lo0), (adde multHi, Hi0),
288// where,
289//  multHi/Lo: product of multiplication
290//  Lo0: initial value of Lo register
291//  Hi0: initial value of Hi register
292// Return true if pattern matching was successful.
293static bool selectMADD(SDNode *ADDENode, SelectionDAG *CurDAG) {
294  // ADDENode's second operand must be a flag output of an ADDC node in order
295  // for the matching to be successful.
296  SDNode *ADDCNode = ADDENode->getOperand(2).getNode();
297
298  if (ADDCNode->getOpcode() != ISD::ADDC)
299    return false;
300
301  SDValue MultHi = ADDENode->getOperand(0);
302  SDValue MultLo = ADDCNode->getOperand(0);
303  SDNode *MultNode = MultHi.getNode();
304  unsigned MultOpc = MultHi.getOpcode();
305
306  // MultHi and MultLo must be generated by the same node,
307  if (MultLo.getNode() != MultNode)
308    return false;
309
310  // and it must be a multiplication.
311  if (MultOpc != ISD::SMUL_LOHI && MultOpc != ISD::UMUL_LOHI)
312    return false;
313
314  // MultLo amd MultHi must be the first and second output of MultNode
315  // respectively.
316  if (MultHi.getResNo() != 1 || MultLo.getResNo() != 0)
317    return false;
318
319  // Transform this to a MADD only if ADDENode and ADDCNode are the only users
320  // of the values of MultNode, in which case MultNode will be removed in later
321  // phases.
322  // If there exist users other than ADDENode or ADDCNode, this function returns
323  // here, which will result in MultNode being mapped to a single MULT
324  // instruction node rather than a pair of MULT and MADD instructions being
325  // produced.
326  if (!MultHi.hasOneUse() || !MultLo.hasOneUse())
327    return false;
328
329  SDLoc DL(ADDENode);
330
331  // Initialize accumulator.
332  SDValue ACCIn = CurDAG->getNode(MipsISD::MTLOHI, DL, MVT::Untyped,
333                                  ADDCNode->getOperand(1),
334                                  ADDENode->getOperand(1));
335
336  // create MipsMAdd(u) node
337  MultOpc = MultOpc == ISD::UMUL_LOHI ? MipsISD::MAddu : MipsISD::MAdd;
338
339  SDValue MAdd = CurDAG->getNode(MultOpc, DL, MVT::Untyped,
340                                 MultNode->getOperand(0),// Factor 0
341                                 MultNode->getOperand(1),// Factor 1
342                                 ACCIn);
343
344  // replace uses of adde and addc here
345  if (!SDValue(ADDCNode, 0).use_empty()) {
346    SDValue LoOut = CurDAG->getNode(MipsISD::MFLO, DL, MVT::i32, MAdd);
347    CurDAG->ReplaceAllUsesOfValueWith(SDValue(ADDCNode, 0), LoOut);
348  }
349  if (!SDValue(ADDENode, 0).use_empty()) {
350    SDValue HiOut = CurDAG->getNode(MipsISD::MFHI, DL, MVT::i32, MAdd);
351    CurDAG->ReplaceAllUsesOfValueWith(SDValue(ADDENode, 0), HiOut);
352  }
353
354  return true;
355}
356
357// selectMSUB -
358// Transforms a subgraph in CurDAG if the following pattern is found:
359//  (addc Lo0, multLo), (sube Hi0, multHi),
360// where,
361//  multHi/Lo: product of multiplication
362//  Lo0: initial value of Lo register
363//  Hi0: initial value of Hi register
364// Return true if pattern matching was successful.
365static bool selectMSUB(SDNode *SUBENode, SelectionDAG *CurDAG) {
366  // SUBENode's second operand must be a flag output of an SUBC node in order
367  // for the matching to be successful.
368  SDNode *SUBCNode = SUBENode->getOperand(2).getNode();
369
370  if (SUBCNode->getOpcode() != ISD::SUBC)
371    return false;
372
373  SDValue MultHi = SUBENode->getOperand(1);
374  SDValue MultLo = SUBCNode->getOperand(1);
375  SDNode *MultNode = MultHi.getNode();
376  unsigned MultOpc = MultHi.getOpcode();
377
378  // MultHi and MultLo must be generated by the same node,
379  if (MultLo.getNode() != MultNode)
380    return false;
381
382  // and it must be a multiplication.
383  if (MultOpc != ISD::SMUL_LOHI && MultOpc != ISD::UMUL_LOHI)
384    return false;
385
386  // MultLo amd MultHi must be the first and second output of MultNode
387  // respectively.
388  if (MultHi.getResNo() != 1 || MultLo.getResNo() != 0)
389    return false;
390
391  // Transform this to a MSUB only if SUBENode and SUBCNode are the only users
392  // of the values of MultNode, in which case MultNode will be removed in later
393  // phases.
394  // If there exist users other than SUBENode or SUBCNode, this function returns
395  // here, which will result in MultNode being mapped to a single MULT
396  // instruction node rather than a pair of MULT and MSUB instructions being
397  // produced.
398  if (!MultHi.hasOneUse() || !MultLo.hasOneUse())
399    return false;
400
401  SDLoc DL(SUBENode);
402
403  // Initialize accumulator.
404  SDValue ACCIn = CurDAG->getNode(MipsISD::MTLOHI, DL, MVT::Untyped,
405                                  SUBCNode->getOperand(0),
406                                  SUBENode->getOperand(0));
407
408  // create MipsSub(u) node
409  MultOpc = MultOpc == ISD::UMUL_LOHI ? MipsISD::MSubu : MipsISD::MSub;
410
411  SDValue MSub = CurDAG->getNode(MultOpc, DL, MVT::Glue,
412                                 MultNode->getOperand(0),// Factor 0
413                                 MultNode->getOperand(1),// Factor 1
414                                 ACCIn);
415
416  // replace uses of sube and subc here
417  if (!SDValue(SUBCNode, 0).use_empty()) {
418    SDValue LoOut = CurDAG->getNode(MipsISD::MFLO, DL, MVT::i32, MSub);
419    CurDAG->ReplaceAllUsesOfValueWith(SDValue(SUBCNode, 0), LoOut);
420  }
421  if (!SDValue(SUBENode, 0).use_empty()) {
422    SDValue HiOut = CurDAG->getNode(MipsISD::MFHI, DL, MVT::i32, MSub);
423    CurDAG->ReplaceAllUsesOfValueWith(SDValue(SUBENode, 0), HiOut);
424  }
425
426  return true;
427}
428
429static SDValue performADDECombine(SDNode *N, SelectionDAG &DAG,
430                                  TargetLowering::DAGCombinerInfo &DCI,
431                                  const MipsSubtarget *Subtarget) {
432  if (DCI.isBeforeLegalize())
433    return SDValue();
434
435  if (Subtarget->hasMips32() && N->getValueType(0) == MVT::i32 &&
436      selectMADD(N, &DAG))
437    return SDValue(N, 0);
438
439  return SDValue();
440}
441
442// Fold zero extensions into MipsISD::VEXTRACT_[SZ]EXT_ELT
443//
444// Performs the following transformations:
445// - Changes MipsISD::VEXTRACT_[SZ]EXT_ELT to zero extension if its
446//   sign/zero-extension is completely overwritten by the new one performed by
447//   the ISD::AND.
448// - Removes redundant zero extensions performed by an ISD::AND.
449static SDValue performANDCombine(SDNode *N, SelectionDAG &DAG,
450                                 TargetLowering::DAGCombinerInfo &DCI,
451                                 const MipsSubtarget *Subtarget) {
452  if (!Subtarget->hasMSA())
453    return SDValue();
454
455  SDValue Op0 = N->getOperand(0);
456  SDValue Op1 = N->getOperand(1);
457  unsigned Op0Opcode = Op0->getOpcode();
458
459  // (and (MipsVExtract[SZ]Ext $a, $b, $c), imm:$d)
460  // where $d + 1 == 2^n and n == 32
461  // or    $d + 1 == 2^n and n <= 32 and ZExt
462  // -> (MipsVExtractZExt $a, $b, $c)
463  if (Op0Opcode == MipsISD::VEXTRACT_SEXT_ELT ||
464      Op0Opcode == MipsISD::VEXTRACT_ZEXT_ELT) {
465    ConstantSDNode *Mask = dyn_cast<ConstantSDNode>(Op1);
466
467    if (!Mask)
468      return SDValue();
469
470    int32_t Log2IfPositive = (Mask->getAPIntValue() + 1).exactLogBase2();
471
472    if (Log2IfPositive <= 0)
473      return SDValue(); // Mask+1 is not a power of 2
474
475    SDValue Op0Op2 = Op0->getOperand(2);
476    EVT ExtendTy = cast<VTSDNode>(Op0Op2)->getVT();
477    unsigned ExtendTySize = ExtendTy.getSizeInBits();
478    unsigned Log2 = Log2IfPositive;
479
480    if ((Op0Opcode == MipsISD::VEXTRACT_ZEXT_ELT && Log2 >= ExtendTySize) ||
481        Log2 == ExtendTySize) {
482      SDValue Ops[] = { Op0->getOperand(0), Op0->getOperand(1), Op0Op2 };
483      DAG.MorphNodeTo(Op0.getNode(), MipsISD::VEXTRACT_ZEXT_ELT,
484                      Op0->getVTList(), Ops, Op0->getNumOperands());
485      return Op0;
486    }
487  }
488
489  return SDValue();
490}
491
492// Determine if the specified node is a constant vector splat.
493//
494// Returns true and sets Imm if:
495// * N is a ISD::BUILD_VECTOR representing a constant splat
496//
497// This function is quite similar to MipsSEDAGToDAGISel::selectVSplat. The
498// differences are that it assumes the MSA has already been checked and the
499// arbitrary requirement for a maximum of 32-bit integers isn't applied (and
500// must not be in order for binsri.d to be selectable).
501static bool isVSplat(SDValue N, APInt &Imm, bool IsLittleEndian) {
502  BuildVectorSDNode *Node = dyn_cast<BuildVectorSDNode>(N.getNode());
503
504  if (Node == NULL)
505    return false;
506
507  APInt SplatValue, SplatUndef;
508  unsigned SplatBitSize;
509  bool HasAnyUndefs;
510
511  if (!Node->isConstantSplat(SplatValue, SplatUndef, SplatBitSize, HasAnyUndefs,
512                             8, !IsLittleEndian))
513    return false;
514
515  Imm = SplatValue;
516
517  return true;
518}
519
520// Test whether the given node is an all-ones build_vector.
521static bool isVectorAllOnes(SDValue N) {
522  // Look through bitcasts. Endianness doesn't matter because we are looking
523  // for an all-ones value.
524  if (N->getOpcode() == ISD::BITCAST)
525    N = N->getOperand(0);
526
527  BuildVectorSDNode *BVN = dyn_cast<BuildVectorSDNode>(N);
528
529  if (!BVN)
530    return false;
531
532  APInt SplatValue, SplatUndef;
533  unsigned SplatBitSize;
534  bool HasAnyUndefs;
535
536  // Endianness doesn't matter in this context because we are looking for
537  // an all-ones value.
538  if (BVN->isConstantSplat(SplatValue, SplatUndef, SplatBitSize, HasAnyUndefs))
539    return SplatValue.isAllOnesValue();
540
541  return false;
542}
543
544// Test whether N is the bitwise inverse of OfNode.
545static bool isBitwiseInverse(SDValue N, SDValue OfNode) {
546  if (N->getOpcode() != ISD::XOR)
547    return false;
548
549  if (isVectorAllOnes(N->getOperand(0)))
550    return N->getOperand(1) == OfNode;
551
552  if (isVectorAllOnes(N->getOperand(1)))
553    return N->getOperand(0) == OfNode;
554
555  return false;
556}
557
558// Perform combines where ISD::OR is the root node.
559//
560// Performs the following transformations:
561// - (or (and $a, $mask), (and $b, $inv_mask)) => (vselect $mask, $a, $b)
562//   where $inv_mask is the bitwise inverse of $mask and the 'or' has a 128-bit
563//   vector type.
564static SDValue performORCombine(SDNode *N, SelectionDAG &DAG,
565                                TargetLowering::DAGCombinerInfo &DCI,
566                                const MipsSubtarget *Subtarget) {
567  if (!Subtarget->hasMSA())
568    return SDValue();
569
570  EVT Ty = N->getValueType(0);
571
572  if (!Ty.is128BitVector())
573    return SDValue();
574
575  SDValue Op0 = N->getOperand(0);
576  SDValue Op1 = N->getOperand(1);
577
578  if (Op0->getOpcode() == ISD::AND && Op1->getOpcode() == ISD::AND) {
579    SDValue Op0Op0 = Op0->getOperand(0);
580    SDValue Op0Op1 = Op0->getOperand(1);
581    SDValue Op1Op0 = Op1->getOperand(0);
582    SDValue Op1Op1 = Op1->getOperand(1);
583    bool IsLittleEndian = !Subtarget->isLittle();
584
585    SDValue IfSet, IfClr, Cond;
586    bool IsConstantMask = false;
587    APInt Mask, InvMask;
588
589    // If Op0Op0 is an appropriate mask, try to find it's inverse in either
590    // Op1Op0, or Op1Op1. Keep track of the Cond, IfSet, and IfClr nodes, while
591    // looking.
592    // IfClr will be set if we find a valid match.
593    if (isVSplat(Op0Op0, Mask, IsLittleEndian)) {
594      Cond = Op0Op0;
595      IfSet = Op0Op1;
596
597      if (isVSplat(Op1Op0, InvMask, IsLittleEndian) && Mask == ~InvMask)
598        IfClr = Op1Op1;
599      else if (isVSplat(Op1Op1, InvMask, IsLittleEndian) && Mask == ~InvMask)
600        IfClr = Op1Op0;
601
602      IsConstantMask = true;
603    }
604
605    // If IfClr is not yet set, and Op0Op1 is an appropriate mask, try the same
606    // thing again using this mask.
607    // IfClr will be set if we find a valid match.
608    if (!IfClr.getNode() && isVSplat(Op0Op1, Mask, IsLittleEndian)) {
609      Cond = Op0Op1;
610      IfSet = Op0Op0;
611
612      if (isVSplat(Op1Op0, InvMask, IsLittleEndian) && Mask == ~InvMask)
613        IfClr = Op1Op1;
614      else if (isVSplat(Op1Op1, InvMask, IsLittleEndian) && Mask == ~InvMask)
615        IfClr = Op1Op0;
616
617      IsConstantMask = true;
618    }
619
620    // If IfClr is not yet set, try looking for a non-constant match.
621    // IfClr will be set if we find a valid match amongst the eight
622    // possibilities.
623    if (!IfClr.getNode()) {
624      if (isBitwiseInverse(Op0Op0, Op1Op0)) {
625        Cond = Op1Op0;
626        IfSet = Op1Op1;
627        IfClr = Op0Op1;
628      } else if (isBitwiseInverse(Op0Op1, Op1Op0)) {
629        Cond = Op1Op0;
630        IfSet = Op1Op1;
631        IfClr = Op0Op0;
632      } else if (isBitwiseInverse(Op0Op0, Op1Op1)) {
633        Cond = Op1Op1;
634        IfSet = Op1Op0;
635        IfClr = Op0Op1;
636      } else if (isBitwiseInverse(Op0Op1, Op1Op1)) {
637        Cond = Op1Op1;
638        IfSet = Op1Op0;
639        IfClr = Op0Op0;
640      } else if (isBitwiseInverse(Op1Op0, Op0Op0)) {
641        Cond = Op0Op0;
642        IfSet = Op0Op1;
643        IfClr = Op1Op1;
644      } else if (isBitwiseInverse(Op1Op1, Op0Op0)) {
645        Cond = Op0Op0;
646        IfSet = Op0Op1;
647        IfClr = Op1Op0;
648      } else if (isBitwiseInverse(Op1Op0, Op0Op1)) {
649        Cond = Op0Op1;
650        IfSet = Op0Op0;
651        IfClr = Op1Op1;
652      } else if (isBitwiseInverse(Op1Op1, Op0Op1)) {
653        Cond = Op0Op1;
654        IfSet = Op0Op0;
655        IfClr = Op1Op0;
656      }
657    }
658
659    // At this point, IfClr will be set if we have a valid match.
660    if (!IfClr.getNode())
661      return SDValue();
662
663    assert(Cond.getNode() && IfSet.getNode());
664
665    // Fold degenerate cases.
666    if (IsConstantMask) {
667      if (Mask.isAllOnesValue())
668        return IfSet;
669      else if (Mask == 0)
670        return IfClr;
671    }
672
673    // Transform the DAG into an equivalent VSELECT.
674    return DAG.getNode(ISD::VSELECT, SDLoc(N), Ty, Cond, IfClr, IfSet);
675  }
676
677  return SDValue();
678}
679
680static SDValue performSUBECombine(SDNode *N, SelectionDAG &DAG,
681                                  TargetLowering::DAGCombinerInfo &DCI,
682                                  const MipsSubtarget *Subtarget) {
683  if (DCI.isBeforeLegalize())
684    return SDValue();
685
686  if (Subtarget->hasMips32() && N->getValueType(0) == MVT::i32 &&
687      selectMSUB(N, &DAG))
688    return SDValue(N, 0);
689
690  return SDValue();
691}
692
693static SDValue genConstMult(SDValue X, uint64_t C, SDLoc DL, EVT VT,
694                            EVT ShiftTy, SelectionDAG &DAG) {
695  // Clear the upper (64 - VT.sizeInBits) bits.
696  C &= ((uint64_t)-1) >> (64 - VT.getSizeInBits());
697
698  // Return 0.
699  if (C == 0)
700    return DAG.getConstant(0, VT);
701
702  // Return x.
703  if (C == 1)
704    return X;
705
706  // If c is power of 2, return (shl x, log2(c)).
707  if (isPowerOf2_64(C))
708    return DAG.getNode(ISD::SHL, DL, VT, X,
709                       DAG.getConstant(Log2_64(C), ShiftTy));
710
711  unsigned Log2Ceil = Log2_64_Ceil(C);
712  uint64_t Floor = 1LL << Log2_64(C);
713  uint64_t Ceil = Log2Ceil == 64 ? 0LL : 1LL << Log2Ceil;
714
715  // If |c - floor_c| <= |c - ceil_c|,
716  // where floor_c = pow(2, floor(log2(c))) and ceil_c = pow(2, ceil(log2(c))),
717  // return (add constMult(x, floor_c), constMult(x, c - floor_c)).
718  if (C - Floor <= Ceil - C) {
719    SDValue Op0 = genConstMult(X, Floor, DL, VT, ShiftTy, DAG);
720    SDValue Op1 = genConstMult(X, C - Floor, DL, VT, ShiftTy, DAG);
721    return DAG.getNode(ISD::ADD, DL, VT, Op0, Op1);
722  }
723
724  // If |c - floor_c| > |c - ceil_c|,
725  // return (sub constMult(x, ceil_c), constMult(x, ceil_c - c)).
726  SDValue Op0 = genConstMult(X, Ceil, DL, VT, ShiftTy, DAG);
727  SDValue Op1 = genConstMult(X, Ceil - C, DL, VT, ShiftTy, DAG);
728  return DAG.getNode(ISD::SUB, DL, VT, Op0, Op1);
729}
730
731static SDValue performMULCombine(SDNode *N, SelectionDAG &DAG,
732                                 const TargetLowering::DAGCombinerInfo &DCI,
733                                 const MipsSETargetLowering *TL) {
734  EVT VT = N->getValueType(0);
735
736  if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(N->getOperand(1)))
737    if (!VT.isVector())
738      return genConstMult(N->getOperand(0), C->getZExtValue(), SDLoc(N),
739                          VT, TL->getScalarShiftAmountTy(VT), DAG);
740
741  return SDValue(N, 0);
742}
743
744static SDValue performDSPShiftCombine(unsigned Opc, SDNode *N, EVT Ty,
745                                      SelectionDAG &DAG,
746                                      const MipsSubtarget *Subtarget) {
747  // See if this is a vector splat immediate node.
748  APInt SplatValue, SplatUndef;
749  unsigned SplatBitSize;
750  bool HasAnyUndefs;
751  unsigned EltSize = Ty.getVectorElementType().getSizeInBits();
752  BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(N->getOperand(1));
753
754  if (!BV ||
755      !BV->isConstantSplat(SplatValue, SplatUndef, SplatBitSize, HasAnyUndefs,
756                           EltSize, !Subtarget->isLittle()) ||
757      (SplatBitSize != EltSize) ||
758      (SplatValue.getZExtValue() >= EltSize))
759    return SDValue();
760
761  return DAG.getNode(Opc, SDLoc(N), Ty, N->getOperand(0),
762                     DAG.getConstant(SplatValue.getZExtValue(), MVT::i32));
763}
764
765static SDValue performSHLCombine(SDNode *N, SelectionDAG &DAG,
766                                 TargetLowering::DAGCombinerInfo &DCI,
767                                 const MipsSubtarget *Subtarget) {
768  EVT Ty = N->getValueType(0);
769
770  if ((Ty != MVT::v2i16) && (Ty != MVT::v4i8))
771    return SDValue();
772
773  return performDSPShiftCombine(MipsISD::SHLL_DSP, N, Ty, DAG, Subtarget);
774}
775
776// Fold sign-extensions into MipsISD::VEXTRACT_[SZ]EXT_ELT for MSA and fold
777// constant splats into MipsISD::SHRA_DSP for DSPr2.
778//
779// Performs the following transformations:
780// - Changes MipsISD::VEXTRACT_[SZ]EXT_ELT to sign extension if its
781//   sign/zero-extension is completely overwritten by the new one performed by
782//   the ISD::SRA and ISD::SHL nodes.
783// - Removes redundant sign extensions performed by an ISD::SRA and ISD::SHL
784//   sequence.
785//
786// See performDSPShiftCombine for more information about the transformation
787// used for DSPr2.
788static SDValue performSRACombine(SDNode *N, SelectionDAG &DAG,
789                                 TargetLowering::DAGCombinerInfo &DCI,
790                                 const MipsSubtarget *Subtarget) {
791  EVT Ty = N->getValueType(0);
792
793  if (Subtarget->hasMSA()) {
794    SDValue Op0 = N->getOperand(0);
795    SDValue Op1 = N->getOperand(1);
796
797    // (sra (shl (MipsVExtract[SZ]Ext $a, $b, $c), imm:$d), imm:$d)
798    // where $d + sizeof($c) == 32
799    // or    $d + sizeof($c) <= 32 and SExt
800    // -> (MipsVExtractSExt $a, $b, $c)
801    if (Op0->getOpcode() == ISD::SHL && Op1 == Op0->getOperand(1)) {
802      SDValue Op0Op0 = Op0->getOperand(0);
803      ConstantSDNode *ShAmount = dyn_cast<ConstantSDNode>(Op1);
804
805      if (!ShAmount)
806        return SDValue();
807
808      if (Op0Op0->getOpcode() != MipsISD::VEXTRACT_SEXT_ELT &&
809          Op0Op0->getOpcode() != MipsISD::VEXTRACT_ZEXT_ELT)
810        return SDValue();
811
812      EVT ExtendTy = cast<VTSDNode>(Op0Op0->getOperand(2))->getVT();
813      unsigned TotalBits = ShAmount->getZExtValue() + ExtendTy.getSizeInBits();
814
815      if (TotalBits == 32 ||
816          (Op0Op0->getOpcode() == MipsISD::VEXTRACT_SEXT_ELT &&
817           TotalBits <= 32)) {
818        SDValue Ops[] = { Op0Op0->getOperand(0), Op0Op0->getOperand(1),
819                          Op0Op0->getOperand(2) };
820        DAG.MorphNodeTo(Op0Op0.getNode(), MipsISD::VEXTRACT_SEXT_ELT,
821                        Op0Op0->getVTList(), Ops, Op0Op0->getNumOperands());
822        return Op0Op0;
823      }
824    }
825  }
826
827  if ((Ty != MVT::v2i16) && ((Ty != MVT::v4i8) || !Subtarget->hasDSPR2()))
828    return SDValue();
829
830  return performDSPShiftCombine(MipsISD::SHRA_DSP, N, Ty, DAG, Subtarget);
831}
832
833
834static SDValue performSRLCombine(SDNode *N, SelectionDAG &DAG,
835                                 TargetLowering::DAGCombinerInfo &DCI,
836                                 const MipsSubtarget *Subtarget) {
837  EVT Ty = N->getValueType(0);
838
839  if (((Ty != MVT::v2i16) || !Subtarget->hasDSPR2()) && (Ty != MVT::v4i8))
840    return SDValue();
841
842  return performDSPShiftCombine(MipsISD::SHRL_DSP, N, Ty, DAG, Subtarget);
843}
844
845static bool isLegalDSPCondCode(EVT Ty, ISD::CondCode CC) {
846  bool IsV216 = (Ty == MVT::v2i16);
847
848  switch (CC) {
849  case ISD::SETEQ:
850  case ISD::SETNE:  return true;
851  case ISD::SETLT:
852  case ISD::SETLE:
853  case ISD::SETGT:
854  case ISD::SETGE:  return IsV216;
855  case ISD::SETULT:
856  case ISD::SETULE:
857  case ISD::SETUGT:
858  case ISD::SETUGE: return !IsV216;
859  default:          return false;
860  }
861}
862
863static SDValue performSETCCCombine(SDNode *N, SelectionDAG &DAG) {
864  EVT Ty = N->getValueType(0);
865
866  if ((Ty != MVT::v2i16) && (Ty != MVT::v4i8))
867    return SDValue();
868
869  if (!isLegalDSPCondCode(Ty, cast<CondCodeSDNode>(N->getOperand(2))->get()))
870    return SDValue();
871
872  return DAG.getNode(MipsISD::SETCC_DSP, SDLoc(N), Ty, N->getOperand(0),
873                     N->getOperand(1), N->getOperand(2));
874}
875
876static SDValue performVSELECTCombine(SDNode *N, SelectionDAG &DAG) {
877  EVT Ty = N->getValueType(0);
878
879  if (Ty.is128BitVector() && Ty.isInteger()) {
880    // Try the following combines:
881    //   (vselect (setcc $a, $b, SETLT), $b, $a)) -> (vsmax $a, $b)
882    //   (vselect (setcc $a, $b, SETLE), $b, $a)) -> (vsmax $a, $b)
883    //   (vselect (setcc $a, $b, SETLT), $a, $b)) -> (vsmin $a, $b)
884    //   (vselect (setcc $a, $b, SETLE), $a, $b)) -> (vsmin $a, $b)
885    //   (vselect (setcc $a, $b, SETULT), $b, $a)) -> (vumax $a, $b)
886    //   (vselect (setcc $a, $b, SETULE), $b, $a)) -> (vumax $a, $b)
887    //   (vselect (setcc $a, $b, SETULT), $a, $b)) -> (vumin $a, $b)
888    //   (vselect (setcc $a, $b, SETULE), $a, $b)) -> (vumin $a, $b)
889    // SETGT/SETGE/SETUGT/SETUGE variants of these will show up initially but
890    // will be expanded to equivalent SETLT/SETLE/SETULT/SETULE versions by the
891    // legalizer.
892    SDValue Op0 = N->getOperand(0);
893
894    if (Op0->getOpcode() != ISD::SETCC)
895      return SDValue();
896
897    ISD::CondCode CondCode = cast<CondCodeSDNode>(Op0->getOperand(2))->get();
898    bool Signed;
899
900    if (CondCode == ISD::SETLT  || CondCode == ISD::SETLE)
901      Signed = true;
902    else if (CondCode == ISD::SETULT || CondCode == ISD::SETULE)
903      Signed = false;
904    else
905      return SDValue();
906
907    SDValue Op1 = N->getOperand(1);
908    SDValue Op2 = N->getOperand(2);
909    SDValue Op0Op0 = Op0->getOperand(0);
910    SDValue Op0Op1 = Op0->getOperand(1);
911
912    if (Op1 == Op0Op0 && Op2 == Op0Op1)
913      return DAG.getNode(Signed ? MipsISD::VSMIN : MipsISD::VUMIN, SDLoc(N),
914                         Ty, Op1, Op2);
915    else if (Op1 == Op0Op1 && Op2 == Op0Op0)
916      return DAG.getNode(Signed ? MipsISD::VSMAX : MipsISD::VUMAX, SDLoc(N),
917                         Ty, Op1, Op2);
918  } else if ((Ty == MVT::v2i16) || (Ty == MVT::v4i8)) {
919    SDValue SetCC = N->getOperand(0);
920
921    if (SetCC.getOpcode() != MipsISD::SETCC_DSP)
922      return SDValue();
923
924    return DAG.getNode(MipsISD::SELECT_CC_DSP, SDLoc(N), Ty,
925                       SetCC.getOperand(0), SetCC.getOperand(1),
926                       N->getOperand(1), N->getOperand(2), SetCC.getOperand(2));
927  }
928
929  return SDValue();
930}
931
932static SDValue performXORCombine(SDNode *N, SelectionDAG &DAG,
933                                 const MipsSubtarget *Subtarget) {
934  EVT Ty = N->getValueType(0);
935
936  if (Subtarget->hasMSA() && Ty.is128BitVector() && Ty.isInteger()) {
937    // Try the following combines:
938    //   (xor (or $a, $b), (build_vector allones))
939    //   (xor (or $a, $b), (bitcast (build_vector allones)))
940    SDValue Op0 = N->getOperand(0);
941    SDValue Op1 = N->getOperand(1);
942    SDValue NotOp;
943
944    if (ISD::isBuildVectorAllOnes(Op0.getNode()))
945      NotOp = Op1;
946    else if (ISD::isBuildVectorAllOnes(Op1.getNode()))
947      NotOp = Op0;
948    else
949      return SDValue();
950
951    if (NotOp->getOpcode() == ISD::OR)
952      return DAG.getNode(MipsISD::VNOR, SDLoc(N), Ty, NotOp->getOperand(0),
953                         NotOp->getOperand(1));
954  }
955
956  return SDValue();
957}
958
959SDValue
960MipsSETargetLowering::PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const {
961  SelectionDAG &DAG = DCI.DAG;
962  SDValue Val;
963
964  switch (N->getOpcode()) {
965  case ISD::ADDE:
966    return performADDECombine(N, DAG, DCI, Subtarget);
967  case ISD::AND:
968    Val = performANDCombine(N, DAG, DCI, Subtarget);
969    break;
970  case ISD::OR:
971    Val = performORCombine(N, DAG, DCI, Subtarget);
972    break;
973  case ISD::SUBE:
974    return performSUBECombine(N, DAG, DCI, Subtarget);
975  case ISD::MUL:
976    return performMULCombine(N, DAG, DCI, this);
977  case ISD::SHL:
978    return performSHLCombine(N, DAG, DCI, Subtarget);
979  case ISD::SRA:
980    return performSRACombine(N, DAG, DCI, Subtarget);
981  case ISD::SRL:
982    return performSRLCombine(N, DAG, DCI, Subtarget);
983  case ISD::VSELECT:
984    return performVSELECTCombine(N, DAG);
985  case ISD::XOR:
986    Val = performXORCombine(N, DAG, Subtarget);
987    break;
988  case ISD::SETCC:
989    Val = performSETCCCombine(N, DAG);
990    break;
991  }
992
993  if (Val.getNode()) {
994    DEBUG(dbgs() << "\nMipsSE DAG Combine:\n";
995          N->printrWithDepth(dbgs(), &DAG);
996          dbgs() << "\n=> \n";
997          Val.getNode()->printrWithDepth(dbgs(), &DAG);
998          dbgs() << "\n");
999    return Val;
1000  }
1001
1002  return MipsTargetLowering::PerformDAGCombine(N, DCI);
1003}
1004
1005MachineBasicBlock *
1006MipsSETargetLowering::EmitInstrWithCustomInserter(MachineInstr *MI,
1007                                                  MachineBasicBlock *BB) const {
1008  switch (MI->getOpcode()) {
1009  default:
1010    return MipsTargetLowering::EmitInstrWithCustomInserter(MI, BB);
1011  case Mips::BPOSGE32_PSEUDO:
1012    return emitBPOSGE32(MI, BB);
1013  case Mips::SNZ_B_PSEUDO:
1014    return emitMSACBranchPseudo(MI, BB, Mips::BNZ_B);
1015  case Mips::SNZ_H_PSEUDO:
1016    return emitMSACBranchPseudo(MI, BB, Mips::BNZ_H);
1017  case Mips::SNZ_W_PSEUDO:
1018    return emitMSACBranchPseudo(MI, BB, Mips::BNZ_W);
1019  case Mips::SNZ_D_PSEUDO:
1020    return emitMSACBranchPseudo(MI, BB, Mips::BNZ_D);
1021  case Mips::SNZ_V_PSEUDO:
1022    return emitMSACBranchPseudo(MI, BB, Mips::BNZ_V);
1023  case Mips::SZ_B_PSEUDO:
1024    return emitMSACBranchPseudo(MI, BB, Mips::BZ_B);
1025  case Mips::SZ_H_PSEUDO:
1026    return emitMSACBranchPseudo(MI, BB, Mips::BZ_H);
1027  case Mips::SZ_W_PSEUDO:
1028    return emitMSACBranchPseudo(MI, BB, Mips::BZ_W);
1029  case Mips::SZ_D_PSEUDO:
1030    return emitMSACBranchPseudo(MI, BB, Mips::BZ_D);
1031  case Mips::SZ_V_PSEUDO:
1032    return emitMSACBranchPseudo(MI, BB, Mips::BZ_V);
1033  case Mips::COPY_FW_PSEUDO:
1034    return emitCOPY_FW(MI, BB);
1035  case Mips::COPY_FD_PSEUDO:
1036    return emitCOPY_FD(MI, BB);
1037  case Mips::INSERT_FW_PSEUDO:
1038    return emitINSERT_FW(MI, BB);
1039  case Mips::INSERT_FD_PSEUDO:
1040    return emitINSERT_FD(MI, BB);
1041  case Mips::FILL_FW_PSEUDO:
1042    return emitFILL_FW(MI, BB);
1043  case Mips::FILL_FD_PSEUDO:
1044    return emitFILL_FD(MI, BB);
1045  case Mips::FEXP2_W_1_PSEUDO:
1046    return emitFEXP2_W_1(MI, BB);
1047  case Mips::FEXP2_D_1_PSEUDO:
1048    return emitFEXP2_D_1(MI, BB);
1049  }
1050}
1051
1052bool MipsSETargetLowering::
1053isEligibleForTailCallOptimization(const MipsCC &MipsCCInfo,
1054                                  unsigned NextStackOffset,
1055                                  const MipsFunctionInfo& FI) const {
1056  if (!EnableMipsTailCalls)
1057    return false;
1058
1059  // Return false if either the callee or caller has a byval argument.
1060  if (MipsCCInfo.hasByValArg() || FI.hasByvalArg())
1061    return false;
1062
1063  // Return true if the callee's argument area is no larger than the
1064  // caller's.
1065  return NextStackOffset <= FI.getIncomingArgSize();
1066}
1067
1068void MipsSETargetLowering::
1069getOpndList(SmallVectorImpl<SDValue> &Ops,
1070            std::deque< std::pair<unsigned, SDValue> > &RegsToPass,
1071            bool IsPICCall, bool GlobalOrExternal, bool InternalLinkage,
1072            CallLoweringInfo &CLI, SDValue Callee, SDValue Chain) const {
1073  // T9 should contain the address of the callee function if
1074  // -reloction-model=pic or it is an indirect call.
1075  if (IsPICCall || !GlobalOrExternal) {
1076    unsigned T9Reg = IsN64 ? Mips::T9_64 : Mips::T9;
1077    RegsToPass.push_front(std::make_pair(T9Reg, Callee));
1078  } else
1079    Ops.push_back(Callee);
1080
1081  MipsTargetLowering::getOpndList(Ops, RegsToPass, IsPICCall, GlobalOrExternal,
1082                                  InternalLinkage, CLI, Callee, Chain);
1083}
1084
1085SDValue MipsSETargetLowering::lowerLOAD(SDValue Op, SelectionDAG &DAG) const {
1086  LoadSDNode &Nd = *cast<LoadSDNode>(Op);
1087
1088  if (Nd.getMemoryVT() != MVT::f64 || !NoDPLoadStore)
1089    return MipsTargetLowering::lowerLOAD(Op, DAG);
1090
1091  // Replace a double precision load with two i32 loads and a buildpair64.
1092  SDLoc DL(Op);
1093  SDValue Ptr = Nd.getBasePtr(), Chain = Nd.getChain();
1094  EVT PtrVT = Ptr.getValueType();
1095
1096  // i32 load from lower address.
1097  SDValue Lo = DAG.getLoad(MVT::i32, DL, Chain, Ptr,
1098                           MachinePointerInfo(), Nd.isVolatile(),
1099                           Nd.isNonTemporal(), Nd.isInvariant(),
1100                           Nd.getAlignment());
1101
1102  // i32 load from higher address.
1103  Ptr = DAG.getNode(ISD::ADD, DL, PtrVT, Ptr, DAG.getConstant(4, PtrVT));
1104  SDValue Hi = DAG.getLoad(MVT::i32, DL, Lo.getValue(1), Ptr,
1105                           MachinePointerInfo(), Nd.isVolatile(),
1106                           Nd.isNonTemporal(), Nd.isInvariant(),
1107                           std::min(Nd.getAlignment(), 4U));
1108
1109  if (!Subtarget->isLittle())
1110    std::swap(Lo, Hi);
1111
1112  SDValue BP = DAG.getNode(MipsISD::BuildPairF64, DL, MVT::f64, Lo, Hi);
1113  SDValue Ops[2] = {BP, Hi.getValue(1)};
1114  return DAG.getMergeValues(Ops, 2, DL);
1115}
1116
1117SDValue MipsSETargetLowering::lowerSTORE(SDValue Op, SelectionDAG &DAG) const {
1118  StoreSDNode &Nd = *cast<StoreSDNode>(Op);
1119
1120  if (Nd.getMemoryVT() != MVT::f64 || !NoDPLoadStore)
1121    return MipsTargetLowering::lowerSTORE(Op, DAG);
1122
1123  // Replace a double precision store with two extractelement64s and i32 stores.
1124  SDLoc DL(Op);
1125  SDValue Val = Nd.getValue(), Ptr = Nd.getBasePtr(), Chain = Nd.getChain();
1126  EVT PtrVT = Ptr.getValueType();
1127  SDValue Lo = DAG.getNode(MipsISD::ExtractElementF64, DL, MVT::i32,
1128                           Val, DAG.getConstant(0, MVT::i32));
1129  SDValue Hi = DAG.getNode(MipsISD::ExtractElementF64, DL, MVT::i32,
1130                           Val, DAG.getConstant(1, MVT::i32));
1131
1132  if (!Subtarget->isLittle())
1133    std::swap(Lo, Hi);
1134
1135  // i32 store to lower address.
1136  Chain = DAG.getStore(Chain, DL, Lo, Ptr, MachinePointerInfo(),
1137                       Nd.isVolatile(), Nd.isNonTemporal(), Nd.getAlignment(),
1138                       Nd.getTBAAInfo());
1139
1140  // i32 store to higher address.
1141  Ptr = DAG.getNode(ISD::ADD, DL, PtrVT, Ptr, DAG.getConstant(4, PtrVT));
1142  return DAG.getStore(Chain, DL, Hi, Ptr, MachinePointerInfo(),
1143                      Nd.isVolatile(), Nd.isNonTemporal(),
1144                      std::min(Nd.getAlignment(), 4U), Nd.getTBAAInfo());
1145}
1146
1147SDValue MipsSETargetLowering::lowerMulDiv(SDValue Op, unsigned NewOpc,
1148                                          bool HasLo, bool HasHi,
1149                                          SelectionDAG &DAG) const {
1150  EVT Ty = Op.getOperand(0).getValueType();
1151  SDLoc DL(Op);
1152  SDValue Mult = DAG.getNode(NewOpc, DL, MVT::Untyped,
1153                             Op.getOperand(0), Op.getOperand(1));
1154  SDValue Lo, Hi;
1155
1156  if (HasLo)
1157    Lo = DAG.getNode(MipsISD::MFLO, DL, Ty, Mult);
1158  if (HasHi)
1159    Hi = DAG.getNode(MipsISD::MFHI, DL, Ty, Mult);
1160
1161  if (!HasLo || !HasHi)
1162    return HasLo ? Lo : Hi;
1163
1164  SDValue Vals[] = { Lo, Hi };
1165  return DAG.getMergeValues(Vals, 2, DL);
1166}
1167
1168
1169static SDValue initAccumulator(SDValue In, SDLoc DL, SelectionDAG &DAG) {
1170  SDValue InLo = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i32, In,
1171                             DAG.getConstant(0, MVT::i32));
1172  SDValue InHi = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i32, In,
1173                             DAG.getConstant(1, MVT::i32));
1174  return DAG.getNode(MipsISD::MTLOHI, DL, MVT::Untyped, InLo, InHi);
1175}
1176
1177static SDValue extractLOHI(SDValue Op, SDLoc DL, SelectionDAG &DAG) {
1178  SDValue Lo = DAG.getNode(MipsISD::MFLO, DL, MVT::i32, Op);
1179  SDValue Hi = DAG.getNode(MipsISD::MFHI, DL, MVT::i32, Op);
1180  return DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i64, Lo, Hi);
1181}
1182
1183// This function expands mips intrinsic nodes which have 64-bit input operands
1184// or output values.
1185//
1186// out64 = intrinsic-node in64
1187// =>
1188// lo = copy (extract-element (in64, 0))
1189// hi = copy (extract-element (in64, 1))
1190// mips-specific-node
1191// v0 = copy lo
1192// v1 = copy hi
1193// out64 = merge-values (v0, v1)
1194//
1195static SDValue lowerDSPIntr(SDValue Op, SelectionDAG &DAG, unsigned Opc) {
1196  SDLoc DL(Op);
1197  bool HasChainIn = Op->getOperand(0).getValueType() == MVT::Other;
1198  SmallVector<SDValue, 3> Ops;
1199  unsigned OpNo = 0;
1200
1201  // See if Op has a chain input.
1202  if (HasChainIn)
1203    Ops.push_back(Op->getOperand(OpNo++));
1204
1205  // The next operand is the intrinsic opcode.
1206  assert(Op->getOperand(OpNo).getOpcode() == ISD::TargetConstant);
1207
1208  // See if the next operand has type i64.
1209  SDValue Opnd = Op->getOperand(++OpNo), In64;
1210
1211  if (Opnd.getValueType() == MVT::i64)
1212    In64 = initAccumulator(Opnd, DL, DAG);
1213  else
1214    Ops.push_back(Opnd);
1215
1216  // Push the remaining operands.
1217  for (++OpNo ; OpNo < Op->getNumOperands(); ++OpNo)
1218    Ops.push_back(Op->getOperand(OpNo));
1219
1220  // Add In64 to the end of the list.
1221  if (In64.getNode())
1222    Ops.push_back(In64);
1223
1224  // Scan output.
1225  SmallVector<EVT, 2> ResTys;
1226
1227  for (SDNode::value_iterator I = Op->value_begin(), E = Op->value_end();
1228       I != E; ++I)
1229    ResTys.push_back((*I == MVT::i64) ? MVT::Untyped : *I);
1230
1231  // Create node.
1232  SDValue Val = DAG.getNode(Opc, DL, ResTys, &Ops[0], Ops.size());
1233  SDValue Out = (ResTys[0] == MVT::Untyped) ? extractLOHI(Val, DL, DAG) : Val;
1234
1235  if (!HasChainIn)
1236    return Out;
1237
1238  assert(Val->getValueType(1) == MVT::Other);
1239  SDValue Vals[] = { Out, SDValue(Val.getNode(), 1) };
1240  return DAG.getMergeValues(Vals, 2, DL);
1241}
1242
1243// Lower an MSA copy intrinsic into the specified SelectionDAG node
1244static SDValue lowerMSACopyIntr(SDValue Op, SelectionDAG &DAG, unsigned Opc) {
1245  SDLoc DL(Op);
1246  SDValue Vec = Op->getOperand(1);
1247  SDValue Idx = Op->getOperand(2);
1248  EVT ResTy = Op->getValueType(0);
1249  EVT EltTy = Vec->getValueType(0).getVectorElementType();
1250
1251  SDValue Result = DAG.getNode(Opc, DL, ResTy, Vec, Idx,
1252                               DAG.getValueType(EltTy));
1253
1254  return Result;
1255}
1256
1257static SDValue
1258lowerMSASplatImm(SDLoc DL, EVT ResTy, SDValue ImmOp, SelectionDAG &DAG) {
1259  EVT ViaVecTy = ResTy;
1260  SmallVector<SDValue, 16> Ops;
1261  SDValue ImmHiOp;
1262
1263  if (ViaVecTy == MVT::v2i64) {
1264    ImmHiOp = DAG.getNode(ISD::SRA, DL, MVT::i32, ImmOp,
1265                          DAG.getConstant(31, MVT::i32));
1266    for (unsigned i = 0; i < ViaVecTy.getVectorNumElements(); ++i) {
1267      Ops.push_back(ImmHiOp);
1268      Ops.push_back(ImmOp);
1269    }
1270    ViaVecTy = MVT::v4i32;
1271  } else {
1272    for (unsigned i = 0; i < ResTy.getVectorNumElements(); ++i)
1273      Ops.push_back(ImmOp);
1274  }
1275
1276  SDValue Result = DAG.getNode(ISD::BUILD_VECTOR, DL, ViaVecTy, &Ops[0],
1277                               Ops.size());
1278
1279  if (ResTy != ViaVecTy)
1280    Result = DAG.getNode(ISD::BITCAST, DL, ResTy, Result);
1281
1282  return Result;
1283}
1284
1285static SDValue
1286lowerMSASplatImm(SDValue Op, unsigned ImmOp, SelectionDAG &DAG) {
1287  return lowerMSASplatImm(SDLoc(Op), Op->getValueType(0),
1288                          Op->getOperand(ImmOp), DAG);
1289}
1290
1291SDValue MipsSETargetLowering::lowerINTRINSIC_WO_CHAIN(SDValue Op,
1292                                                      SelectionDAG &DAG) const {
1293  SDLoc DL(Op);
1294
1295  switch (cast<ConstantSDNode>(Op->getOperand(0))->getZExtValue()) {
1296  default:
1297    return SDValue();
1298  case Intrinsic::mips_shilo:
1299    return lowerDSPIntr(Op, DAG, MipsISD::SHILO);
1300  case Intrinsic::mips_dpau_h_qbl:
1301    return lowerDSPIntr(Op, DAG, MipsISD::DPAU_H_QBL);
1302  case Intrinsic::mips_dpau_h_qbr:
1303    return lowerDSPIntr(Op, DAG, MipsISD::DPAU_H_QBR);
1304  case Intrinsic::mips_dpsu_h_qbl:
1305    return lowerDSPIntr(Op, DAG, MipsISD::DPSU_H_QBL);
1306  case Intrinsic::mips_dpsu_h_qbr:
1307    return lowerDSPIntr(Op, DAG, MipsISD::DPSU_H_QBR);
1308  case Intrinsic::mips_dpa_w_ph:
1309    return lowerDSPIntr(Op, DAG, MipsISD::DPA_W_PH);
1310  case Intrinsic::mips_dps_w_ph:
1311    return lowerDSPIntr(Op, DAG, MipsISD::DPS_W_PH);
1312  case Intrinsic::mips_dpax_w_ph:
1313    return lowerDSPIntr(Op, DAG, MipsISD::DPAX_W_PH);
1314  case Intrinsic::mips_dpsx_w_ph:
1315    return lowerDSPIntr(Op, DAG, MipsISD::DPSX_W_PH);
1316  case Intrinsic::mips_mulsa_w_ph:
1317    return lowerDSPIntr(Op, DAG, MipsISD::MULSA_W_PH);
1318  case Intrinsic::mips_mult:
1319    return lowerDSPIntr(Op, DAG, MipsISD::Mult);
1320  case Intrinsic::mips_multu:
1321    return lowerDSPIntr(Op, DAG, MipsISD::Multu);
1322  case Intrinsic::mips_madd:
1323    return lowerDSPIntr(Op, DAG, MipsISD::MAdd);
1324  case Intrinsic::mips_maddu:
1325    return lowerDSPIntr(Op, DAG, MipsISD::MAddu);
1326  case Intrinsic::mips_msub:
1327    return lowerDSPIntr(Op, DAG, MipsISD::MSub);
1328  case Intrinsic::mips_msubu:
1329    return lowerDSPIntr(Op, DAG, MipsISD::MSubu);
1330  case Intrinsic::mips_addv_b:
1331  case Intrinsic::mips_addv_h:
1332  case Intrinsic::mips_addv_w:
1333  case Intrinsic::mips_addv_d:
1334    return DAG.getNode(ISD::ADD, DL, Op->getValueType(0), Op->getOperand(1),
1335                       Op->getOperand(2));
1336  case Intrinsic::mips_addvi_b:
1337  case Intrinsic::mips_addvi_h:
1338  case Intrinsic::mips_addvi_w:
1339  case Intrinsic::mips_addvi_d:
1340    return DAG.getNode(ISD::ADD, DL, Op->getValueType(0), Op->getOperand(1),
1341                       lowerMSASplatImm(Op, 2, DAG));
1342  case Intrinsic::mips_and_v:
1343    return DAG.getNode(ISD::AND, DL, Op->getValueType(0), Op->getOperand(1),
1344                       Op->getOperand(2));
1345  case Intrinsic::mips_andi_b:
1346    return DAG.getNode(ISD::AND, DL, Op->getValueType(0), Op->getOperand(1),
1347                       lowerMSASplatImm(Op, 2, DAG));
1348  case Intrinsic::mips_binsli_b:
1349  case Intrinsic::mips_binsli_h:
1350  case Intrinsic::mips_binsli_w:
1351  case Intrinsic::mips_binsli_d: {
1352    EVT VecTy = Op->getValueType(0);
1353    EVT EltTy = VecTy.getVectorElementType();
1354    APInt Mask = APInt::getHighBitsSet(EltTy.getSizeInBits(),
1355                                       Op->getConstantOperandVal(3));
1356    return DAG.getNode(ISD::VSELECT, DL, VecTy,
1357                       DAG.getConstant(Mask, VecTy, true), Op->getOperand(1),
1358                       Op->getOperand(2));
1359  }
1360  case Intrinsic::mips_binsri_b:
1361  case Intrinsic::mips_binsri_h:
1362  case Intrinsic::mips_binsri_w:
1363  case Intrinsic::mips_binsri_d: {
1364    EVT VecTy = Op->getValueType(0);
1365    EVT EltTy = VecTy.getVectorElementType();
1366    APInt Mask = APInt::getLowBitsSet(EltTy.getSizeInBits(),
1367                                      Op->getConstantOperandVal(3));
1368    return DAG.getNode(ISD::VSELECT, DL, VecTy,
1369                       DAG.getConstant(Mask, VecTy, true), Op->getOperand(1),
1370                       Op->getOperand(2));
1371  }
1372  case Intrinsic::mips_bmnz_v:
1373    return DAG.getNode(ISD::VSELECT, DL, Op->getValueType(0), Op->getOperand(3),
1374                       Op->getOperand(2), Op->getOperand(1));
1375  case Intrinsic::mips_bmnzi_b:
1376    return DAG.getNode(ISD::VSELECT, DL, Op->getValueType(0),
1377                       lowerMSASplatImm(Op, 3, DAG), Op->getOperand(2),
1378                       Op->getOperand(1));
1379  case Intrinsic::mips_bmz_v:
1380    return DAG.getNode(ISD::VSELECT, DL, Op->getValueType(0), Op->getOperand(3),
1381                       Op->getOperand(1), Op->getOperand(2));
1382  case Intrinsic::mips_bmzi_b:
1383    return DAG.getNode(ISD::VSELECT, DL, Op->getValueType(0),
1384                       lowerMSASplatImm(Op, 3, DAG), Op->getOperand(1),
1385                       Op->getOperand(2));
1386  case Intrinsic::mips_bnz_b:
1387  case Intrinsic::mips_bnz_h:
1388  case Intrinsic::mips_bnz_w:
1389  case Intrinsic::mips_bnz_d:
1390    return DAG.getNode(MipsISD::VALL_NONZERO, DL, Op->getValueType(0),
1391                       Op->getOperand(1));
1392  case Intrinsic::mips_bnz_v:
1393    return DAG.getNode(MipsISD::VANY_NONZERO, DL, Op->getValueType(0),
1394                       Op->getOperand(1));
1395  case Intrinsic::mips_bsel_v:
1396    return DAG.getNode(ISD::VSELECT, DL, Op->getValueType(0),
1397                       Op->getOperand(1), Op->getOperand(2),
1398                       Op->getOperand(3));
1399  case Intrinsic::mips_bseli_b:
1400    return DAG.getNode(ISD::VSELECT, DL, Op->getValueType(0),
1401                       Op->getOperand(1), Op->getOperand(2),
1402                       lowerMSASplatImm(Op, 3, DAG));
1403  case Intrinsic::mips_bz_b:
1404  case Intrinsic::mips_bz_h:
1405  case Intrinsic::mips_bz_w:
1406  case Intrinsic::mips_bz_d:
1407    return DAG.getNode(MipsISD::VALL_ZERO, DL, Op->getValueType(0),
1408                       Op->getOperand(1));
1409  case Intrinsic::mips_bz_v:
1410    return DAG.getNode(MipsISD::VANY_ZERO, DL, Op->getValueType(0),
1411                       Op->getOperand(1));
1412  case Intrinsic::mips_ceq_b:
1413  case Intrinsic::mips_ceq_h:
1414  case Intrinsic::mips_ceq_w:
1415  case Intrinsic::mips_ceq_d:
1416    return DAG.getSetCC(DL, Op->getValueType(0), Op->getOperand(1),
1417                        Op->getOperand(2), ISD::SETEQ);
1418  case Intrinsic::mips_ceqi_b:
1419  case Intrinsic::mips_ceqi_h:
1420  case Intrinsic::mips_ceqi_w:
1421  case Intrinsic::mips_ceqi_d:
1422    return DAG.getSetCC(DL, Op->getValueType(0), Op->getOperand(1),
1423                        lowerMSASplatImm(Op, 2, DAG), ISD::SETEQ);
1424  case Intrinsic::mips_cle_s_b:
1425  case Intrinsic::mips_cle_s_h:
1426  case Intrinsic::mips_cle_s_w:
1427  case Intrinsic::mips_cle_s_d:
1428    return DAG.getSetCC(DL, Op->getValueType(0), Op->getOperand(1),
1429                        Op->getOperand(2), ISD::SETLE);
1430  case Intrinsic::mips_clei_s_b:
1431  case Intrinsic::mips_clei_s_h:
1432  case Intrinsic::mips_clei_s_w:
1433  case Intrinsic::mips_clei_s_d:
1434    return DAG.getSetCC(DL, Op->getValueType(0), Op->getOperand(1),
1435                        lowerMSASplatImm(Op, 2, DAG), ISD::SETLE);
1436  case Intrinsic::mips_cle_u_b:
1437  case Intrinsic::mips_cle_u_h:
1438  case Intrinsic::mips_cle_u_w:
1439  case Intrinsic::mips_cle_u_d:
1440    return DAG.getSetCC(DL, Op->getValueType(0), Op->getOperand(1),
1441                        Op->getOperand(2), ISD::SETULE);
1442  case Intrinsic::mips_clei_u_b:
1443  case Intrinsic::mips_clei_u_h:
1444  case Intrinsic::mips_clei_u_w:
1445  case Intrinsic::mips_clei_u_d:
1446    return DAG.getSetCC(DL, Op->getValueType(0), Op->getOperand(1),
1447                        lowerMSASplatImm(Op, 2, DAG), ISD::SETULE);
1448  case Intrinsic::mips_clt_s_b:
1449  case Intrinsic::mips_clt_s_h:
1450  case Intrinsic::mips_clt_s_w:
1451  case Intrinsic::mips_clt_s_d:
1452    return DAG.getSetCC(DL, Op->getValueType(0), Op->getOperand(1),
1453                        Op->getOperand(2), ISD::SETLT);
1454  case Intrinsic::mips_clti_s_b:
1455  case Intrinsic::mips_clti_s_h:
1456  case Intrinsic::mips_clti_s_w:
1457  case Intrinsic::mips_clti_s_d:
1458    return DAG.getSetCC(DL, Op->getValueType(0), Op->getOperand(1),
1459                        lowerMSASplatImm(Op, 2, DAG), ISD::SETLT);
1460  case Intrinsic::mips_clt_u_b:
1461  case Intrinsic::mips_clt_u_h:
1462  case Intrinsic::mips_clt_u_w:
1463  case Intrinsic::mips_clt_u_d:
1464    return DAG.getSetCC(DL, Op->getValueType(0), Op->getOperand(1),
1465                        Op->getOperand(2), ISD::SETULT);
1466  case Intrinsic::mips_clti_u_b:
1467  case Intrinsic::mips_clti_u_h:
1468  case Intrinsic::mips_clti_u_w:
1469  case Intrinsic::mips_clti_u_d:
1470    return DAG.getSetCC(DL, Op->getValueType(0), Op->getOperand(1),
1471                        lowerMSASplatImm(Op, 2, DAG), ISD::SETULT);
1472  case Intrinsic::mips_copy_s_b:
1473  case Intrinsic::mips_copy_s_h:
1474  case Intrinsic::mips_copy_s_w:
1475    return lowerMSACopyIntr(Op, DAG, MipsISD::VEXTRACT_SEXT_ELT);
1476  case Intrinsic::mips_copy_s_d:
1477    // Don't lower directly into VEXTRACT_SEXT_ELT since i64 might be illegal.
1478    // Instead lower to the generic EXTRACT_VECTOR_ELT node and let the type
1479    // legalizer and EXTRACT_VECTOR_ELT lowering sort it out.
1480    return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(Op), Op->getValueType(0),
1481                       Op->getOperand(1), Op->getOperand(2));
1482  case Intrinsic::mips_copy_u_b:
1483  case Intrinsic::mips_copy_u_h:
1484  case Intrinsic::mips_copy_u_w:
1485    return lowerMSACopyIntr(Op, DAG, MipsISD::VEXTRACT_ZEXT_ELT);
1486  case Intrinsic::mips_copy_u_d:
1487    // Don't lower directly into VEXTRACT_ZEXT_ELT since i64 might be illegal.
1488    // Instead lower to the generic EXTRACT_VECTOR_ELT node and let the type
1489    // legalizer and EXTRACT_VECTOR_ELT lowering sort it out.
1490    //
1491    // Note: When i64 is illegal, this results in copy_s.w instructions instead
1492    // of copy_u.w instructions. This makes no difference to the behaviour
1493    // since i64 is only illegal when the register file is 32-bit.
1494    return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(Op), Op->getValueType(0),
1495                       Op->getOperand(1), Op->getOperand(2));
1496  case Intrinsic::mips_div_s_b:
1497  case Intrinsic::mips_div_s_h:
1498  case Intrinsic::mips_div_s_w:
1499  case Intrinsic::mips_div_s_d:
1500    return DAG.getNode(ISD::SDIV, DL, Op->getValueType(0), Op->getOperand(1),
1501                       Op->getOperand(2));
1502  case Intrinsic::mips_div_u_b:
1503  case Intrinsic::mips_div_u_h:
1504  case Intrinsic::mips_div_u_w:
1505  case Intrinsic::mips_div_u_d:
1506    return DAG.getNode(ISD::UDIV, DL, Op->getValueType(0), Op->getOperand(1),
1507                       Op->getOperand(2));
1508  case Intrinsic::mips_fadd_w:
1509  case Intrinsic::mips_fadd_d:
1510    return DAG.getNode(ISD::FADD, DL, Op->getValueType(0), Op->getOperand(1),
1511                       Op->getOperand(2));
1512  // Don't lower mips_fcaf_[wd] since LLVM folds SETFALSE condcodes away
1513  case Intrinsic::mips_fceq_w:
1514  case Intrinsic::mips_fceq_d:
1515    return DAG.getSetCC(DL, Op->getValueType(0), Op->getOperand(1),
1516                        Op->getOperand(2), ISD::SETOEQ);
1517  case Intrinsic::mips_fcle_w:
1518  case Intrinsic::mips_fcle_d:
1519    return DAG.getSetCC(DL, Op->getValueType(0), Op->getOperand(1),
1520                        Op->getOperand(2), ISD::SETOLE);
1521  case Intrinsic::mips_fclt_w:
1522  case Intrinsic::mips_fclt_d:
1523    return DAG.getSetCC(DL, Op->getValueType(0), Op->getOperand(1),
1524                        Op->getOperand(2), ISD::SETOLT);
1525  case Intrinsic::mips_fcne_w:
1526  case Intrinsic::mips_fcne_d:
1527    return DAG.getSetCC(DL, Op->getValueType(0), Op->getOperand(1),
1528                        Op->getOperand(2), ISD::SETONE);
1529  case Intrinsic::mips_fcor_w:
1530  case Intrinsic::mips_fcor_d:
1531    return DAG.getSetCC(DL, Op->getValueType(0), Op->getOperand(1),
1532                        Op->getOperand(2), ISD::SETO);
1533  case Intrinsic::mips_fcueq_w:
1534  case Intrinsic::mips_fcueq_d:
1535    return DAG.getSetCC(DL, Op->getValueType(0), Op->getOperand(1),
1536                        Op->getOperand(2), ISD::SETUEQ);
1537  case Intrinsic::mips_fcule_w:
1538  case Intrinsic::mips_fcule_d:
1539    return DAG.getSetCC(DL, Op->getValueType(0), Op->getOperand(1),
1540                        Op->getOperand(2), ISD::SETULE);
1541  case Intrinsic::mips_fcult_w:
1542  case Intrinsic::mips_fcult_d:
1543    return DAG.getSetCC(DL, Op->getValueType(0), Op->getOperand(1),
1544                        Op->getOperand(2), ISD::SETULT);
1545  case Intrinsic::mips_fcun_w:
1546  case Intrinsic::mips_fcun_d:
1547    return DAG.getSetCC(DL, Op->getValueType(0), Op->getOperand(1),
1548                        Op->getOperand(2), ISD::SETUO);
1549  case Intrinsic::mips_fcune_w:
1550  case Intrinsic::mips_fcune_d:
1551    return DAG.getSetCC(DL, Op->getValueType(0), Op->getOperand(1),
1552                        Op->getOperand(2), ISD::SETUNE);
1553  case Intrinsic::mips_fdiv_w:
1554  case Intrinsic::mips_fdiv_d:
1555    return DAG.getNode(ISD::FDIV, DL, Op->getValueType(0), Op->getOperand(1),
1556                       Op->getOperand(2));
1557  case Intrinsic::mips_ffint_u_w:
1558  case Intrinsic::mips_ffint_u_d:
1559    return DAG.getNode(ISD::UINT_TO_FP, DL, Op->getValueType(0),
1560                       Op->getOperand(1));
1561  case Intrinsic::mips_ffint_s_w:
1562  case Intrinsic::mips_ffint_s_d:
1563    return DAG.getNode(ISD::SINT_TO_FP, DL, Op->getValueType(0),
1564                       Op->getOperand(1));
1565  case Intrinsic::mips_fill_b:
1566  case Intrinsic::mips_fill_h:
1567  case Intrinsic::mips_fill_w:
1568  case Intrinsic::mips_fill_d: {
1569    SmallVector<SDValue, 16> Ops;
1570    EVT ResTy = Op->getValueType(0);
1571
1572    for (unsigned i = 0; i < ResTy.getVectorNumElements(); ++i)
1573      Ops.push_back(Op->getOperand(1));
1574
1575    // If ResTy is v2i64 then the type legalizer will break this node down into
1576    // an equivalent v4i32.
1577    return DAG.getNode(ISD::BUILD_VECTOR, DL, ResTy, &Ops[0], Ops.size());
1578  }
1579  case Intrinsic::mips_fexp2_w:
1580  case Intrinsic::mips_fexp2_d: {
1581    EVT ResTy = Op->getValueType(0);
1582    return DAG.getNode(
1583        ISD::FMUL, SDLoc(Op), ResTy, Op->getOperand(1),
1584        DAG.getNode(ISD::FEXP2, SDLoc(Op), ResTy, Op->getOperand(2)));
1585  }
1586  case Intrinsic::mips_flog2_w:
1587  case Intrinsic::mips_flog2_d:
1588    return DAG.getNode(ISD::FLOG2, DL, Op->getValueType(0), Op->getOperand(1));
1589  case Intrinsic::mips_fmadd_w:
1590  case Intrinsic::mips_fmadd_d:
1591    return DAG.getNode(ISD::FMA, SDLoc(Op), Op->getValueType(0),
1592                       Op->getOperand(1), Op->getOperand(2), Op->getOperand(3));
1593  case Intrinsic::mips_fmul_w:
1594  case Intrinsic::mips_fmul_d:
1595    return DAG.getNode(ISD::FMUL, DL, Op->getValueType(0), Op->getOperand(1),
1596                       Op->getOperand(2));
1597  case Intrinsic::mips_fmsub_w:
1598  case Intrinsic::mips_fmsub_d: {
1599    EVT ResTy = Op->getValueType(0);
1600    return DAG.getNode(ISD::FSUB, SDLoc(Op), ResTy, Op->getOperand(1),
1601                       DAG.getNode(ISD::FMUL, SDLoc(Op), ResTy,
1602                                   Op->getOperand(2), Op->getOperand(3)));
1603  }
1604  case Intrinsic::mips_frint_w:
1605  case Intrinsic::mips_frint_d:
1606    return DAG.getNode(ISD::FRINT, DL, Op->getValueType(0), Op->getOperand(1));
1607  case Intrinsic::mips_fsqrt_w:
1608  case Intrinsic::mips_fsqrt_d:
1609    return DAG.getNode(ISD::FSQRT, DL, Op->getValueType(0), Op->getOperand(1));
1610  case Intrinsic::mips_fsub_w:
1611  case Intrinsic::mips_fsub_d:
1612    return DAG.getNode(ISD::FSUB, DL, Op->getValueType(0), Op->getOperand(1),
1613                       Op->getOperand(2));
1614  case Intrinsic::mips_ftrunc_u_w:
1615  case Intrinsic::mips_ftrunc_u_d:
1616    return DAG.getNode(ISD::FP_TO_UINT, DL, Op->getValueType(0),
1617                       Op->getOperand(1));
1618  case Intrinsic::mips_ftrunc_s_w:
1619  case Intrinsic::mips_ftrunc_s_d:
1620    return DAG.getNode(ISD::FP_TO_SINT, DL, Op->getValueType(0),
1621                       Op->getOperand(1));
1622  case Intrinsic::mips_ilvev_b:
1623  case Intrinsic::mips_ilvev_h:
1624  case Intrinsic::mips_ilvev_w:
1625  case Intrinsic::mips_ilvev_d:
1626    return DAG.getNode(MipsISD::ILVEV, DL, Op->getValueType(0),
1627                       Op->getOperand(1), Op->getOperand(2));
1628  case Intrinsic::mips_ilvl_b:
1629  case Intrinsic::mips_ilvl_h:
1630  case Intrinsic::mips_ilvl_w:
1631  case Intrinsic::mips_ilvl_d:
1632    return DAG.getNode(MipsISD::ILVL, DL, Op->getValueType(0),
1633                       Op->getOperand(1), Op->getOperand(2));
1634  case Intrinsic::mips_ilvod_b:
1635  case Intrinsic::mips_ilvod_h:
1636  case Intrinsic::mips_ilvod_w:
1637  case Intrinsic::mips_ilvod_d:
1638    return DAG.getNode(MipsISD::ILVOD, DL, Op->getValueType(0),
1639                       Op->getOperand(1), Op->getOperand(2));
1640  case Intrinsic::mips_ilvr_b:
1641  case Intrinsic::mips_ilvr_h:
1642  case Intrinsic::mips_ilvr_w:
1643  case Intrinsic::mips_ilvr_d:
1644    return DAG.getNode(MipsISD::ILVR, DL, Op->getValueType(0),
1645                       Op->getOperand(1), Op->getOperand(2));
1646  case Intrinsic::mips_insert_b:
1647  case Intrinsic::mips_insert_h:
1648  case Intrinsic::mips_insert_w:
1649  case Intrinsic::mips_insert_d:
1650    return DAG.getNode(ISD::INSERT_VECTOR_ELT, SDLoc(Op), Op->getValueType(0),
1651                       Op->getOperand(1), Op->getOperand(3), Op->getOperand(2));
1652  case Intrinsic::mips_ldi_b:
1653  case Intrinsic::mips_ldi_h:
1654  case Intrinsic::mips_ldi_w:
1655  case Intrinsic::mips_ldi_d:
1656    return lowerMSASplatImm(Op, 1, DAG);
1657  case Intrinsic::mips_lsa: {
1658    EVT ResTy = Op->getValueType(0);
1659    return DAG.getNode(ISD::ADD, SDLoc(Op), ResTy, Op->getOperand(1),
1660                       DAG.getNode(ISD::SHL, SDLoc(Op), ResTy,
1661                                   Op->getOperand(2), Op->getOperand(3)));
1662  }
1663  case Intrinsic::mips_maddv_b:
1664  case Intrinsic::mips_maddv_h:
1665  case Intrinsic::mips_maddv_w:
1666  case Intrinsic::mips_maddv_d: {
1667    EVT ResTy = Op->getValueType(0);
1668    return DAG.getNode(ISD::ADD, SDLoc(Op), ResTy, Op->getOperand(1),
1669                       DAG.getNode(ISD::MUL, SDLoc(Op), ResTy,
1670                                   Op->getOperand(2), Op->getOperand(3)));
1671  }
1672  case Intrinsic::mips_max_s_b:
1673  case Intrinsic::mips_max_s_h:
1674  case Intrinsic::mips_max_s_w:
1675  case Intrinsic::mips_max_s_d:
1676    return DAG.getNode(MipsISD::VSMAX, DL, Op->getValueType(0),
1677                       Op->getOperand(1), Op->getOperand(2));
1678  case Intrinsic::mips_max_u_b:
1679  case Intrinsic::mips_max_u_h:
1680  case Intrinsic::mips_max_u_w:
1681  case Intrinsic::mips_max_u_d:
1682    return DAG.getNode(MipsISD::VUMAX, DL, Op->getValueType(0),
1683                       Op->getOperand(1), Op->getOperand(2));
1684  case Intrinsic::mips_maxi_s_b:
1685  case Intrinsic::mips_maxi_s_h:
1686  case Intrinsic::mips_maxi_s_w:
1687  case Intrinsic::mips_maxi_s_d:
1688    return DAG.getNode(MipsISD::VSMAX, DL, Op->getValueType(0),
1689                       Op->getOperand(1), lowerMSASplatImm(Op, 2, DAG));
1690  case Intrinsic::mips_maxi_u_b:
1691  case Intrinsic::mips_maxi_u_h:
1692  case Intrinsic::mips_maxi_u_w:
1693  case Intrinsic::mips_maxi_u_d:
1694    return DAG.getNode(MipsISD::VUMAX, DL, Op->getValueType(0),
1695                       Op->getOperand(1), lowerMSASplatImm(Op, 2, DAG));
1696  case Intrinsic::mips_min_s_b:
1697  case Intrinsic::mips_min_s_h:
1698  case Intrinsic::mips_min_s_w:
1699  case Intrinsic::mips_min_s_d:
1700    return DAG.getNode(MipsISD::VSMIN, DL, Op->getValueType(0),
1701                       Op->getOperand(1), Op->getOperand(2));
1702  case Intrinsic::mips_min_u_b:
1703  case Intrinsic::mips_min_u_h:
1704  case Intrinsic::mips_min_u_w:
1705  case Intrinsic::mips_min_u_d:
1706    return DAG.getNode(MipsISD::VUMIN, DL, Op->getValueType(0),
1707                       Op->getOperand(1), Op->getOperand(2));
1708  case Intrinsic::mips_mini_s_b:
1709  case Intrinsic::mips_mini_s_h:
1710  case Intrinsic::mips_mini_s_w:
1711  case Intrinsic::mips_mini_s_d:
1712    return DAG.getNode(MipsISD::VSMIN, DL, Op->getValueType(0),
1713                       Op->getOperand(1), lowerMSASplatImm(Op, 2, DAG));
1714  case Intrinsic::mips_mini_u_b:
1715  case Intrinsic::mips_mini_u_h:
1716  case Intrinsic::mips_mini_u_w:
1717  case Intrinsic::mips_mini_u_d:
1718    return DAG.getNode(MipsISD::VUMIN, DL, Op->getValueType(0),
1719                       Op->getOperand(1), lowerMSASplatImm(Op, 2, DAG));
1720  case Intrinsic::mips_mod_s_b:
1721  case Intrinsic::mips_mod_s_h:
1722  case Intrinsic::mips_mod_s_w:
1723  case Intrinsic::mips_mod_s_d:
1724    return DAG.getNode(ISD::SREM, DL, Op->getValueType(0), Op->getOperand(1),
1725                       Op->getOperand(2));
1726  case Intrinsic::mips_mod_u_b:
1727  case Intrinsic::mips_mod_u_h:
1728  case Intrinsic::mips_mod_u_w:
1729  case Intrinsic::mips_mod_u_d:
1730    return DAG.getNode(ISD::UREM, DL, Op->getValueType(0), Op->getOperand(1),
1731                       Op->getOperand(2));
1732  case Intrinsic::mips_mulv_b:
1733  case Intrinsic::mips_mulv_h:
1734  case Intrinsic::mips_mulv_w:
1735  case Intrinsic::mips_mulv_d:
1736    return DAG.getNode(ISD::MUL, DL, Op->getValueType(0), Op->getOperand(1),
1737                       Op->getOperand(2));
1738  case Intrinsic::mips_msubv_b:
1739  case Intrinsic::mips_msubv_h:
1740  case Intrinsic::mips_msubv_w:
1741  case Intrinsic::mips_msubv_d: {
1742    EVT ResTy = Op->getValueType(0);
1743    return DAG.getNode(ISD::SUB, SDLoc(Op), ResTy, Op->getOperand(1),
1744                       DAG.getNode(ISD::MUL, SDLoc(Op), ResTy,
1745                                   Op->getOperand(2), Op->getOperand(3)));
1746  }
1747  case Intrinsic::mips_nlzc_b:
1748  case Intrinsic::mips_nlzc_h:
1749  case Intrinsic::mips_nlzc_w:
1750  case Intrinsic::mips_nlzc_d:
1751    return DAG.getNode(ISD::CTLZ, DL, Op->getValueType(0), Op->getOperand(1));
1752  case Intrinsic::mips_nor_v: {
1753    SDValue Res = DAG.getNode(ISD::OR, DL, Op->getValueType(0),
1754                              Op->getOperand(1), Op->getOperand(2));
1755    return DAG.getNOT(DL, Res, Res->getValueType(0));
1756  }
1757  case Intrinsic::mips_nori_b: {
1758    SDValue Res =  DAG.getNode(ISD::OR, DL, Op->getValueType(0),
1759                               Op->getOperand(1),
1760                               lowerMSASplatImm(Op, 2, DAG));
1761    return DAG.getNOT(DL, Res, Res->getValueType(0));
1762  }
1763  case Intrinsic::mips_or_v:
1764    return DAG.getNode(ISD::OR, DL, Op->getValueType(0), Op->getOperand(1),
1765                       Op->getOperand(2));
1766  case Intrinsic::mips_ori_b:
1767    return DAG.getNode(ISD::OR, DL, Op->getValueType(0),
1768                       Op->getOperand(1), lowerMSASplatImm(Op, 2, DAG));
1769  case Intrinsic::mips_pckev_b:
1770  case Intrinsic::mips_pckev_h:
1771  case Intrinsic::mips_pckev_w:
1772  case Intrinsic::mips_pckev_d:
1773    return DAG.getNode(MipsISD::PCKEV, DL, Op->getValueType(0),
1774                       Op->getOperand(1), Op->getOperand(2));
1775  case Intrinsic::mips_pckod_b:
1776  case Intrinsic::mips_pckod_h:
1777  case Intrinsic::mips_pckod_w:
1778  case Intrinsic::mips_pckod_d:
1779    return DAG.getNode(MipsISD::PCKOD, DL, Op->getValueType(0),
1780                       Op->getOperand(1), Op->getOperand(2));
1781  case Intrinsic::mips_pcnt_b:
1782  case Intrinsic::mips_pcnt_h:
1783  case Intrinsic::mips_pcnt_w:
1784  case Intrinsic::mips_pcnt_d:
1785    return DAG.getNode(ISD::CTPOP, DL, Op->getValueType(0), Op->getOperand(1));
1786  case Intrinsic::mips_shf_b:
1787  case Intrinsic::mips_shf_h:
1788  case Intrinsic::mips_shf_w:
1789    return DAG.getNode(MipsISD::SHF, DL, Op->getValueType(0),
1790                       Op->getOperand(2), Op->getOperand(1));
1791  case Intrinsic::mips_sll_b:
1792  case Intrinsic::mips_sll_h:
1793  case Intrinsic::mips_sll_w:
1794  case Intrinsic::mips_sll_d:
1795    return DAG.getNode(ISD::SHL, DL, Op->getValueType(0), Op->getOperand(1),
1796                       Op->getOperand(2));
1797  case Intrinsic::mips_slli_b:
1798  case Intrinsic::mips_slli_h:
1799  case Intrinsic::mips_slli_w:
1800  case Intrinsic::mips_slli_d:
1801    return DAG.getNode(ISD::SHL, DL, Op->getValueType(0),
1802                       Op->getOperand(1), lowerMSASplatImm(Op, 2, DAG));
1803  case Intrinsic::mips_splat_b:
1804  case Intrinsic::mips_splat_h:
1805  case Intrinsic::mips_splat_w:
1806  case Intrinsic::mips_splat_d:
1807    // We can't lower via VECTOR_SHUFFLE because it requires constant shuffle
1808    // masks, nor can we lower via BUILD_VECTOR & EXTRACT_VECTOR_ELT because
1809    // EXTRACT_VECTOR_ELT can't extract i64's on MIPS32.
1810    // Instead we lower to MipsISD::VSHF and match from there.
1811    return DAG.getNode(MipsISD::VSHF, DL, Op->getValueType(0),
1812                       lowerMSASplatImm(Op, 2, DAG), Op->getOperand(1),
1813                       Op->getOperand(1));
1814  case Intrinsic::mips_splati_b:
1815  case Intrinsic::mips_splati_h:
1816  case Intrinsic::mips_splati_w:
1817  case Intrinsic::mips_splati_d:
1818    return DAG.getNode(MipsISD::VSHF, DL, Op->getValueType(0),
1819                       lowerMSASplatImm(Op, 2, DAG), Op->getOperand(1),
1820                       Op->getOperand(1));
1821  case Intrinsic::mips_sra_b:
1822  case Intrinsic::mips_sra_h:
1823  case Intrinsic::mips_sra_w:
1824  case Intrinsic::mips_sra_d:
1825    return DAG.getNode(ISD::SRA, DL, Op->getValueType(0), Op->getOperand(1),
1826                       Op->getOperand(2));
1827  case Intrinsic::mips_srai_b:
1828  case Intrinsic::mips_srai_h:
1829  case Intrinsic::mips_srai_w:
1830  case Intrinsic::mips_srai_d:
1831    return DAG.getNode(ISD::SRA, DL, Op->getValueType(0),
1832                       Op->getOperand(1), lowerMSASplatImm(Op, 2, DAG));
1833  case Intrinsic::mips_srl_b:
1834  case Intrinsic::mips_srl_h:
1835  case Intrinsic::mips_srl_w:
1836  case Intrinsic::mips_srl_d:
1837    return DAG.getNode(ISD::SRL, DL, Op->getValueType(0), Op->getOperand(1),
1838                       Op->getOperand(2));
1839  case Intrinsic::mips_srli_b:
1840  case Intrinsic::mips_srli_h:
1841  case Intrinsic::mips_srli_w:
1842  case Intrinsic::mips_srli_d:
1843    return DAG.getNode(ISD::SRL, DL, Op->getValueType(0),
1844                       Op->getOperand(1), lowerMSASplatImm(Op, 2, DAG));
1845  case Intrinsic::mips_subv_b:
1846  case Intrinsic::mips_subv_h:
1847  case Intrinsic::mips_subv_w:
1848  case Intrinsic::mips_subv_d:
1849    return DAG.getNode(ISD::SUB, DL, Op->getValueType(0), Op->getOperand(1),
1850                       Op->getOperand(2));
1851  case Intrinsic::mips_subvi_b:
1852  case Intrinsic::mips_subvi_h:
1853  case Intrinsic::mips_subvi_w:
1854  case Intrinsic::mips_subvi_d:
1855    return DAG.getNode(ISD::SUB, DL, Op->getValueType(0),
1856                       Op->getOperand(1), lowerMSASplatImm(Op, 2, DAG));
1857  case Intrinsic::mips_vshf_b:
1858  case Intrinsic::mips_vshf_h:
1859  case Intrinsic::mips_vshf_w:
1860  case Intrinsic::mips_vshf_d:
1861    return DAG.getNode(MipsISD::VSHF, DL, Op->getValueType(0),
1862                       Op->getOperand(1), Op->getOperand(2), Op->getOperand(3));
1863  case Intrinsic::mips_xor_v:
1864    return DAG.getNode(ISD::XOR, DL, Op->getValueType(0), Op->getOperand(1),
1865                       Op->getOperand(2));
1866  case Intrinsic::mips_xori_b:
1867    return DAG.getNode(ISD::XOR, DL, Op->getValueType(0),
1868                       Op->getOperand(1), lowerMSASplatImm(Op, 2, DAG));
1869  }
1870}
1871
1872static SDValue lowerMSALoadIntr(SDValue Op, SelectionDAG &DAG, unsigned Intr) {
1873  SDLoc DL(Op);
1874  SDValue ChainIn = Op->getOperand(0);
1875  SDValue Address = Op->getOperand(2);
1876  SDValue Offset  = Op->getOperand(3);
1877  EVT ResTy = Op->getValueType(0);
1878  EVT PtrTy = Address->getValueType(0);
1879
1880  Address = DAG.getNode(ISD::ADD, DL, PtrTy, Address, Offset);
1881
1882  return DAG.getLoad(ResTy, DL, ChainIn, Address, MachinePointerInfo(), false,
1883                     false, false, 16);
1884}
1885
1886SDValue MipsSETargetLowering::lowerINTRINSIC_W_CHAIN(SDValue Op,
1887                                                     SelectionDAG &DAG) const {
1888  unsigned Intr = cast<ConstantSDNode>(Op->getOperand(1))->getZExtValue();
1889  switch (Intr) {
1890  default:
1891    return SDValue();
1892  case Intrinsic::mips_extp:
1893    return lowerDSPIntr(Op, DAG, MipsISD::EXTP);
1894  case Intrinsic::mips_extpdp:
1895    return lowerDSPIntr(Op, DAG, MipsISD::EXTPDP);
1896  case Intrinsic::mips_extr_w:
1897    return lowerDSPIntr(Op, DAG, MipsISD::EXTR_W);
1898  case Intrinsic::mips_extr_r_w:
1899    return lowerDSPIntr(Op, DAG, MipsISD::EXTR_R_W);
1900  case Intrinsic::mips_extr_rs_w:
1901    return lowerDSPIntr(Op, DAG, MipsISD::EXTR_RS_W);
1902  case Intrinsic::mips_extr_s_h:
1903    return lowerDSPIntr(Op, DAG, MipsISD::EXTR_S_H);
1904  case Intrinsic::mips_mthlip:
1905    return lowerDSPIntr(Op, DAG, MipsISD::MTHLIP);
1906  case Intrinsic::mips_mulsaq_s_w_ph:
1907    return lowerDSPIntr(Op, DAG, MipsISD::MULSAQ_S_W_PH);
1908  case Intrinsic::mips_maq_s_w_phl:
1909    return lowerDSPIntr(Op, DAG, MipsISD::MAQ_S_W_PHL);
1910  case Intrinsic::mips_maq_s_w_phr:
1911    return lowerDSPIntr(Op, DAG, MipsISD::MAQ_S_W_PHR);
1912  case Intrinsic::mips_maq_sa_w_phl:
1913    return lowerDSPIntr(Op, DAG, MipsISD::MAQ_SA_W_PHL);
1914  case Intrinsic::mips_maq_sa_w_phr:
1915    return lowerDSPIntr(Op, DAG, MipsISD::MAQ_SA_W_PHR);
1916  case Intrinsic::mips_dpaq_s_w_ph:
1917    return lowerDSPIntr(Op, DAG, MipsISD::DPAQ_S_W_PH);
1918  case Intrinsic::mips_dpsq_s_w_ph:
1919    return lowerDSPIntr(Op, DAG, MipsISD::DPSQ_S_W_PH);
1920  case Intrinsic::mips_dpaq_sa_l_w:
1921    return lowerDSPIntr(Op, DAG, MipsISD::DPAQ_SA_L_W);
1922  case Intrinsic::mips_dpsq_sa_l_w:
1923    return lowerDSPIntr(Op, DAG, MipsISD::DPSQ_SA_L_W);
1924  case Intrinsic::mips_dpaqx_s_w_ph:
1925    return lowerDSPIntr(Op, DAG, MipsISD::DPAQX_S_W_PH);
1926  case Intrinsic::mips_dpaqx_sa_w_ph:
1927    return lowerDSPIntr(Op, DAG, MipsISD::DPAQX_SA_W_PH);
1928  case Intrinsic::mips_dpsqx_s_w_ph:
1929    return lowerDSPIntr(Op, DAG, MipsISD::DPSQX_S_W_PH);
1930  case Intrinsic::mips_dpsqx_sa_w_ph:
1931    return lowerDSPIntr(Op, DAG, MipsISD::DPSQX_SA_W_PH);
1932  case Intrinsic::mips_ld_b:
1933  case Intrinsic::mips_ld_h:
1934  case Intrinsic::mips_ld_w:
1935  case Intrinsic::mips_ld_d:
1936   return lowerMSALoadIntr(Op, DAG, Intr);
1937  }
1938}
1939
1940static SDValue lowerMSAStoreIntr(SDValue Op, SelectionDAG &DAG, unsigned Intr) {
1941  SDLoc DL(Op);
1942  SDValue ChainIn = Op->getOperand(0);
1943  SDValue Value   = Op->getOperand(2);
1944  SDValue Address = Op->getOperand(3);
1945  SDValue Offset  = Op->getOperand(4);
1946  EVT PtrTy = Address->getValueType(0);
1947
1948  Address = DAG.getNode(ISD::ADD, DL, PtrTy, Address, Offset);
1949
1950  return DAG.getStore(ChainIn, DL, Value, Address, MachinePointerInfo(), false,
1951                      false, 16);
1952}
1953
1954SDValue MipsSETargetLowering::lowerINTRINSIC_VOID(SDValue Op,
1955                                                  SelectionDAG &DAG) const {
1956  unsigned Intr = cast<ConstantSDNode>(Op->getOperand(1))->getZExtValue();
1957  switch (Intr) {
1958  default:
1959    return SDValue();
1960  case Intrinsic::mips_st_b:
1961  case Intrinsic::mips_st_h:
1962  case Intrinsic::mips_st_w:
1963  case Intrinsic::mips_st_d:
1964    return lowerMSAStoreIntr(Op, DAG, Intr);
1965  }
1966}
1967
1968/// \brief Check if the given BuildVectorSDNode is a splat.
1969/// This method currently relies on DAG nodes being reused when equivalent,
1970/// so it's possible for this to return false even when isConstantSplat returns
1971/// true.
1972static bool isSplatVector(const BuildVectorSDNode *N) {
1973  unsigned int nOps = N->getNumOperands();
1974  assert(nOps > 1 && "isSplatVector has 0 or 1 sized build vector");
1975
1976  SDValue Operand0 = N->getOperand(0);
1977
1978  for (unsigned int i = 1; i < nOps; ++i) {
1979    if (N->getOperand(i) != Operand0)
1980      return false;
1981  }
1982
1983  return true;
1984}
1985
1986// Lower ISD::EXTRACT_VECTOR_ELT into MipsISD::VEXTRACT_SEXT_ELT.
1987//
1988// The non-value bits resulting from ISD::EXTRACT_VECTOR_ELT are undefined. We
1989// choose to sign-extend but we could have equally chosen zero-extend. The
1990// DAGCombiner will fold any sign/zero extension of the ISD::EXTRACT_VECTOR_ELT
1991// result into this node later (possibly changing it to a zero-extend in the
1992// process).
1993SDValue MipsSETargetLowering::
1994lowerEXTRACT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG) const {
1995  SDLoc DL(Op);
1996  EVT ResTy = Op->getValueType(0);
1997  SDValue Op0 = Op->getOperand(0);
1998  EVT VecTy = Op0->getValueType(0);
1999
2000  if (!VecTy.is128BitVector())
2001    return SDValue();
2002
2003  if (ResTy.isInteger()) {
2004    SDValue Op1 = Op->getOperand(1);
2005    EVT EltTy = VecTy.getVectorElementType();
2006    return DAG.getNode(MipsISD::VEXTRACT_SEXT_ELT, DL, ResTy, Op0, Op1,
2007                       DAG.getValueType(EltTy));
2008  }
2009
2010  return Op;
2011}
2012
2013static bool isConstantOrUndef(const SDValue Op) {
2014  if (Op->getOpcode() == ISD::UNDEF)
2015    return true;
2016  if (dyn_cast<ConstantSDNode>(Op))
2017    return true;
2018  if (dyn_cast<ConstantFPSDNode>(Op))
2019    return true;
2020  return false;
2021}
2022
2023static bool isConstantOrUndefBUILD_VECTOR(const BuildVectorSDNode *Op) {
2024  for (unsigned i = 0; i < Op->getNumOperands(); ++i)
2025    if (isConstantOrUndef(Op->getOperand(i)))
2026      return true;
2027  return false;
2028}
2029
2030// Lowers ISD::BUILD_VECTOR into appropriate SelectionDAG nodes for the
2031// backend.
2032//
2033// Lowers according to the following rules:
2034// - Constant splats are legal as-is as long as the SplatBitSize is a power of
2035//   2 less than or equal to 64 and the value fits into a signed 10-bit
2036//   immediate
2037// - Constant splats are lowered to bitconverted BUILD_VECTORs if SplatBitSize
2038//   is a power of 2 less than or equal to 64 and the value does not fit into a
2039//   signed 10-bit immediate
2040// - Non-constant splats are legal as-is.
2041// - Non-constant non-splats are lowered to sequences of INSERT_VECTOR_ELT.
2042// - All others are illegal and must be expanded.
2043SDValue MipsSETargetLowering::lowerBUILD_VECTOR(SDValue Op,
2044                                                SelectionDAG &DAG) const {
2045  BuildVectorSDNode *Node = cast<BuildVectorSDNode>(Op);
2046  EVT ResTy = Op->getValueType(0);
2047  SDLoc DL(Op);
2048  APInt SplatValue, SplatUndef;
2049  unsigned SplatBitSize;
2050  bool HasAnyUndefs;
2051
2052  if (!Subtarget->hasMSA() || !ResTy.is128BitVector())
2053    return SDValue();
2054
2055  if (Node->isConstantSplat(SplatValue, SplatUndef, SplatBitSize,
2056                            HasAnyUndefs, 8,
2057                            !Subtarget->isLittle()) && SplatBitSize <= 64) {
2058    // We can only cope with 8, 16, 32, or 64-bit elements
2059    if (SplatBitSize != 8 && SplatBitSize != 16 && SplatBitSize != 32 &&
2060        SplatBitSize != 64)
2061      return SDValue();
2062
2063    // If the value fits into a simm10 then we can use ldi.[bhwd]
2064    if (SplatValue.isSignedIntN(10))
2065      return Op;
2066
2067    EVT ViaVecTy;
2068
2069    switch (SplatBitSize) {
2070    default:
2071      return SDValue();
2072    case 8:
2073      ViaVecTy = MVT::v16i8;
2074      break;
2075    case 16:
2076      ViaVecTy = MVT::v8i16;
2077      break;
2078    case 32:
2079      ViaVecTy = MVT::v4i32;
2080      break;
2081    case 64:
2082      // There's no fill.d to fall back on for 64-bit values
2083      return SDValue();
2084    }
2085
2086    SmallVector<SDValue, 16> Ops;
2087    SDValue Constant = DAG.getConstant(SplatValue.sextOrSelf(32), MVT::i32);
2088
2089    for (unsigned i = 0; i < ViaVecTy.getVectorNumElements(); ++i)
2090      Ops.push_back(Constant);
2091
2092    SDValue Result = DAG.getNode(ISD::BUILD_VECTOR, SDLoc(Node), ViaVecTy,
2093                                 &Ops[0], Ops.size());
2094
2095    if (ViaVecTy != ResTy)
2096      Result = DAG.getNode(ISD::BITCAST, SDLoc(Node), ResTy, Result);
2097
2098    return Result;
2099  } else if (isSplatVector(Node))
2100    return Op;
2101  else if (!isConstantOrUndefBUILD_VECTOR(Node)) {
2102    // Use INSERT_VECTOR_ELT operations rather than expand to stores.
2103    // The resulting code is the same length as the expansion, but it doesn't
2104    // use memory operations
2105    EVT ResTy = Node->getValueType(0);
2106
2107    assert(ResTy.isVector());
2108
2109    unsigned NumElts = ResTy.getVectorNumElements();
2110    SDValue Vector = DAG.getUNDEF(ResTy);
2111    for (unsigned i = 0; i < NumElts; ++i) {
2112      Vector = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, ResTy, Vector,
2113                           Node->getOperand(i),
2114                           DAG.getConstant(i, MVT::i32));
2115    }
2116    return Vector;
2117  }
2118
2119  return SDValue();
2120}
2121
2122// Lower VECTOR_SHUFFLE into SHF (if possible).
2123//
2124// SHF splits the vector into blocks of four elements, then shuffles these
2125// elements according to a <4 x i2> constant (encoded as an integer immediate).
2126//
2127// It is therefore possible to lower into SHF when the mask takes the form:
2128//   <a, b, c, d, a+4, b+4, c+4, d+4, a+8, b+8, c+8, d+8, ...>
2129// When undef's appear they are treated as if they were whatever value is
2130// necessary in order to fit the above form.
2131//
2132// For example:
2133//   %2 = shufflevector <8 x i16> %0, <8 x i16> undef,
2134//                      <8 x i32> <i32 3, i32 2, i32 1, i32 0,
2135//                                 i32 7, i32 6, i32 5, i32 4>
2136// is lowered to:
2137//   (SHF_H $w0, $w1, 27)
2138// where the 27 comes from:
2139//   3 + (2 << 2) + (1 << 4) + (0 << 6)
2140static SDValue lowerVECTOR_SHUFFLE_SHF(SDValue Op, EVT ResTy,
2141                                       SmallVector<int, 16> Indices,
2142                                       SelectionDAG &DAG) {
2143  int SHFIndices[4] = { -1, -1, -1, -1 };
2144
2145  if (Indices.size() < 4)
2146    return SDValue();
2147
2148  for (unsigned i = 0; i < 4; ++i) {
2149    for (unsigned j = i; j < Indices.size(); j += 4) {
2150      int Idx = Indices[j];
2151
2152      // Convert from vector index to 4-element subvector index
2153      // If an index refers to an element outside of the subvector then give up
2154      if (Idx != -1) {
2155        Idx -= 4 * (j / 4);
2156        if (Idx < 0 || Idx >= 4)
2157          return SDValue();
2158      }
2159
2160      // If the mask has an undef, replace it with the current index.
2161      // Note that it might still be undef if the current index is also undef
2162      if (SHFIndices[i] == -1)
2163        SHFIndices[i] = Idx;
2164
2165      // Check that non-undef values are the same as in the mask. If they
2166      // aren't then give up
2167      if (!(Idx == -1 || Idx == SHFIndices[i]))
2168        return SDValue();
2169    }
2170  }
2171
2172  // Calculate the immediate. Replace any remaining undefs with zero
2173  APInt Imm(32, 0);
2174  for (int i = 3; i >= 0; --i) {
2175    int Idx = SHFIndices[i];
2176
2177    if (Idx == -1)
2178      Idx = 0;
2179
2180    Imm <<= 2;
2181    Imm |= Idx & 0x3;
2182  }
2183
2184  return DAG.getNode(MipsISD::SHF, SDLoc(Op), ResTy,
2185                     DAG.getConstant(Imm, MVT::i32), Op->getOperand(0));
2186}
2187
2188// Lower VECTOR_SHUFFLE into ILVEV (if possible).
2189//
2190// ILVEV interleaves the even elements from each vector.
2191//
2192// It is possible to lower into ILVEV when the mask takes the form:
2193//   <0, n, 2, n+2, 4, n+4, ...>
2194// where n is the number of elements in the vector.
2195//
2196// When undef's appear in the mask they are treated as if they were whatever
2197// value is necessary in order to fit the above form.
2198static SDValue lowerVECTOR_SHUFFLE_ILVEV(SDValue Op, EVT ResTy,
2199                                         SmallVector<int, 16> Indices,
2200                                         SelectionDAG &DAG) {
2201  assert ((Indices.size() % 2) == 0);
2202  int WsIdx = 0;
2203  int WtIdx = ResTy.getVectorNumElements();
2204
2205  for (unsigned i = 0; i < Indices.size(); i += 2) {
2206    if (Indices[i] != -1 && Indices[i] != WsIdx)
2207      return SDValue();
2208    if (Indices[i+1] != -1 && Indices[i+1] != WtIdx)
2209      return SDValue();
2210    WsIdx += 2;
2211    WtIdx += 2;
2212  }
2213
2214  return DAG.getNode(MipsISD::ILVEV, SDLoc(Op), ResTy, Op->getOperand(0),
2215                     Op->getOperand(1));
2216}
2217
2218// Lower VECTOR_SHUFFLE into ILVOD (if possible).
2219//
2220// ILVOD interleaves the odd elements from each vector.
2221//
2222// It is possible to lower into ILVOD when the mask takes the form:
2223//   <1, n+1, 3, n+3, 5, n+5, ...>
2224// where n is the number of elements in the vector.
2225//
2226// When undef's appear in the mask they are treated as if they were whatever
2227// value is necessary in order to fit the above form.
2228static SDValue lowerVECTOR_SHUFFLE_ILVOD(SDValue Op, EVT ResTy,
2229                                         SmallVector<int, 16> Indices,
2230                                         SelectionDAG &DAG) {
2231  assert ((Indices.size() % 2) == 0);
2232  int WsIdx = 1;
2233  int WtIdx = ResTy.getVectorNumElements() + 1;
2234
2235  for (unsigned i = 0; i < Indices.size(); i += 2) {
2236    if (Indices[i] != -1 && Indices[i] != WsIdx)
2237      return SDValue();
2238    if (Indices[i+1] != -1 && Indices[i+1] != WtIdx)
2239      return SDValue();
2240    WsIdx += 2;
2241    WtIdx += 2;
2242  }
2243
2244  return DAG.getNode(MipsISD::ILVOD, SDLoc(Op), ResTy, Op->getOperand(0),
2245                     Op->getOperand(1));
2246}
2247
2248// Lower VECTOR_SHUFFLE into ILVL (if possible).
2249//
2250// ILVL interleaves consecutive elements from the left half of each vector.
2251//
2252// It is possible to lower into ILVL when the mask takes the form:
2253//   <0, n, 1, n+1, 2, n+2, ...>
2254// where n is the number of elements in the vector.
2255//
2256// When undef's appear in the mask they are treated as if they were whatever
2257// value is necessary in order to fit the above form.
2258static SDValue lowerVECTOR_SHUFFLE_ILVL(SDValue Op, EVT ResTy,
2259                                        SmallVector<int, 16> Indices,
2260                                        SelectionDAG &DAG) {
2261  assert ((Indices.size() % 2) == 0);
2262  int WsIdx = 0;
2263  int WtIdx = ResTy.getVectorNumElements();
2264
2265  for (unsigned i = 0; i < Indices.size(); i += 2) {
2266    if (Indices[i] != -1 && Indices[i] != WsIdx)
2267      return SDValue();
2268    if (Indices[i+1] != -1 && Indices[i+1] != WtIdx)
2269      return SDValue();
2270    WsIdx ++;
2271    WtIdx ++;
2272  }
2273
2274  return DAG.getNode(MipsISD::ILVL, SDLoc(Op), ResTy, Op->getOperand(0),
2275                     Op->getOperand(1));
2276}
2277
2278// Lower VECTOR_SHUFFLE into ILVR (if possible).
2279//
2280// ILVR interleaves consecutive elements from the right half of each vector.
2281//
2282// It is possible to lower into ILVR when the mask takes the form:
2283//   <x, n+x, x+1, n+x+1, x+2, n+x+2, ...>
2284// where n is the number of elements in the vector and x is half n.
2285//
2286// When undef's appear in the mask they are treated as if they were whatever
2287// value is necessary in order to fit the above form.
2288static SDValue lowerVECTOR_SHUFFLE_ILVR(SDValue Op, EVT ResTy,
2289                                        SmallVector<int, 16> Indices,
2290                                        SelectionDAG &DAG) {
2291  assert ((Indices.size() % 2) == 0);
2292  unsigned NumElts = ResTy.getVectorNumElements();
2293  int WsIdx = NumElts / 2;
2294  int WtIdx = NumElts + NumElts / 2;
2295
2296  for (unsigned i = 0; i < Indices.size(); i += 2) {
2297    if (Indices[i] != -1 && Indices[i] != WsIdx)
2298      return SDValue();
2299    if (Indices[i+1] != -1 && Indices[i+1] != WtIdx)
2300      return SDValue();
2301    WsIdx ++;
2302    WtIdx ++;
2303  }
2304
2305  return DAG.getNode(MipsISD::ILVR, SDLoc(Op), ResTy, Op->getOperand(0),
2306                     Op->getOperand(1));
2307}
2308
2309// Lower VECTOR_SHUFFLE into PCKEV (if possible).
2310//
2311// PCKEV copies the even elements of each vector into the result vector.
2312//
2313// It is possible to lower into PCKEV when the mask takes the form:
2314//   <0, 2, 4, ..., n, n+2, n+4, ...>
2315// where n is the number of elements in the vector.
2316//
2317// When undef's appear in the mask they are treated as if they were whatever
2318// value is necessary in order to fit the above form.
2319static SDValue lowerVECTOR_SHUFFLE_PCKEV(SDValue Op, EVT ResTy,
2320                                         SmallVector<int, 16> Indices,
2321                                         SelectionDAG &DAG) {
2322  assert ((Indices.size() % 2) == 0);
2323  int Idx = 0;
2324
2325  for (unsigned i = 0; i < Indices.size(); ++i) {
2326    if (Indices[i] != -1 && Indices[i] != Idx)
2327      return SDValue();
2328    Idx += 2;
2329  }
2330
2331  return DAG.getNode(MipsISD::PCKEV, SDLoc(Op), ResTy, Op->getOperand(0),
2332                     Op->getOperand(1));
2333}
2334
2335// Lower VECTOR_SHUFFLE into PCKOD (if possible).
2336//
2337// PCKOD copies the odd elements of each vector into the result vector.
2338//
2339// It is possible to lower into PCKOD when the mask takes the form:
2340//   <1, 3, 5, ..., n+1, n+3, n+5, ...>
2341// where n is the number of elements in the vector.
2342//
2343// When undef's appear in the mask they are treated as if they were whatever
2344// value is necessary in order to fit the above form.
2345static SDValue lowerVECTOR_SHUFFLE_PCKOD(SDValue Op, EVT ResTy,
2346                                         SmallVector<int, 16> Indices,
2347                                         SelectionDAG &DAG) {
2348  assert ((Indices.size() % 2) == 0);
2349  int Idx = 1;
2350
2351  for (unsigned i = 0; i < Indices.size(); ++i) {
2352    if (Indices[i] != -1 && Indices[i] != Idx)
2353      return SDValue();
2354    Idx += 2;
2355  }
2356
2357  return DAG.getNode(MipsISD::PCKOD, SDLoc(Op), ResTy, Op->getOperand(0),
2358                     Op->getOperand(1));
2359}
2360
2361// Lower VECTOR_SHUFFLE into VSHF.
2362//
2363// This mostly consists of converting the shuffle indices in Indices into a
2364// BUILD_VECTOR and adding it as an operand to the resulting VSHF. There is
2365// also code to eliminate unused operands of the VECTOR_SHUFFLE. For example,
2366// if the type is v8i16 and all the indices are less than 8 then the second
2367// operand is unused and can be replaced with anything. We choose to replace it
2368// with the used operand since this reduces the number of instructions overall.
2369static SDValue lowerVECTOR_SHUFFLE_VSHF(SDValue Op, EVT ResTy,
2370                                        SmallVector<int, 16> Indices,
2371                                        SelectionDAG &DAG) {
2372  SmallVector<SDValue, 16> Ops;
2373  SDValue Op0;
2374  SDValue Op1;
2375  EVT MaskVecTy = ResTy.changeVectorElementTypeToInteger();
2376  EVT MaskEltTy = MaskVecTy.getVectorElementType();
2377  bool Using1stVec = false;
2378  bool Using2ndVec = false;
2379  SDLoc DL(Op);
2380  int ResTyNumElts = ResTy.getVectorNumElements();
2381
2382  for (int i = 0; i < ResTyNumElts; ++i) {
2383    // Idx == -1 means UNDEF
2384    int Idx = Indices[i];
2385
2386    if (0 <= Idx && Idx < ResTyNumElts)
2387      Using1stVec = true;
2388    if (ResTyNumElts <= Idx && Idx < ResTyNumElts * 2)
2389      Using2ndVec = true;
2390  }
2391
2392  for (SmallVector<int, 16>::iterator I = Indices.begin(); I != Indices.end();
2393       ++I)
2394    Ops.push_back(DAG.getTargetConstant(*I, MaskEltTy));
2395
2396  SDValue MaskVec = DAG.getNode(ISD::BUILD_VECTOR, DL, MaskVecTy, &Ops[0],
2397                                Ops.size());
2398
2399  if (Using1stVec && Using2ndVec) {
2400    Op0 = Op->getOperand(0);
2401    Op1 = Op->getOperand(1);
2402  } else if (Using1stVec)
2403    Op0 = Op1 = Op->getOperand(0);
2404  else if (Using2ndVec)
2405    Op0 = Op1 = Op->getOperand(1);
2406  else
2407    llvm_unreachable("shuffle vector mask references neither vector operand?");
2408
2409  return DAG.getNode(MipsISD::VSHF, DL, ResTy, MaskVec, Op0, Op1);
2410}
2411
2412// Lower VECTOR_SHUFFLE into one of a number of instructions depending on the
2413// indices in the shuffle.
2414SDValue MipsSETargetLowering::lowerVECTOR_SHUFFLE(SDValue Op,
2415                                                  SelectionDAG &DAG) const {
2416  ShuffleVectorSDNode *Node = cast<ShuffleVectorSDNode>(Op);
2417  EVT ResTy = Op->getValueType(0);
2418
2419  if (!ResTy.is128BitVector())
2420    return SDValue();
2421
2422  int ResTyNumElts = ResTy.getVectorNumElements();
2423  SmallVector<int, 16> Indices;
2424
2425  for (int i = 0; i < ResTyNumElts; ++i)
2426    Indices.push_back(Node->getMaskElt(i));
2427
2428  SDValue Result = lowerVECTOR_SHUFFLE_SHF(Op, ResTy, Indices, DAG);
2429  if (Result.getNode())
2430    return Result;
2431  Result = lowerVECTOR_SHUFFLE_ILVEV(Op, ResTy, Indices, DAG);
2432  if (Result.getNode())
2433    return Result;
2434  Result = lowerVECTOR_SHUFFLE_ILVOD(Op, ResTy, Indices, DAG);
2435  if (Result.getNode())
2436    return Result;
2437  Result = lowerVECTOR_SHUFFLE_ILVL(Op, ResTy, Indices, DAG);
2438  if (Result.getNode())
2439    return Result;
2440  Result = lowerVECTOR_SHUFFLE_ILVR(Op, ResTy, Indices, DAG);
2441  if (Result.getNode())
2442    return Result;
2443  Result = lowerVECTOR_SHUFFLE_PCKEV(Op, ResTy, Indices, DAG);
2444  if (Result.getNode())
2445    return Result;
2446  Result = lowerVECTOR_SHUFFLE_PCKOD(Op, ResTy, Indices, DAG);
2447  if (Result.getNode())
2448    return Result;
2449  return lowerVECTOR_SHUFFLE_VSHF(Op, ResTy, Indices, DAG);
2450}
2451
2452MachineBasicBlock * MipsSETargetLowering::
2453emitBPOSGE32(MachineInstr *MI, MachineBasicBlock *BB) const{
2454  // $bb:
2455  //  bposge32_pseudo $vr0
2456  //  =>
2457  // $bb:
2458  //  bposge32 $tbb
2459  // $fbb:
2460  //  li $vr2, 0
2461  //  b $sink
2462  // $tbb:
2463  //  li $vr1, 1
2464  // $sink:
2465  //  $vr0 = phi($vr2, $fbb, $vr1, $tbb)
2466
2467  MachineRegisterInfo &RegInfo = BB->getParent()->getRegInfo();
2468  const TargetInstrInfo *TII = getTargetMachine().getInstrInfo();
2469  const TargetRegisterClass *RC = &Mips::GPR32RegClass;
2470  DebugLoc DL = MI->getDebugLoc();
2471  const BasicBlock *LLVM_BB = BB->getBasicBlock();
2472  MachineFunction::iterator It = llvm::next(MachineFunction::iterator(BB));
2473  MachineFunction *F = BB->getParent();
2474  MachineBasicBlock *FBB = F->CreateMachineBasicBlock(LLVM_BB);
2475  MachineBasicBlock *TBB = F->CreateMachineBasicBlock(LLVM_BB);
2476  MachineBasicBlock *Sink  = F->CreateMachineBasicBlock(LLVM_BB);
2477  F->insert(It, FBB);
2478  F->insert(It, TBB);
2479  F->insert(It, Sink);
2480
2481  // Transfer the remainder of BB and its successor edges to Sink.
2482  Sink->splice(Sink->begin(), BB, llvm::next(MachineBasicBlock::iterator(MI)),
2483               BB->end());
2484  Sink->transferSuccessorsAndUpdatePHIs(BB);
2485
2486  // Add successors.
2487  BB->addSuccessor(FBB);
2488  BB->addSuccessor(TBB);
2489  FBB->addSuccessor(Sink);
2490  TBB->addSuccessor(Sink);
2491
2492  // Insert the real bposge32 instruction to $BB.
2493  BuildMI(BB, DL, TII->get(Mips::BPOSGE32)).addMBB(TBB);
2494
2495  // Fill $FBB.
2496  unsigned VR2 = RegInfo.createVirtualRegister(RC);
2497  BuildMI(*FBB, FBB->end(), DL, TII->get(Mips::ADDiu), VR2)
2498    .addReg(Mips::ZERO).addImm(0);
2499  BuildMI(*FBB, FBB->end(), DL, TII->get(Mips::B)).addMBB(Sink);
2500
2501  // Fill $TBB.
2502  unsigned VR1 = RegInfo.createVirtualRegister(RC);
2503  BuildMI(*TBB, TBB->end(), DL, TII->get(Mips::ADDiu), VR1)
2504    .addReg(Mips::ZERO).addImm(1);
2505
2506  // Insert phi function to $Sink.
2507  BuildMI(*Sink, Sink->begin(), DL, TII->get(Mips::PHI),
2508          MI->getOperand(0).getReg())
2509    .addReg(VR2).addMBB(FBB).addReg(VR1).addMBB(TBB);
2510
2511  MI->eraseFromParent();   // The pseudo instruction is gone now.
2512  return Sink;
2513}
2514
2515MachineBasicBlock * MipsSETargetLowering::
2516emitMSACBranchPseudo(MachineInstr *MI, MachineBasicBlock *BB,
2517                     unsigned BranchOp) const{
2518  // $bb:
2519  //  vany_nonzero $rd, $ws
2520  //  =>
2521  // $bb:
2522  //  bnz.b $ws, $tbb
2523  //  b $fbb
2524  // $fbb:
2525  //  li $rd1, 0
2526  //  b $sink
2527  // $tbb:
2528  //  li $rd2, 1
2529  // $sink:
2530  //  $rd = phi($rd1, $fbb, $rd2, $tbb)
2531
2532  MachineRegisterInfo &RegInfo = BB->getParent()->getRegInfo();
2533  const TargetInstrInfo *TII = getTargetMachine().getInstrInfo();
2534  const TargetRegisterClass *RC = &Mips::GPR32RegClass;
2535  DebugLoc DL = MI->getDebugLoc();
2536  const BasicBlock *LLVM_BB = BB->getBasicBlock();
2537  MachineFunction::iterator It = llvm::next(MachineFunction::iterator(BB));
2538  MachineFunction *F = BB->getParent();
2539  MachineBasicBlock *FBB = F->CreateMachineBasicBlock(LLVM_BB);
2540  MachineBasicBlock *TBB = F->CreateMachineBasicBlock(LLVM_BB);
2541  MachineBasicBlock *Sink  = F->CreateMachineBasicBlock(LLVM_BB);
2542  F->insert(It, FBB);
2543  F->insert(It, TBB);
2544  F->insert(It, Sink);
2545
2546  // Transfer the remainder of BB and its successor edges to Sink.
2547  Sink->splice(Sink->begin(), BB, llvm::next(MachineBasicBlock::iterator(MI)),
2548               BB->end());
2549  Sink->transferSuccessorsAndUpdatePHIs(BB);
2550
2551  // Add successors.
2552  BB->addSuccessor(FBB);
2553  BB->addSuccessor(TBB);
2554  FBB->addSuccessor(Sink);
2555  TBB->addSuccessor(Sink);
2556
2557  // Insert the real bnz.b instruction to $BB.
2558  BuildMI(BB, DL, TII->get(BranchOp))
2559    .addReg(MI->getOperand(1).getReg())
2560    .addMBB(TBB);
2561
2562  // Fill $FBB.
2563  unsigned RD1 = RegInfo.createVirtualRegister(RC);
2564  BuildMI(*FBB, FBB->end(), DL, TII->get(Mips::ADDiu), RD1)
2565    .addReg(Mips::ZERO).addImm(0);
2566  BuildMI(*FBB, FBB->end(), DL, TII->get(Mips::B)).addMBB(Sink);
2567
2568  // Fill $TBB.
2569  unsigned RD2 = RegInfo.createVirtualRegister(RC);
2570  BuildMI(*TBB, TBB->end(), DL, TII->get(Mips::ADDiu), RD2)
2571    .addReg(Mips::ZERO).addImm(1);
2572
2573  // Insert phi function to $Sink.
2574  BuildMI(*Sink, Sink->begin(), DL, TII->get(Mips::PHI),
2575          MI->getOperand(0).getReg())
2576    .addReg(RD1).addMBB(FBB).addReg(RD2).addMBB(TBB);
2577
2578  MI->eraseFromParent();   // The pseudo instruction is gone now.
2579  return Sink;
2580}
2581
2582// Emit the COPY_FW pseudo instruction.
2583//
2584// copy_fw_pseudo $fd, $ws, n
2585// =>
2586// copy_u_w $rt, $ws, $n
2587// mtc1     $rt, $fd
2588//
2589// When n is zero, the equivalent operation can be performed with (potentially)
2590// zero instructions due to register overlaps. This optimization is never valid
2591// for lane 1 because it would require FR=0 mode which isn't supported by MSA.
2592MachineBasicBlock * MipsSETargetLowering::
2593emitCOPY_FW(MachineInstr *MI, MachineBasicBlock *BB) const{
2594  const TargetInstrInfo *TII = getTargetMachine().getInstrInfo();
2595  MachineRegisterInfo &RegInfo = BB->getParent()->getRegInfo();
2596  DebugLoc DL = MI->getDebugLoc();
2597  unsigned Fd = MI->getOperand(0).getReg();
2598  unsigned Ws = MI->getOperand(1).getReg();
2599  unsigned Lane = MI->getOperand(2).getImm();
2600
2601  if (Lane == 0)
2602    BuildMI(*BB, MI, DL, TII->get(Mips::COPY), Fd).addReg(Ws, 0, Mips::sub_lo);
2603  else {
2604    unsigned Wt = RegInfo.createVirtualRegister(&Mips::MSA128WRegClass);
2605
2606    BuildMI(*BB, MI, DL, TII->get(Mips::SPLATI_W), Wt).addReg(Ws).addImm(1);
2607    BuildMI(*BB, MI, DL, TII->get(Mips::COPY), Fd).addReg(Wt, 0, Mips::sub_lo);
2608  }
2609
2610  MI->eraseFromParent();   // The pseudo instruction is gone now.
2611  return BB;
2612}
2613
2614// Emit the COPY_FD pseudo instruction.
2615//
2616// copy_fd_pseudo $fd, $ws, n
2617// =>
2618// splati.d $wt, $ws, $n
2619// copy $fd, $wt:sub_64
2620//
2621// When n is zero, the equivalent operation can be performed with (potentially)
2622// zero instructions due to register overlaps. This optimization is always
2623// valid because FR=1 mode which is the only supported mode in MSA.
2624MachineBasicBlock * MipsSETargetLowering::
2625emitCOPY_FD(MachineInstr *MI, MachineBasicBlock *BB) const{
2626  assert(Subtarget->isFP64bit());
2627
2628  const TargetInstrInfo *TII = getTargetMachine().getInstrInfo();
2629  MachineRegisterInfo &RegInfo = BB->getParent()->getRegInfo();
2630  unsigned Fd  = MI->getOperand(0).getReg();
2631  unsigned Ws  = MI->getOperand(1).getReg();
2632  unsigned Lane = MI->getOperand(2).getImm() * 2;
2633  DebugLoc DL = MI->getDebugLoc();
2634
2635  if (Lane == 0)
2636    BuildMI(*BB, MI, DL, TII->get(Mips::COPY), Fd).addReg(Ws, 0, Mips::sub_64);
2637  else {
2638    unsigned Wt = RegInfo.createVirtualRegister(&Mips::MSA128DRegClass);
2639
2640    BuildMI(*BB, MI, DL, TII->get(Mips::SPLATI_D), Wt).addReg(Ws).addImm(1);
2641    BuildMI(*BB, MI, DL, TII->get(Mips::COPY), Fd).addReg(Wt, 0, Mips::sub_64);
2642  }
2643
2644  MI->eraseFromParent();   // The pseudo instruction is gone now.
2645  return BB;
2646}
2647
2648// Emit the INSERT_FW pseudo instruction.
2649//
2650// insert_fw_pseudo $wd, $wd_in, $n, $fs
2651// =>
2652// subreg_to_reg $wt:sub_lo, $fs
2653// insve_w $wd[$n], $wd_in, $wt[0]
2654MachineBasicBlock *
2655MipsSETargetLowering::emitINSERT_FW(MachineInstr *MI,
2656                                    MachineBasicBlock *BB) const {
2657  const TargetInstrInfo *TII = getTargetMachine().getInstrInfo();
2658  MachineRegisterInfo &RegInfo = BB->getParent()->getRegInfo();
2659  DebugLoc DL = MI->getDebugLoc();
2660  unsigned Wd = MI->getOperand(0).getReg();
2661  unsigned Wd_in = MI->getOperand(1).getReg();
2662  unsigned Lane = MI->getOperand(2).getImm();
2663  unsigned Fs = MI->getOperand(3).getReg();
2664  unsigned Wt = RegInfo.createVirtualRegister(&Mips::MSA128WRegClass);
2665
2666  BuildMI(*BB, MI, DL, TII->get(Mips::SUBREG_TO_REG), Wt)
2667      .addImm(0)
2668      .addReg(Fs)
2669      .addImm(Mips::sub_lo);
2670  BuildMI(*BB, MI, DL, TII->get(Mips::INSVE_W), Wd)
2671      .addReg(Wd_in)
2672      .addImm(Lane)
2673      .addReg(Wt);
2674
2675  MI->eraseFromParent(); // The pseudo instruction is gone now.
2676  return BB;
2677}
2678
2679// Emit the INSERT_FD pseudo instruction.
2680//
2681// insert_fd_pseudo $wd, $fs, n
2682// =>
2683// subreg_to_reg $wt:sub_64, $fs
2684// insve_d $wd[$n], $wd_in, $wt[0]
2685MachineBasicBlock *
2686MipsSETargetLowering::emitINSERT_FD(MachineInstr *MI,
2687                                    MachineBasicBlock *BB) const {
2688  assert(Subtarget->isFP64bit());
2689
2690  const TargetInstrInfo *TII = getTargetMachine().getInstrInfo();
2691  MachineRegisterInfo &RegInfo = BB->getParent()->getRegInfo();
2692  DebugLoc DL = MI->getDebugLoc();
2693  unsigned Wd = MI->getOperand(0).getReg();
2694  unsigned Wd_in = MI->getOperand(1).getReg();
2695  unsigned Lane = MI->getOperand(2).getImm();
2696  unsigned Fs = MI->getOperand(3).getReg();
2697  unsigned Wt = RegInfo.createVirtualRegister(&Mips::MSA128DRegClass);
2698
2699  BuildMI(*BB, MI, DL, TII->get(Mips::SUBREG_TO_REG), Wt)
2700      .addImm(0)
2701      .addReg(Fs)
2702      .addImm(Mips::sub_64);
2703  BuildMI(*BB, MI, DL, TII->get(Mips::INSVE_D), Wd)
2704      .addReg(Wd_in)
2705      .addImm(Lane)
2706      .addReg(Wt);
2707
2708  MI->eraseFromParent(); // The pseudo instruction is gone now.
2709  return BB;
2710}
2711
2712// Emit the FILL_FW pseudo instruction.
2713//
2714// fill_fw_pseudo $wd, $fs
2715// =>
2716// implicit_def $wt1
2717// insert_subreg $wt2:subreg_lo, $wt1, $fs
2718// splati.w $wd, $wt2[0]
2719MachineBasicBlock *
2720MipsSETargetLowering::emitFILL_FW(MachineInstr *MI,
2721                                  MachineBasicBlock *BB) const {
2722  const TargetInstrInfo *TII = getTargetMachine().getInstrInfo();
2723  MachineRegisterInfo &RegInfo = BB->getParent()->getRegInfo();
2724  DebugLoc DL = MI->getDebugLoc();
2725  unsigned Wd = MI->getOperand(0).getReg();
2726  unsigned Fs = MI->getOperand(1).getReg();
2727  unsigned Wt1 = RegInfo.createVirtualRegister(&Mips::MSA128WRegClass);
2728  unsigned Wt2 = RegInfo.createVirtualRegister(&Mips::MSA128WRegClass);
2729
2730  BuildMI(*BB, MI, DL, TII->get(Mips::IMPLICIT_DEF), Wt1);
2731  BuildMI(*BB, MI, DL, TII->get(Mips::INSERT_SUBREG), Wt2)
2732      .addReg(Wt1)
2733      .addReg(Fs)
2734      .addImm(Mips::sub_lo);
2735  BuildMI(*BB, MI, DL, TII->get(Mips::SPLATI_W), Wd).addReg(Wt2).addImm(0);
2736
2737  MI->eraseFromParent(); // The pseudo instruction is gone now.
2738  return BB;
2739}
2740
2741// Emit the FILL_FD pseudo instruction.
2742//
2743// fill_fd_pseudo $wd, $fs
2744// =>
2745// implicit_def $wt1
2746// insert_subreg $wt2:subreg_64, $wt1, $fs
2747// splati.d $wd, $wt2[0]
2748MachineBasicBlock *
2749MipsSETargetLowering::emitFILL_FD(MachineInstr *MI,
2750                                  MachineBasicBlock *BB) const {
2751  assert(Subtarget->isFP64bit());
2752
2753  const TargetInstrInfo *TII = getTargetMachine().getInstrInfo();
2754  MachineRegisterInfo &RegInfo = BB->getParent()->getRegInfo();
2755  DebugLoc DL = MI->getDebugLoc();
2756  unsigned Wd = MI->getOperand(0).getReg();
2757  unsigned Fs = MI->getOperand(1).getReg();
2758  unsigned Wt1 = RegInfo.createVirtualRegister(&Mips::MSA128DRegClass);
2759  unsigned Wt2 = RegInfo.createVirtualRegister(&Mips::MSA128DRegClass);
2760
2761  BuildMI(*BB, MI, DL, TII->get(Mips::IMPLICIT_DEF), Wt1);
2762  BuildMI(*BB, MI, DL, TII->get(Mips::INSERT_SUBREG), Wt2)
2763      .addReg(Wt1)
2764      .addReg(Fs)
2765      .addImm(Mips::sub_64);
2766  BuildMI(*BB, MI, DL, TII->get(Mips::SPLATI_D), Wd).addReg(Wt2).addImm(0);
2767
2768  MI->eraseFromParent();   // The pseudo instruction is gone now.
2769  return BB;
2770}
2771
2772// Emit the FEXP2_W_1 pseudo instructions.
2773//
2774// fexp2_w_1_pseudo $wd, $wt
2775// =>
2776// ldi.w $ws, 1
2777// fexp2.w $wd, $ws, $wt
2778MachineBasicBlock *
2779MipsSETargetLowering::emitFEXP2_W_1(MachineInstr *MI,
2780                                    MachineBasicBlock *BB) const {
2781  const TargetInstrInfo *TII = getTargetMachine().getInstrInfo();
2782  MachineRegisterInfo &RegInfo = BB->getParent()->getRegInfo();
2783  const TargetRegisterClass *RC = &Mips::MSA128WRegClass;
2784  unsigned Ws1 = RegInfo.createVirtualRegister(RC);
2785  unsigned Ws2 = RegInfo.createVirtualRegister(RC);
2786  DebugLoc DL = MI->getDebugLoc();
2787
2788  // Splat 1.0 into a vector
2789  BuildMI(*BB, MI, DL, TII->get(Mips::LDI_W), Ws1).addImm(1);
2790  BuildMI(*BB, MI, DL, TII->get(Mips::FFINT_U_W), Ws2).addReg(Ws1);
2791
2792  // Emit 1.0 * fexp2(Wt)
2793  BuildMI(*BB, MI, DL, TII->get(Mips::FEXP2_W), MI->getOperand(0).getReg())
2794      .addReg(Ws2)
2795      .addReg(MI->getOperand(1).getReg());
2796
2797  MI->eraseFromParent(); // The pseudo instruction is gone now.
2798  return BB;
2799}
2800
2801// Emit the FEXP2_D_1 pseudo instructions.
2802//
2803// fexp2_d_1_pseudo $wd, $wt
2804// =>
2805// ldi.d $ws, 1
2806// fexp2.d $wd, $ws, $wt
2807MachineBasicBlock *
2808MipsSETargetLowering::emitFEXP2_D_1(MachineInstr *MI,
2809                                    MachineBasicBlock *BB) const {
2810  const TargetInstrInfo *TII = getTargetMachine().getInstrInfo();
2811  MachineRegisterInfo &RegInfo = BB->getParent()->getRegInfo();
2812  const TargetRegisterClass *RC = &Mips::MSA128DRegClass;
2813  unsigned Ws1 = RegInfo.createVirtualRegister(RC);
2814  unsigned Ws2 = RegInfo.createVirtualRegister(RC);
2815  DebugLoc DL = MI->getDebugLoc();
2816
2817  // Splat 1.0 into a vector
2818  BuildMI(*BB, MI, DL, TII->get(Mips::LDI_D), Ws1).addImm(1);
2819  BuildMI(*BB, MI, DL, TII->get(Mips::FFINT_U_D), Ws2).addReg(Ws1);
2820
2821  // Emit 1.0 * fexp2(Wt)
2822  BuildMI(*BB, MI, DL, TII->get(Mips::FEXP2_D), MI->getOperand(0).getReg())
2823      .addReg(Ws2)
2824      .addReg(MI->getOperand(1).getReg());
2825
2826  MI->eraseFromParent(); // The pseudo instruction is gone now.
2827  return BB;
2828}
2829