MipsSEISelLowering.cpp revision 7e0df9aa2966d0462e34511524a4958e226b74ee
1//===-- MipsSEISelLowering.cpp - MipsSE DAG Lowering Interface --*- C++ -*-===// 2// 3// The LLVM Compiler Infrastructure 4// 5// This file is distributed under the University of Illinois Open Source 6// License. See LICENSE.TXT for details. 7// 8//===----------------------------------------------------------------------===// 9// 10// Subclass of MipsTargetLowering specialized for mips32/64. 11// 12//===----------------------------------------------------------------------===// 13#include "MipsSEISelLowering.h" 14#include "MipsRegisterInfo.h" 15#include "MipsTargetMachine.h" 16#include "llvm/CodeGen/MachineInstrBuilder.h" 17#include "llvm/CodeGen/MachineRegisterInfo.h" 18#include "llvm/IR/Intrinsics.h" 19#include "llvm/Support/CommandLine.h" 20#include "llvm/Target/TargetInstrInfo.h" 21 22using namespace llvm; 23 24static cl::opt<bool> 25EnableMipsTailCalls("enable-mips-tail-calls", cl::Hidden, 26 cl::desc("MIPS: Enable tail calls."), cl::init(false)); 27 28static cl::opt<bool> NoDPLoadStore("mno-ldc1-sdc1", cl::init(false), 29 cl::desc("Expand double precision loads and " 30 "stores to their single precision " 31 "counterparts")); 32 33MipsSETargetLowering::MipsSETargetLowering(MipsTargetMachine &TM) 34 : MipsTargetLowering(TM) { 35 // Set up the register classes 36 37 clearRegisterClasses(); 38 39 addRegisterClass(MVT::i32, &Mips::GPR32RegClass); 40 41 if (HasMips64) 42 addRegisterClass(MVT::i64, &Mips::GPR64RegClass); 43 44 if (Subtarget->hasDSP()) { 45 MVT::SimpleValueType VecTys[2] = {MVT::v2i16, MVT::v4i8}; 46 47 for (unsigned i = 0; i < array_lengthof(VecTys); ++i) { 48 addRegisterClass(VecTys[i], &Mips::DSPRRegClass); 49 50 // Expand all builtin opcodes. 51 for (unsigned Opc = 0; Opc < ISD::BUILTIN_OP_END; ++Opc) 52 setOperationAction(Opc, VecTys[i], Expand); 53 54 setOperationAction(ISD::ADD, VecTys[i], Legal); 55 setOperationAction(ISD::SUB, VecTys[i], Legal); 56 setOperationAction(ISD::LOAD, VecTys[i], Legal); 57 setOperationAction(ISD::STORE, VecTys[i], Legal); 58 setOperationAction(ISD::BITCAST, VecTys[i], Legal); 59 } 60 61 // Expand all truncating stores and extending loads. 62 unsigned FirstVT = (unsigned)MVT::FIRST_VECTOR_VALUETYPE; 63 unsigned LastVT = (unsigned)MVT::LAST_VECTOR_VALUETYPE; 64 65 for (unsigned VT0 = FirstVT; VT0 <= LastVT; ++VT0) { 66 for (unsigned VT1 = FirstVT; VT1 <= LastVT; ++VT1) 67 setTruncStoreAction((MVT::SimpleValueType)VT0, 68 (MVT::SimpleValueType)VT1, Expand); 69 70 setLoadExtAction(ISD::SEXTLOAD, (MVT::SimpleValueType)VT0, Expand); 71 setLoadExtAction(ISD::ZEXTLOAD, (MVT::SimpleValueType)VT0, Expand); 72 setLoadExtAction(ISD::EXTLOAD, (MVT::SimpleValueType)VT0, Expand); 73 } 74 75 setTargetDAGCombine(ISD::SHL); 76 setTargetDAGCombine(ISD::SRA); 77 setTargetDAGCombine(ISD::SRL); 78 setTargetDAGCombine(ISD::SETCC); 79 setTargetDAGCombine(ISD::VSELECT); 80 } 81 82 if (Subtarget->hasDSPR2()) 83 setOperationAction(ISD::MUL, MVT::v2i16, Legal); 84 85 if (Subtarget->hasMSA()) { 86 addMSAIntType(MVT::v16i8, &Mips::MSA128BRegClass); 87 addMSAIntType(MVT::v8i16, &Mips::MSA128HRegClass); 88 addMSAIntType(MVT::v4i32, &Mips::MSA128WRegClass); 89 addMSAIntType(MVT::v2i64, &Mips::MSA128DRegClass); 90 addMSAFloatType(MVT::v8f16, &Mips::MSA128HRegClass); 91 addMSAFloatType(MVT::v4f32, &Mips::MSA128WRegClass); 92 addMSAFloatType(MVT::v2f64, &Mips::MSA128DRegClass); 93 94 setTargetDAGCombine(ISD::AND); 95 setTargetDAGCombine(ISD::SRA); 96 setTargetDAGCombine(ISD::VSELECT); 97 setTargetDAGCombine(ISD::XOR); 98 } 99 100 if (!Subtarget->mipsSEUsesSoftFloat()) { 101 addRegisterClass(MVT::f32, &Mips::FGR32RegClass); 102 103 // When dealing with single precision only, use libcalls 104 if (!Subtarget->isSingleFloat()) { 105 if (Subtarget->isFP64bit()) 106 addRegisterClass(MVT::f64, &Mips::FGR64RegClass); 107 else 108 addRegisterClass(MVT::f64, &Mips::AFGR64RegClass); 109 } 110 } 111 112 setOperationAction(ISD::SMUL_LOHI, MVT::i32, Custom); 113 setOperationAction(ISD::UMUL_LOHI, MVT::i32, Custom); 114 setOperationAction(ISD::MULHS, MVT::i32, Custom); 115 setOperationAction(ISD::MULHU, MVT::i32, Custom); 116 117 if (HasMips64) { 118 setOperationAction(ISD::MULHS, MVT::i64, Custom); 119 setOperationAction(ISD::MULHU, MVT::i64, Custom); 120 setOperationAction(ISD::MUL, MVT::i64, Custom); 121 } 122 123 setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::i64, Custom); 124 setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::i64, Custom); 125 126 setOperationAction(ISD::SDIVREM, MVT::i32, Custom); 127 setOperationAction(ISD::UDIVREM, MVT::i32, Custom); 128 setOperationAction(ISD::SDIVREM, MVT::i64, Custom); 129 setOperationAction(ISD::UDIVREM, MVT::i64, Custom); 130 setOperationAction(ISD::ATOMIC_FENCE, MVT::Other, Custom); 131 setOperationAction(ISD::LOAD, MVT::i32, Custom); 132 setOperationAction(ISD::STORE, MVT::i32, Custom); 133 134 setTargetDAGCombine(ISD::ADDE); 135 setTargetDAGCombine(ISD::SUBE); 136 setTargetDAGCombine(ISD::MUL); 137 138 setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::Other, Custom); 139 setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::Other, Custom); 140 setOperationAction(ISD::INTRINSIC_VOID, MVT::Other, Custom); 141 142 if (NoDPLoadStore) { 143 setOperationAction(ISD::LOAD, MVT::f64, Custom); 144 setOperationAction(ISD::STORE, MVT::f64, Custom); 145 } 146 147 computeRegisterProperties(); 148} 149 150const MipsTargetLowering * 151llvm::createMipsSETargetLowering(MipsTargetMachine &TM) { 152 return new MipsSETargetLowering(TM); 153} 154 155// Enable MSA support for the given integer type and Register class. 156void MipsSETargetLowering:: 157addMSAIntType(MVT::SimpleValueType Ty, const TargetRegisterClass *RC) { 158 addRegisterClass(Ty, RC); 159 160 // Expand all builtin opcodes. 161 for (unsigned Opc = 0; Opc < ISD::BUILTIN_OP_END; ++Opc) 162 setOperationAction(Opc, Ty, Expand); 163 164 setOperationAction(ISD::BITCAST, Ty, Legal); 165 setOperationAction(ISD::LOAD, Ty, Legal); 166 setOperationAction(ISD::STORE, Ty, Legal); 167 setOperationAction(ISD::EXTRACT_VECTOR_ELT, Ty, Custom); 168 setOperationAction(ISD::INSERT_VECTOR_ELT, Ty, Legal); 169 setOperationAction(ISD::BUILD_VECTOR, Ty, Custom); 170 171 setOperationAction(ISD::ADD, Ty, Legal); 172 setOperationAction(ISD::AND, Ty, Legal); 173 setOperationAction(ISD::CTLZ, Ty, Legal); 174 setOperationAction(ISD::CTPOP, Ty, Legal); 175 setOperationAction(ISD::MUL, Ty, Legal); 176 setOperationAction(ISD::OR, Ty, Legal); 177 setOperationAction(ISD::SDIV, Ty, Legal); 178 setOperationAction(ISD::SHL, Ty, Legal); 179 setOperationAction(ISD::SRA, Ty, Legal); 180 setOperationAction(ISD::SRL, Ty, Legal); 181 setOperationAction(ISD::SUB, Ty, Legal); 182 setOperationAction(ISD::UDIV, Ty, Legal); 183 setOperationAction(ISD::VECTOR_SHUFFLE, Ty, Custom); 184 setOperationAction(ISD::VSELECT, Ty, Legal); 185 setOperationAction(ISD::XOR, Ty, Legal); 186 187 setOperationAction(ISD::SETCC, Ty, Legal); 188 setCondCodeAction(ISD::SETNE, Ty, Expand); 189 setCondCodeAction(ISD::SETGE, Ty, Expand); 190 setCondCodeAction(ISD::SETGT, Ty, Expand); 191 setCondCodeAction(ISD::SETUGE, Ty, Expand); 192 setCondCodeAction(ISD::SETUGT, Ty, Expand); 193} 194 195// Enable MSA support for the given floating-point type and Register class. 196void MipsSETargetLowering:: 197addMSAFloatType(MVT::SimpleValueType Ty, const TargetRegisterClass *RC) { 198 addRegisterClass(Ty, RC); 199 200 // Expand all builtin opcodes. 201 for (unsigned Opc = 0; Opc < ISD::BUILTIN_OP_END; ++Opc) 202 setOperationAction(Opc, Ty, Expand); 203 204 setOperationAction(ISD::LOAD, Ty, Legal); 205 setOperationAction(ISD::STORE, Ty, Legal); 206 setOperationAction(ISD::BITCAST, Ty, Legal); 207 setOperationAction(ISD::EXTRACT_VECTOR_ELT, Ty, Legal); 208 209 if (Ty != MVT::v8f16) { 210 setOperationAction(ISD::FABS, Ty, Legal); 211 setOperationAction(ISD::FADD, Ty, Legal); 212 setOperationAction(ISD::FDIV, Ty, Legal); 213 setOperationAction(ISD::FLOG2, Ty, Legal); 214 setOperationAction(ISD::FMUL, Ty, Legal); 215 setOperationAction(ISD::FRINT, Ty, Legal); 216 setOperationAction(ISD::FSQRT, Ty, Legal); 217 setOperationAction(ISD::FSUB, Ty, Legal); 218 setOperationAction(ISD::VSELECT, Ty, Legal); 219 220 setOperationAction(ISD::SETCC, Ty, Legal); 221 setCondCodeAction(ISD::SETOGE, Ty, Expand); 222 setCondCodeAction(ISD::SETOGT, Ty, Expand); 223 setCondCodeAction(ISD::SETUGE, Ty, Expand); 224 setCondCodeAction(ISD::SETUGT, Ty, Expand); 225 setCondCodeAction(ISD::SETGE, Ty, Expand); 226 setCondCodeAction(ISD::SETGT, Ty, Expand); 227 } 228} 229 230bool 231MipsSETargetLowering::allowsUnalignedMemoryAccesses(EVT VT, bool *Fast) const { 232 MVT::SimpleValueType SVT = VT.getSimpleVT().SimpleTy; 233 234 switch (SVT) { 235 case MVT::i64: 236 case MVT::i32: 237 if (Fast) 238 *Fast = true; 239 return true; 240 default: 241 return false; 242 } 243} 244 245SDValue MipsSETargetLowering::LowerOperation(SDValue Op, 246 SelectionDAG &DAG) const { 247 switch(Op.getOpcode()) { 248 case ISD::LOAD: return lowerLOAD(Op, DAG); 249 case ISD::STORE: return lowerSTORE(Op, DAG); 250 case ISD::SMUL_LOHI: return lowerMulDiv(Op, MipsISD::Mult, true, true, DAG); 251 case ISD::UMUL_LOHI: return lowerMulDiv(Op, MipsISD::Multu, true, true, DAG); 252 case ISD::MULHS: return lowerMulDiv(Op, MipsISD::Mult, false, true, DAG); 253 case ISD::MULHU: return lowerMulDiv(Op, MipsISD::Multu, false, true, DAG); 254 case ISD::MUL: return lowerMulDiv(Op, MipsISD::Mult, true, false, DAG); 255 case ISD::SDIVREM: return lowerMulDiv(Op, MipsISD::DivRem, true, true, DAG); 256 case ISD::UDIVREM: return lowerMulDiv(Op, MipsISD::DivRemU, true, true, 257 DAG); 258 case ISD::INTRINSIC_WO_CHAIN: return lowerINTRINSIC_WO_CHAIN(Op, DAG); 259 case ISD::INTRINSIC_W_CHAIN: return lowerINTRINSIC_W_CHAIN(Op, DAG); 260 case ISD::INTRINSIC_VOID: return lowerINTRINSIC_VOID(Op, DAG); 261 case ISD::EXTRACT_VECTOR_ELT: return lowerEXTRACT_VECTOR_ELT(Op, DAG); 262 case ISD::BUILD_VECTOR: return lowerBUILD_VECTOR(Op, DAG); 263 case ISD::VECTOR_SHUFFLE: return lowerVECTOR_SHUFFLE(Op, DAG); 264 } 265 266 return MipsTargetLowering::LowerOperation(Op, DAG); 267} 268 269// selectMADD - 270// Transforms a subgraph in CurDAG if the following pattern is found: 271// (addc multLo, Lo0), (adde multHi, Hi0), 272// where, 273// multHi/Lo: product of multiplication 274// Lo0: initial value of Lo register 275// Hi0: initial value of Hi register 276// Return true if pattern matching was successful. 277static bool selectMADD(SDNode *ADDENode, SelectionDAG *CurDAG) { 278 // ADDENode's second operand must be a flag output of an ADDC node in order 279 // for the matching to be successful. 280 SDNode *ADDCNode = ADDENode->getOperand(2).getNode(); 281 282 if (ADDCNode->getOpcode() != ISD::ADDC) 283 return false; 284 285 SDValue MultHi = ADDENode->getOperand(0); 286 SDValue MultLo = ADDCNode->getOperand(0); 287 SDNode *MultNode = MultHi.getNode(); 288 unsigned MultOpc = MultHi.getOpcode(); 289 290 // MultHi and MultLo must be generated by the same node, 291 if (MultLo.getNode() != MultNode) 292 return false; 293 294 // and it must be a multiplication. 295 if (MultOpc != ISD::SMUL_LOHI && MultOpc != ISD::UMUL_LOHI) 296 return false; 297 298 // MultLo amd MultHi must be the first and second output of MultNode 299 // respectively. 300 if (MultHi.getResNo() != 1 || MultLo.getResNo() != 0) 301 return false; 302 303 // Transform this to a MADD only if ADDENode and ADDCNode are the only users 304 // of the values of MultNode, in which case MultNode will be removed in later 305 // phases. 306 // If there exist users other than ADDENode or ADDCNode, this function returns 307 // here, which will result in MultNode being mapped to a single MULT 308 // instruction node rather than a pair of MULT and MADD instructions being 309 // produced. 310 if (!MultHi.hasOneUse() || !MultLo.hasOneUse()) 311 return false; 312 313 SDLoc DL(ADDENode); 314 315 // Initialize accumulator. 316 SDValue ACCIn = CurDAG->getNode(MipsISD::InsertLOHI, DL, MVT::Untyped, 317 ADDCNode->getOperand(1), 318 ADDENode->getOperand(1)); 319 320 // create MipsMAdd(u) node 321 MultOpc = MultOpc == ISD::UMUL_LOHI ? MipsISD::MAddu : MipsISD::MAdd; 322 323 SDValue MAdd = CurDAG->getNode(MultOpc, DL, MVT::Untyped, 324 MultNode->getOperand(0),// Factor 0 325 MultNode->getOperand(1),// Factor 1 326 ACCIn); 327 328 // replace uses of adde and addc here 329 if (!SDValue(ADDCNode, 0).use_empty()) { 330 SDValue LoIdx = CurDAG->getConstant(Mips::sub_lo, MVT::i32); 331 SDValue LoOut = CurDAG->getNode(MipsISD::ExtractLOHI, DL, MVT::i32, MAdd, 332 LoIdx); 333 CurDAG->ReplaceAllUsesOfValueWith(SDValue(ADDCNode, 0), LoOut); 334 } 335 if (!SDValue(ADDENode, 0).use_empty()) { 336 SDValue HiIdx = CurDAG->getConstant(Mips::sub_hi, MVT::i32); 337 SDValue HiOut = CurDAG->getNode(MipsISD::ExtractLOHI, DL, MVT::i32, MAdd, 338 HiIdx); 339 CurDAG->ReplaceAllUsesOfValueWith(SDValue(ADDENode, 0), HiOut); 340 } 341 342 return true; 343} 344 345// selectMSUB - 346// Transforms a subgraph in CurDAG if the following pattern is found: 347// (addc Lo0, multLo), (sube Hi0, multHi), 348// where, 349// multHi/Lo: product of multiplication 350// Lo0: initial value of Lo register 351// Hi0: initial value of Hi register 352// Return true if pattern matching was successful. 353static bool selectMSUB(SDNode *SUBENode, SelectionDAG *CurDAG) { 354 // SUBENode's second operand must be a flag output of an SUBC node in order 355 // for the matching to be successful. 356 SDNode *SUBCNode = SUBENode->getOperand(2).getNode(); 357 358 if (SUBCNode->getOpcode() != ISD::SUBC) 359 return false; 360 361 SDValue MultHi = SUBENode->getOperand(1); 362 SDValue MultLo = SUBCNode->getOperand(1); 363 SDNode *MultNode = MultHi.getNode(); 364 unsigned MultOpc = MultHi.getOpcode(); 365 366 // MultHi and MultLo must be generated by the same node, 367 if (MultLo.getNode() != MultNode) 368 return false; 369 370 // and it must be a multiplication. 371 if (MultOpc != ISD::SMUL_LOHI && MultOpc != ISD::UMUL_LOHI) 372 return false; 373 374 // MultLo amd MultHi must be the first and second output of MultNode 375 // respectively. 376 if (MultHi.getResNo() != 1 || MultLo.getResNo() != 0) 377 return false; 378 379 // Transform this to a MSUB only if SUBENode and SUBCNode are the only users 380 // of the values of MultNode, in which case MultNode will be removed in later 381 // phases. 382 // If there exist users other than SUBENode or SUBCNode, this function returns 383 // here, which will result in MultNode being mapped to a single MULT 384 // instruction node rather than a pair of MULT and MSUB instructions being 385 // produced. 386 if (!MultHi.hasOneUse() || !MultLo.hasOneUse()) 387 return false; 388 389 SDLoc DL(SUBENode); 390 391 // Initialize accumulator. 392 SDValue ACCIn = CurDAG->getNode(MipsISD::InsertLOHI, DL, MVT::Untyped, 393 SUBCNode->getOperand(0), 394 SUBENode->getOperand(0)); 395 396 // create MipsSub(u) node 397 MultOpc = MultOpc == ISD::UMUL_LOHI ? MipsISD::MSubu : MipsISD::MSub; 398 399 SDValue MSub = CurDAG->getNode(MultOpc, DL, MVT::Glue, 400 MultNode->getOperand(0),// Factor 0 401 MultNode->getOperand(1),// Factor 1 402 ACCIn); 403 404 // replace uses of sube and subc here 405 if (!SDValue(SUBCNode, 0).use_empty()) { 406 SDValue LoIdx = CurDAG->getConstant(Mips::sub_lo, MVT::i32); 407 SDValue LoOut = CurDAG->getNode(MipsISD::ExtractLOHI, DL, MVT::i32, MSub, 408 LoIdx); 409 CurDAG->ReplaceAllUsesOfValueWith(SDValue(SUBCNode, 0), LoOut); 410 } 411 if (!SDValue(SUBENode, 0).use_empty()) { 412 SDValue HiIdx = CurDAG->getConstant(Mips::sub_hi, MVT::i32); 413 SDValue HiOut = CurDAG->getNode(MipsISD::ExtractLOHI, DL, MVT::i32, MSub, 414 HiIdx); 415 CurDAG->ReplaceAllUsesOfValueWith(SDValue(SUBENode, 0), HiOut); 416 } 417 418 return true; 419} 420 421static SDValue performADDECombine(SDNode *N, SelectionDAG &DAG, 422 TargetLowering::DAGCombinerInfo &DCI, 423 const MipsSubtarget *Subtarget) { 424 if (DCI.isBeforeLegalize()) 425 return SDValue(); 426 427 if (Subtarget->hasMips32() && N->getValueType(0) == MVT::i32 && 428 selectMADD(N, &DAG)) 429 return SDValue(N, 0); 430 431 return SDValue(); 432} 433 434// Fold zero extensions into MipsISD::VEXTRACT_[SZ]EXT_ELT 435// 436// Performs the following transformations: 437// - Changes MipsISD::VEXTRACT_[SZ]EXT_ELT to zero extension if its 438// sign/zero-extension is completely overwritten by the new one performed by 439// the ISD::AND. 440// - Removes redundant zero extensions performed by an ISD::AND. 441static SDValue performANDCombine(SDNode *N, SelectionDAG &DAG, 442 TargetLowering::DAGCombinerInfo &DCI, 443 const MipsSubtarget *Subtarget) { 444 if (!Subtarget->hasMSA()) 445 return SDValue(); 446 447 SDValue Op0 = N->getOperand(0); 448 SDValue Op1 = N->getOperand(1); 449 unsigned Op0Opcode = Op0->getOpcode(); 450 451 // (and (MipsVExtract[SZ]Ext $a, $b, $c), imm:$d) 452 // where $d + 1 == 2^n and n == 32 453 // or $d + 1 == 2^n and n <= 32 and ZExt 454 // -> (MipsVExtractZExt $a, $b, $c) 455 if (Op0Opcode == MipsISD::VEXTRACT_SEXT_ELT || 456 Op0Opcode == MipsISD::VEXTRACT_ZEXT_ELT) { 457 ConstantSDNode *Mask = dyn_cast<ConstantSDNode>(Op1); 458 459 if (!Mask) 460 return SDValue(); 461 462 int32_t Log2IfPositive = (Mask->getAPIntValue() + 1).exactLogBase2(); 463 464 if (Log2IfPositive <= 0) 465 return SDValue(); // Mask+1 is not a power of 2 466 467 SDValue Op0Op2 = Op0->getOperand(2); 468 EVT ExtendTy = cast<VTSDNode>(Op0Op2)->getVT(); 469 unsigned ExtendTySize = ExtendTy.getSizeInBits(); 470 unsigned Log2 = Log2IfPositive; 471 472 if ((Op0Opcode == MipsISD::VEXTRACT_ZEXT_ELT && Log2 >= ExtendTySize) || 473 Log2 == ExtendTySize) { 474 SDValue Ops[] = { Op0->getOperand(0), Op0->getOperand(1), Op0Op2 }; 475 DAG.MorphNodeTo(Op0.getNode(), MipsISD::VEXTRACT_ZEXT_ELT, 476 Op0->getVTList(), Ops, Op0->getNumOperands()); 477 return Op0; 478 } 479 } 480 481 return SDValue(); 482} 483 484static SDValue performSUBECombine(SDNode *N, SelectionDAG &DAG, 485 TargetLowering::DAGCombinerInfo &DCI, 486 const MipsSubtarget *Subtarget) { 487 if (DCI.isBeforeLegalize()) 488 return SDValue(); 489 490 if (Subtarget->hasMips32() && N->getValueType(0) == MVT::i32 && 491 selectMSUB(N, &DAG)) 492 return SDValue(N, 0); 493 494 return SDValue(); 495} 496 497static SDValue genConstMult(SDValue X, uint64_t C, SDLoc DL, EVT VT, 498 EVT ShiftTy, SelectionDAG &DAG) { 499 // Clear the upper (64 - VT.sizeInBits) bits. 500 C &= ((uint64_t)-1) >> (64 - VT.getSizeInBits()); 501 502 // Return 0. 503 if (C == 0) 504 return DAG.getConstant(0, VT); 505 506 // Return x. 507 if (C == 1) 508 return X; 509 510 // If c is power of 2, return (shl x, log2(c)). 511 if (isPowerOf2_64(C)) 512 return DAG.getNode(ISD::SHL, DL, VT, X, 513 DAG.getConstant(Log2_64(C), ShiftTy)); 514 515 unsigned Log2Ceil = Log2_64_Ceil(C); 516 uint64_t Floor = 1LL << Log2_64(C); 517 uint64_t Ceil = Log2Ceil == 64 ? 0LL : 1LL << Log2Ceil; 518 519 // If |c - floor_c| <= |c - ceil_c|, 520 // where floor_c = pow(2, floor(log2(c))) and ceil_c = pow(2, ceil(log2(c))), 521 // return (add constMult(x, floor_c), constMult(x, c - floor_c)). 522 if (C - Floor <= Ceil - C) { 523 SDValue Op0 = genConstMult(X, Floor, DL, VT, ShiftTy, DAG); 524 SDValue Op1 = genConstMult(X, C - Floor, DL, VT, ShiftTy, DAG); 525 return DAG.getNode(ISD::ADD, DL, VT, Op0, Op1); 526 } 527 528 // If |c - floor_c| > |c - ceil_c|, 529 // return (sub constMult(x, ceil_c), constMult(x, ceil_c - c)). 530 SDValue Op0 = genConstMult(X, Ceil, DL, VT, ShiftTy, DAG); 531 SDValue Op1 = genConstMult(X, Ceil - C, DL, VT, ShiftTy, DAG); 532 return DAG.getNode(ISD::SUB, DL, VT, Op0, Op1); 533} 534 535static SDValue performMULCombine(SDNode *N, SelectionDAG &DAG, 536 const TargetLowering::DAGCombinerInfo &DCI, 537 const MipsSETargetLowering *TL) { 538 EVT VT = N->getValueType(0); 539 540 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(N->getOperand(1))) 541 if (!VT.isVector()) 542 return genConstMult(N->getOperand(0), C->getZExtValue(), SDLoc(N), 543 VT, TL->getScalarShiftAmountTy(VT), DAG); 544 545 return SDValue(N, 0); 546} 547 548static SDValue performDSPShiftCombine(unsigned Opc, SDNode *N, EVT Ty, 549 SelectionDAG &DAG, 550 const MipsSubtarget *Subtarget) { 551 // See if this is a vector splat immediate node. 552 APInt SplatValue, SplatUndef; 553 unsigned SplatBitSize; 554 bool HasAnyUndefs; 555 unsigned EltSize = Ty.getVectorElementType().getSizeInBits(); 556 BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(N->getOperand(1)); 557 558 if (!BV || 559 !BV->isConstantSplat(SplatValue, SplatUndef, SplatBitSize, HasAnyUndefs, 560 EltSize, !Subtarget->isLittle()) || 561 (SplatBitSize != EltSize) || 562 (SplatValue.getZExtValue() >= EltSize)) 563 return SDValue(); 564 565 return DAG.getNode(Opc, SDLoc(N), Ty, N->getOperand(0), 566 DAG.getConstant(SplatValue.getZExtValue(), MVT::i32)); 567} 568 569static SDValue performSHLCombine(SDNode *N, SelectionDAG &DAG, 570 TargetLowering::DAGCombinerInfo &DCI, 571 const MipsSubtarget *Subtarget) { 572 EVT Ty = N->getValueType(0); 573 574 if ((Ty != MVT::v2i16) && (Ty != MVT::v4i8)) 575 return SDValue(); 576 577 return performDSPShiftCombine(MipsISD::SHLL_DSP, N, Ty, DAG, Subtarget); 578} 579 580// Fold sign-extensions into MipsISD::VEXTRACT_[SZ]EXT_ELT for MSA and fold 581// constant splats into MipsISD::SHRA_DSP for DSPr2. 582// 583// Performs the following transformations: 584// - Changes MipsISD::VEXTRACT_[SZ]EXT_ELT to sign extension if its 585// sign/zero-extension is completely overwritten by the new one performed by 586// the ISD::SRA and ISD::SHL nodes. 587// - Removes redundant sign extensions performed by an ISD::SRA and ISD::SHL 588// sequence. 589// 590// See performDSPShiftCombine for more information about the transformation 591// used for DSPr2. 592static SDValue performSRACombine(SDNode *N, SelectionDAG &DAG, 593 TargetLowering::DAGCombinerInfo &DCI, 594 const MipsSubtarget *Subtarget) { 595 EVT Ty = N->getValueType(0); 596 597 if (Subtarget->hasMSA()) { 598 SDValue Op0 = N->getOperand(0); 599 SDValue Op1 = N->getOperand(1); 600 601 // (sra (shl (MipsVExtract[SZ]Ext $a, $b, $c), imm:$d), imm:$d) 602 // where $d + sizeof($c) == 32 603 // or $d + sizeof($c) <= 32 and SExt 604 // -> (MipsVExtractSExt $a, $b, $c) 605 if (Op0->getOpcode() == ISD::SHL && Op1 == Op0->getOperand(1)) { 606 SDValue Op0Op0 = Op0->getOperand(0); 607 ConstantSDNode *ShAmount = dyn_cast<ConstantSDNode>(Op1); 608 609 if (!ShAmount) 610 return SDValue(); 611 612 EVT ExtendTy = cast<VTSDNode>(Op0Op0->getOperand(2))->getVT(); 613 unsigned TotalBits = ShAmount->getZExtValue() + ExtendTy.getSizeInBits(); 614 615 if (TotalBits == 32 || 616 (Op0Op0->getOpcode() == MipsISD::VEXTRACT_SEXT_ELT && 617 TotalBits <= 32)) { 618 SDValue Ops[] = { Op0Op0->getOperand(0), Op0Op0->getOperand(1), 619 Op0Op0->getOperand(2) }; 620 DAG.MorphNodeTo(Op0Op0.getNode(), MipsISD::VEXTRACT_SEXT_ELT, 621 Op0Op0->getVTList(), Ops, Op0Op0->getNumOperands()); 622 return Op0Op0; 623 } 624 } 625 } 626 627 if ((Ty != MVT::v2i16) && ((Ty != MVT::v4i8) || !Subtarget->hasDSPR2())) 628 return SDValue(); 629 630 return performDSPShiftCombine(MipsISD::SHRA_DSP, N, Ty, DAG, Subtarget); 631} 632 633 634static SDValue performSRLCombine(SDNode *N, SelectionDAG &DAG, 635 TargetLowering::DAGCombinerInfo &DCI, 636 const MipsSubtarget *Subtarget) { 637 EVT Ty = N->getValueType(0); 638 639 if (((Ty != MVT::v2i16) || !Subtarget->hasDSPR2()) && (Ty != MVT::v4i8)) 640 return SDValue(); 641 642 return performDSPShiftCombine(MipsISD::SHRL_DSP, N, Ty, DAG, Subtarget); 643} 644 645static bool isLegalDSPCondCode(EVT Ty, ISD::CondCode CC) { 646 bool IsV216 = (Ty == MVT::v2i16); 647 648 switch (CC) { 649 case ISD::SETEQ: 650 case ISD::SETNE: return true; 651 case ISD::SETLT: 652 case ISD::SETLE: 653 case ISD::SETGT: 654 case ISD::SETGE: return IsV216; 655 case ISD::SETULT: 656 case ISD::SETULE: 657 case ISD::SETUGT: 658 case ISD::SETUGE: return !IsV216; 659 default: return false; 660 } 661} 662 663static SDValue performSETCCCombine(SDNode *N, SelectionDAG &DAG) { 664 EVT Ty = N->getValueType(0); 665 666 if ((Ty != MVT::v2i16) && (Ty != MVT::v4i8)) 667 return SDValue(); 668 669 if (!isLegalDSPCondCode(Ty, cast<CondCodeSDNode>(N->getOperand(2))->get())) 670 return SDValue(); 671 672 return DAG.getNode(MipsISD::SETCC_DSP, SDLoc(N), Ty, N->getOperand(0), 673 N->getOperand(1), N->getOperand(2)); 674} 675 676static SDValue performVSELECTCombine(SDNode *N, SelectionDAG &DAG) { 677 EVT Ty = N->getValueType(0); 678 679 if (Ty.is128BitVector() && Ty.isInteger()) { 680 // Try the following combines: 681 // (vselect (setcc $a, $b, SETLT), $b, $a)) -> (vsmax $a, $b) 682 // (vselect (setcc $a, $b, SETLE), $b, $a)) -> (vsmax $a, $b) 683 // (vselect (setcc $a, $b, SETLT), $a, $b)) -> (vsmin $a, $b) 684 // (vselect (setcc $a, $b, SETLE), $a, $b)) -> (vsmin $a, $b) 685 // (vselect (setcc $a, $b, SETULT), $b, $a)) -> (vumax $a, $b) 686 // (vselect (setcc $a, $b, SETULE), $b, $a)) -> (vumax $a, $b) 687 // (vselect (setcc $a, $b, SETULT), $a, $b)) -> (vumin $a, $b) 688 // (vselect (setcc $a, $b, SETULE), $a, $b)) -> (vumin $a, $b) 689 // SETGT/SETGE/SETUGT/SETUGE variants of these will show up initially but 690 // will be expanded to equivalent SETLT/SETLE/SETULT/SETULE versions by the 691 // legalizer. 692 SDValue Op0 = N->getOperand(0); 693 694 if (Op0->getOpcode() != ISD::SETCC) 695 return SDValue(); 696 697 ISD::CondCode CondCode = cast<CondCodeSDNode>(Op0->getOperand(2))->get(); 698 bool Signed; 699 700 if (CondCode == ISD::SETLT || CondCode == ISD::SETLE) 701 Signed = true; 702 else if (CondCode == ISD::SETULT || CondCode == ISD::SETULE) 703 Signed = false; 704 else 705 return SDValue(); 706 707 SDValue Op1 = N->getOperand(1); 708 SDValue Op2 = N->getOperand(2); 709 SDValue Op0Op0 = Op0->getOperand(0); 710 SDValue Op0Op1 = Op0->getOperand(1); 711 712 if (Op1 == Op0Op0 && Op2 == Op0Op1) 713 return DAG.getNode(Signed ? MipsISD::VSMIN : MipsISD::VUMIN, SDLoc(N), 714 Ty, Op1, Op2); 715 else if (Op1 == Op0Op1 && Op2 == Op0Op0) 716 return DAG.getNode(Signed ? MipsISD::VSMAX : MipsISD::VUMAX, SDLoc(N), 717 Ty, Op1, Op2); 718 } else if ((Ty == MVT::v2i16) || (Ty == MVT::v4i8)) { 719 SDValue SetCC = N->getOperand(0); 720 721 if (SetCC.getOpcode() != MipsISD::SETCC_DSP) 722 return SDValue(); 723 724 return DAG.getNode(MipsISD::SELECT_CC_DSP, SDLoc(N), Ty, 725 SetCC.getOperand(0), SetCC.getOperand(1), 726 N->getOperand(1), N->getOperand(2), SetCC.getOperand(2)); 727 } 728 729 return SDValue(); 730} 731 732static SDValue performXORCombine(SDNode *N, SelectionDAG &DAG, 733 const MipsSubtarget *Subtarget) { 734 EVT Ty = N->getValueType(0); 735 736 if (Subtarget->hasMSA() && Ty.is128BitVector() && Ty.isInteger()) { 737 // Try the following combines: 738 // (xor (or $a, $b), (build_vector allones)) 739 // (xor (or $a, $b), (bitcast (build_vector allones))) 740 SDValue Op0 = N->getOperand(0); 741 SDValue Op1 = N->getOperand(1); 742 SDValue NotOp; 743 744 if (ISD::isBuildVectorAllOnes(Op0.getNode())) 745 NotOp = Op1; 746 else if (ISD::isBuildVectorAllOnes(Op1.getNode())) 747 NotOp = Op0; 748 else 749 return SDValue(); 750 751 if (NotOp->getOpcode() == ISD::OR) 752 return DAG.getNode(MipsISD::VNOR, SDLoc(N), Ty, NotOp->getOperand(0), 753 NotOp->getOperand(1)); 754 } 755 756 return SDValue(); 757} 758 759SDValue 760MipsSETargetLowering::PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const { 761 SelectionDAG &DAG = DCI.DAG; 762 SDValue Val; 763 764 switch (N->getOpcode()) { 765 case ISD::ADDE: 766 return performADDECombine(N, DAG, DCI, Subtarget); 767 case ISD::AND: 768 Val = performANDCombine(N, DAG, DCI, Subtarget); 769 break; 770 case ISD::SUBE: 771 return performSUBECombine(N, DAG, DCI, Subtarget); 772 case ISD::MUL: 773 return performMULCombine(N, DAG, DCI, this); 774 case ISD::SHL: 775 return performSHLCombine(N, DAG, DCI, Subtarget); 776 case ISD::SRA: 777 return performSRACombine(N, DAG, DCI, Subtarget); 778 case ISD::SRL: 779 return performSRLCombine(N, DAG, DCI, Subtarget); 780 case ISD::VSELECT: 781 return performVSELECTCombine(N, DAG); 782 case ISD::XOR: 783 Val = performXORCombine(N, DAG, Subtarget); 784 break; 785 case ISD::SETCC: 786 Val = performSETCCCombine(N, DAG); 787 break; 788 } 789 790 if (Val.getNode()) 791 return Val; 792 793 return MipsTargetLowering::PerformDAGCombine(N, DCI); 794} 795 796MachineBasicBlock * 797MipsSETargetLowering::EmitInstrWithCustomInserter(MachineInstr *MI, 798 MachineBasicBlock *BB) const { 799 switch (MI->getOpcode()) { 800 default: 801 return MipsTargetLowering::EmitInstrWithCustomInserter(MI, BB); 802 case Mips::BPOSGE32_PSEUDO: 803 return emitBPOSGE32(MI, BB); 804 case Mips::SNZ_B_PSEUDO: 805 return emitMSACBranchPseudo(MI, BB, Mips::BNZ_B); 806 case Mips::SNZ_H_PSEUDO: 807 return emitMSACBranchPseudo(MI, BB, Mips::BNZ_H); 808 case Mips::SNZ_W_PSEUDO: 809 return emitMSACBranchPseudo(MI, BB, Mips::BNZ_W); 810 case Mips::SNZ_D_PSEUDO: 811 return emitMSACBranchPseudo(MI, BB, Mips::BNZ_D); 812 case Mips::SNZ_V_PSEUDO: 813 return emitMSACBranchPseudo(MI, BB, Mips::BNZ_V); 814 case Mips::SZ_B_PSEUDO: 815 return emitMSACBranchPseudo(MI, BB, Mips::BZ_B); 816 case Mips::SZ_H_PSEUDO: 817 return emitMSACBranchPseudo(MI, BB, Mips::BZ_H); 818 case Mips::SZ_W_PSEUDO: 819 return emitMSACBranchPseudo(MI, BB, Mips::BZ_W); 820 case Mips::SZ_D_PSEUDO: 821 return emitMSACBranchPseudo(MI, BB, Mips::BZ_D); 822 case Mips::SZ_V_PSEUDO: 823 return emitMSACBranchPseudo(MI, BB, Mips::BZ_V); 824 } 825} 826 827bool MipsSETargetLowering:: 828isEligibleForTailCallOptimization(const MipsCC &MipsCCInfo, 829 unsigned NextStackOffset, 830 const MipsFunctionInfo& FI) const { 831 if (!EnableMipsTailCalls) 832 return false; 833 834 // Return false if either the callee or caller has a byval argument. 835 if (MipsCCInfo.hasByValArg() || FI.hasByvalArg()) 836 return false; 837 838 // Return true if the callee's argument area is no larger than the 839 // caller's. 840 return NextStackOffset <= FI.getIncomingArgSize(); 841} 842 843void MipsSETargetLowering:: 844getOpndList(SmallVectorImpl<SDValue> &Ops, 845 std::deque< std::pair<unsigned, SDValue> > &RegsToPass, 846 bool IsPICCall, bool GlobalOrExternal, bool InternalLinkage, 847 CallLoweringInfo &CLI, SDValue Callee, SDValue Chain) const { 848 // T9 should contain the address of the callee function if 849 // -reloction-model=pic or it is an indirect call. 850 if (IsPICCall || !GlobalOrExternal) { 851 unsigned T9Reg = IsN64 ? Mips::T9_64 : Mips::T9; 852 RegsToPass.push_front(std::make_pair(T9Reg, Callee)); 853 } else 854 Ops.push_back(Callee); 855 856 MipsTargetLowering::getOpndList(Ops, RegsToPass, IsPICCall, GlobalOrExternal, 857 InternalLinkage, CLI, Callee, Chain); 858} 859 860SDValue MipsSETargetLowering::lowerLOAD(SDValue Op, SelectionDAG &DAG) const { 861 LoadSDNode &Nd = *cast<LoadSDNode>(Op); 862 863 if (Nd.getMemoryVT() != MVT::f64 || !NoDPLoadStore) 864 return MipsTargetLowering::lowerLOAD(Op, DAG); 865 866 // Replace a double precision load with two i32 loads and a buildpair64. 867 SDLoc DL(Op); 868 SDValue Ptr = Nd.getBasePtr(), Chain = Nd.getChain(); 869 EVT PtrVT = Ptr.getValueType(); 870 871 // i32 load from lower address. 872 SDValue Lo = DAG.getLoad(MVT::i32, DL, Chain, Ptr, 873 MachinePointerInfo(), Nd.isVolatile(), 874 Nd.isNonTemporal(), Nd.isInvariant(), 875 Nd.getAlignment()); 876 877 // i32 load from higher address. 878 Ptr = DAG.getNode(ISD::ADD, DL, PtrVT, Ptr, DAG.getConstant(4, PtrVT)); 879 SDValue Hi = DAG.getLoad(MVT::i32, DL, Lo.getValue(1), Ptr, 880 MachinePointerInfo(), Nd.isVolatile(), 881 Nd.isNonTemporal(), Nd.isInvariant(), 882 std::min(Nd.getAlignment(), 4U)); 883 884 if (!Subtarget->isLittle()) 885 std::swap(Lo, Hi); 886 887 SDValue BP = DAG.getNode(MipsISD::BuildPairF64, DL, MVT::f64, Lo, Hi); 888 SDValue Ops[2] = {BP, Hi.getValue(1)}; 889 return DAG.getMergeValues(Ops, 2, DL); 890} 891 892SDValue MipsSETargetLowering::lowerSTORE(SDValue Op, SelectionDAG &DAG) const { 893 StoreSDNode &Nd = *cast<StoreSDNode>(Op); 894 895 if (Nd.getMemoryVT() != MVT::f64 || !NoDPLoadStore) 896 return MipsTargetLowering::lowerSTORE(Op, DAG); 897 898 // Replace a double precision store with two extractelement64s and i32 stores. 899 SDLoc DL(Op); 900 SDValue Val = Nd.getValue(), Ptr = Nd.getBasePtr(), Chain = Nd.getChain(); 901 EVT PtrVT = Ptr.getValueType(); 902 SDValue Lo = DAG.getNode(MipsISD::ExtractElementF64, DL, MVT::i32, 903 Val, DAG.getConstant(0, MVT::i32)); 904 SDValue Hi = DAG.getNode(MipsISD::ExtractElementF64, DL, MVT::i32, 905 Val, DAG.getConstant(1, MVT::i32)); 906 907 if (!Subtarget->isLittle()) 908 std::swap(Lo, Hi); 909 910 // i32 store to lower address. 911 Chain = DAG.getStore(Chain, DL, Lo, Ptr, MachinePointerInfo(), 912 Nd.isVolatile(), Nd.isNonTemporal(), Nd.getAlignment(), 913 Nd.getTBAAInfo()); 914 915 // i32 store to higher address. 916 Ptr = DAG.getNode(ISD::ADD, DL, PtrVT, Ptr, DAG.getConstant(4, PtrVT)); 917 return DAG.getStore(Chain, DL, Hi, Ptr, MachinePointerInfo(), 918 Nd.isVolatile(), Nd.isNonTemporal(), 919 std::min(Nd.getAlignment(), 4U), Nd.getTBAAInfo()); 920} 921 922SDValue MipsSETargetLowering::lowerMulDiv(SDValue Op, unsigned NewOpc, 923 bool HasLo, bool HasHi, 924 SelectionDAG &DAG) const { 925 EVT Ty = Op.getOperand(0).getValueType(); 926 SDLoc DL(Op); 927 SDValue Mult = DAG.getNode(NewOpc, DL, MVT::Untyped, 928 Op.getOperand(0), Op.getOperand(1)); 929 SDValue Lo, Hi; 930 931 if (HasLo) 932 Lo = DAG.getNode(MipsISD::ExtractLOHI, DL, Ty, Mult, 933 DAG.getConstant(Mips::sub_lo, MVT::i32)); 934 if (HasHi) 935 Hi = DAG.getNode(MipsISD::ExtractLOHI, DL, Ty, Mult, 936 DAG.getConstant(Mips::sub_hi, MVT::i32)); 937 938 if (!HasLo || !HasHi) 939 return HasLo ? Lo : Hi; 940 941 SDValue Vals[] = { Lo, Hi }; 942 return DAG.getMergeValues(Vals, 2, DL); 943} 944 945 946static SDValue initAccumulator(SDValue In, SDLoc DL, SelectionDAG &DAG) { 947 SDValue InLo = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i32, In, 948 DAG.getConstant(0, MVT::i32)); 949 SDValue InHi = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i32, In, 950 DAG.getConstant(1, MVT::i32)); 951 return DAG.getNode(MipsISD::InsertLOHI, DL, MVT::Untyped, InLo, InHi); 952} 953 954static SDValue extractLOHI(SDValue Op, SDLoc DL, SelectionDAG &DAG) { 955 SDValue Lo = DAG.getNode(MipsISD::ExtractLOHI, DL, MVT::i32, Op, 956 DAG.getConstant(Mips::sub_lo, MVT::i32)); 957 SDValue Hi = DAG.getNode(MipsISD::ExtractLOHI, DL, MVT::i32, Op, 958 DAG.getConstant(Mips::sub_hi, MVT::i32)); 959 return DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i64, Lo, Hi); 960} 961 962// This function expands mips intrinsic nodes which have 64-bit input operands 963// or output values. 964// 965// out64 = intrinsic-node in64 966// => 967// lo = copy (extract-element (in64, 0)) 968// hi = copy (extract-element (in64, 1)) 969// mips-specific-node 970// v0 = copy lo 971// v1 = copy hi 972// out64 = merge-values (v0, v1) 973// 974static SDValue lowerDSPIntr(SDValue Op, SelectionDAG &DAG, unsigned Opc) { 975 SDLoc DL(Op); 976 bool HasChainIn = Op->getOperand(0).getValueType() == MVT::Other; 977 SmallVector<SDValue, 3> Ops; 978 unsigned OpNo = 0; 979 980 // See if Op has a chain input. 981 if (HasChainIn) 982 Ops.push_back(Op->getOperand(OpNo++)); 983 984 // The next operand is the intrinsic opcode. 985 assert(Op->getOperand(OpNo).getOpcode() == ISD::TargetConstant); 986 987 // See if the next operand has type i64. 988 SDValue Opnd = Op->getOperand(++OpNo), In64; 989 990 if (Opnd.getValueType() == MVT::i64) 991 In64 = initAccumulator(Opnd, DL, DAG); 992 else 993 Ops.push_back(Opnd); 994 995 // Push the remaining operands. 996 for (++OpNo ; OpNo < Op->getNumOperands(); ++OpNo) 997 Ops.push_back(Op->getOperand(OpNo)); 998 999 // Add In64 to the end of the list. 1000 if (In64.getNode()) 1001 Ops.push_back(In64); 1002 1003 // Scan output. 1004 SmallVector<EVT, 2> ResTys; 1005 1006 for (SDNode::value_iterator I = Op->value_begin(), E = Op->value_end(); 1007 I != E; ++I) 1008 ResTys.push_back((*I == MVT::i64) ? MVT::Untyped : *I); 1009 1010 // Create node. 1011 SDValue Val = DAG.getNode(Opc, DL, ResTys, &Ops[0], Ops.size()); 1012 SDValue Out = (ResTys[0] == MVT::Untyped) ? extractLOHI(Val, DL, DAG) : Val; 1013 1014 if (!HasChainIn) 1015 return Out; 1016 1017 assert(Val->getValueType(1) == MVT::Other); 1018 SDValue Vals[] = { Out, SDValue(Val.getNode(), 1) }; 1019 return DAG.getMergeValues(Vals, 2, DL); 1020} 1021 1022static SDValue lowerMSABinaryIntr(SDValue Op, SelectionDAG &DAG, unsigned Opc) { 1023 SDLoc DL(Op); 1024 SDValue LHS = Op->getOperand(1); 1025 SDValue RHS = Op->getOperand(2); 1026 EVT ResTy = Op->getValueType(0); 1027 1028 SDValue Result = DAG.getNode(Opc, DL, ResTy, LHS, RHS); 1029 1030 return Result; 1031} 1032 1033static SDValue lowerMSABinaryImmIntr(SDValue Op, SelectionDAG &DAG, 1034 unsigned Opc, SDValue RHS) { 1035 SDValue LHS = Op->getOperand(1); 1036 EVT ResTy = Op->getValueType(0); 1037 1038 return DAG.getNode(Opc, SDLoc(Op), ResTy, LHS, RHS); 1039} 1040 1041static SDValue lowerMSABranchIntr(SDValue Op, SelectionDAG &DAG, unsigned Opc) { 1042 SDLoc DL(Op); 1043 SDValue Value = Op->getOperand(1); 1044 EVT ResTy = Op->getValueType(0); 1045 1046 SDValue Result = DAG.getNode(Opc, DL, ResTy, Value); 1047 1048 return Result; 1049} 1050 1051// Lower an MSA copy intrinsic into the specified SelectionDAG node 1052static SDValue lowerMSACopyIntr(SDValue Op, SelectionDAG &DAG, unsigned Opc) { 1053 SDLoc DL(Op); 1054 SDValue Vec = Op->getOperand(1); 1055 SDValue Idx = Op->getOperand(2); 1056 EVT ResTy = Op->getValueType(0); 1057 EVT EltTy = Vec->getValueType(0).getVectorElementType(); 1058 1059 SDValue Result = DAG.getNode(Opc, DL, ResTy, Vec, Idx, 1060 DAG.getValueType(EltTy)); 1061 1062 return Result; 1063} 1064 1065// Lower an MSA insert intrinsic into the specified SelectionDAG node 1066static SDValue lowerMSAInsertIntr(SDValue Op, SelectionDAG &DAG, unsigned Opc) { 1067 SDLoc DL(Op); 1068 SDValue Op0 = Op->getOperand(1); 1069 SDValue Op1 = Op->getOperand(2); 1070 SDValue Op2 = Op->getOperand(3); 1071 EVT ResTy = Op->getValueType(0); 1072 1073 SDValue Result = DAG.getNode(Opc, DL, ResTy, Op0, Op2, Op1); 1074 1075 return Result; 1076} 1077 1078static SDValue lowerMSASplatImm(SDValue Op, SDValue ImmOp, SelectionDAG &DAG) { 1079 EVT ResTy = Op->getValueType(0); 1080 EVT ViaVecTy = ResTy; 1081 SmallVector<SDValue, 16> Ops; 1082 SDValue ImmHiOp; 1083 SDLoc DL(Op); 1084 1085 if (ViaVecTy == MVT::v2i64) { 1086 ImmHiOp = DAG.getNode(ISD::SRA, DL, MVT::i32, ImmOp, 1087 DAG.getConstant(31, MVT::i32)); 1088 for (unsigned i = 0; i < ViaVecTy.getVectorNumElements(); ++i) { 1089 Ops.push_back(ImmHiOp); 1090 Ops.push_back(ImmOp); 1091 } 1092 ViaVecTy = MVT::v4i32; 1093 } else { 1094 for (unsigned i = 0; i < ResTy.getVectorNumElements(); ++i) 1095 Ops.push_back(ImmOp); 1096 } 1097 1098 SDValue Result = DAG.getNode(ISD::BUILD_VECTOR, DL, ViaVecTy, &Ops[0], 1099 Ops.size()); 1100 1101 if (ResTy != ViaVecTy) 1102 Result = DAG.getNode(ISD::BITCAST, DL, ResTy, Result); 1103 1104 return Result; 1105} 1106 1107static SDValue 1108lowerMSASplatImm(SDValue Op, unsigned ImmOp, SelectionDAG &DAG) { 1109 return lowerMSASplatImm(Op, Op->getOperand(ImmOp), DAG); 1110} 1111 1112static SDValue lowerMSAUnaryIntr(SDValue Op, SelectionDAG &DAG, unsigned Opc) { 1113 SDLoc DL(Op); 1114 SDValue Value = Op->getOperand(1); 1115 EVT ResTy = Op->getValueType(0); 1116 1117 SDValue Result = DAG.getNode(Opc, DL, ResTy, Value); 1118 1119 return Result; 1120} 1121 1122SDValue MipsSETargetLowering::lowerINTRINSIC_WO_CHAIN(SDValue Op, 1123 SelectionDAG &DAG) const { 1124 switch (cast<ConstantSDNode>(Op->getOperand(0))->getZExtValue()) { 1125 default: 1126 return SDValue(); 1127 case Intrinsic::mips_shilo: 1128 return lowerDSPIntr(Op, DAG, MipsISD::SHILO); 1129 case Intrinsic::mips_dpau_h_qbl: 1130 return lowerDSPIntr(Op, DAG, MipsISD::DPAU_H_QBL); 1131 case Intrinsic::mips_dpau_h_qbr: 1132 return lowerDSPIntr(Op, DAG, MipsISD::DPAU_H_QBR); 1133 case Intrinsic::mips_dpsu_h_qbl: 1134 return lowerDSPIntr(Op, DAG, MipsISD::DPSU_H_QBL); 1135 case Intrinsic::mips_dpsu_h_qbr: 1136 return lowerDSPIntr(Op, DAG, MipsISD::DPSU_H_QBR); 1137 case Intrinsic::mips_dpa_w_ph: 1138 return lowerDSPIntr(Op, DAG, MipsISD::DPA_W_PH); 1139 case Intrinsic::mips_dps_w_ph: 1140 return lowerDSPIntr(Op, DAG, MipsISD::DPS_W_PH); 1141 case Intrinsic::mips_dpax_w_ph: 1142 return lowerDSPIntr(Op, DAG, MipsISD::DPAX_W_PH); 1143 case Intrinsic::mips_dpsx_w_ph: 1144 return lowerDSPIntr(Op, DAG, MipsISD::DPSX_W_PH); 1145 case Intrinsic::mips_mulsa_w_ph: 1146 return lowerDSPIntr(Op, DAG, MipsISD::MULSA_W_PH); 1147 case Intrinsic::mips_mult: 1148 return lowerDSPIntr(Op, DAG, MipsISD::Mult); 1149 case Intrinsic::mips_multu: 1150 return lowerDSPIntr(Op, DAG, MipsISD::Multu); 1151 case Intrinsic::mips_madd: 1152 return lowerDSPIntr(Op, DAG, MipsISD::MAdd); 1153 case Intrinsic::mips_maddu: 1154 return lowerDSPIntr(Op, DAG, MipsISD::MAddu); 1155 case Intrinsic::mips_msub: 1156 return lowerDSPIntr(Op, DAG, MipsISD::MSub); 1157 case Intrinsic::mips_msubu: 1158 return lowerDSPIntr(Op, DAG, MipsISD::MSubu); 1159 case Intrinsic::mips_addv_b: 1160 case Intrinsic::mips_addv_h: 1161 case Intrinsic::mips_addv_w: 1162 case Intrinsic::mips_addv_d: 1163 return lowerMSABinaryIntr(Op, DAG, ISD::ADD); 1164 case Intrinsic::mips_addvi_b: 1165 case Intrinsic::mips_addvi_h: 1166 case Intrinsic::mips_addvi_w: 1167 case Intrinsic::mips_addvi_d: 1168 return lowerMSABinaryImmIntr(Op, DAG, ISD::ADD, 1169 lowerMSASplatImm(Op, 2, DAG)); 1170 case Intrinsic::mips_and_v: 1171 return lowerMSABinaryIntr(Op, DAG, ISD::AND); 1172 case Intrinsic::mips_andi_b: 1173 return lowerMSABinaryImmIntr(Op, DAG, ISD::AND, 1174 lowerMSASplatImm(Op, 2, DAG)); 1175 case Intrinsic::mips_bnz_b: 1176 case Intrinsic::mips_bnz_h: 1177 case Intrinsic::mips_bnz_w: 1178 case Intrinsic::mips_bnz_d: 1179 return lowerMSABranchIntr(Op, DAG, MipsISD::VALL_NONZERO); 1180 case Intrinsic::mips_bnz_v: 1181 return lowerMSABranchIntr(Op, DAG, MipsISD::VANY_NONZERO); 1182 case Intrinsic::mips_bsel_v: 1183 return DAG.getNode(ISD::VSELECT, SDLoc(Op), Op->getValueType(0), 1184 Op->getOperand(1), Op->getOperand(2), 1185 Op->getOperand(3)); 1186 case Intrinsic::mips_bseli_b: 1187 return DAG.getNode(ISD::VSELECT, SDLoc(Op), Op->getValueType(0), 1188 Op->getOperand(1), Op->getOperand(2), 1189 lowerMSASplatImm(Op, 3, DAG)); 1190 case Intrinsic::mips_bz_b: 1191 case Intrinsic::mips_bz_h: 1192 case Intrinsic::mips_bz_w: 1193 case Intrinsic::mips_bz_d: 1194 return lowerMSABranchIntr(Op, DAG, MipsISD::VALL_ZERO); 1195 case Intrinsic::mips_bz_v: 1196 return lowerMSABranchIntr(Op, DAG, MipsISD::VANY_ZERO); 1197 case Intrinsic::mips_ceq_b: 1198 case Intrinsic::mips_ceq_h: 1199 case Intrinsic::mips_ceq_w: 1200 case Intrinsic::mips_ceq_d: 1201 return DAG.getSetCC(SDLoc(Op), Op->getValueType(0), Op->getOperand(1), 1202 Op->getOperand(2), ISD::SETEQ); 1203 case Intrinsic::mips_ceqi_b: 1204 case Intrinsic::mips_ceqi_h: 1205 case Intrinsic::mips_ceqi_w: 1206 case Intrinsic::mips_ceqi_d: 1207 return DAG.getSetCC(SDLoc(Op), Op->getValueType(0), Op->getOperand(1), 1208 lowerMSASplatImm(Op, 2, DAG), ISD::SETEQ); 1209 case Intrinsic::mips_cle_s_b: 1210 case Intrinsic::mips_cle_s_h: 1211 case Intrinsic::mips_cle_s_w: 1212 case Intrinsic::mips_cle_s_d: 1213 return DAG.getSetCC(SDLoc(Op), Op->getValueType(0), Op->getOperand(1), 1214 Op->getOperand(2), ISD::SETLE); 1215 case Intrinsic::mips_clei_s_b: 1216 case Intrinsic::mips_clei_s_h: 1217 case Intrinsic::mips_clei_s_w: 1218 case Intrinsic::mips_clei_s_d: 1219 return DAG.getSetCC(SDLoc(Op), Op->getValueType(0), Op->getOperand(1), 1220 lowerMSASplatImm(Op, 2, DAG), ISD::SETLE); 1221 case Intrinsic::mips_cle_u_b: 1222 case Intrinsic::mips_cle_u_h: 1223 case Intrinsic::mips_cle_u_w: 1224 case Intrinsic::mips_cle_u_d: 1225 return DAG.getSetCC(SDLoc(Op), Op->getValueType(0), Op->getOperand(1), 1226 Op->getOperand(2), ISD::SETULE); 1227 case Intrinsic::mips_clei_u_b: 1228 case Intrinsic::mips_clei_u_h: 1229 case Intrinsic::mips_clei_u_w: 1230 case Intrinsic::mips_clei_u_d: 1231 return DAG.getSetCC(SDLoc(Op), Op->getValueType(0), Op->getOperand(1), 1232 lowerMSASplatImm(Op, 2, DAG), ISD::SETULE); 1233 case Intrinsic::mips_clt_s_b: 1234 case Intrinsic::mips_clt_s_h: 1235 case Intrinsic::mips_clt_s_w: 1236 case Intrinsic::mips_clt_s_d: 1237 return DAG.getSetCC(SDLoc(Op), Op->getValueType(0), Op->getOperand(1), 1238 Op->getOperand(2), ISD::SETLT); 1239 case Intrinsic::mips_clti_s_b: 1240 case Intrinsic::mips_clti_s_h: 1241 case Intrinsic::mips_clti_s_w: 1242 case Intrinsic::mips_clti_s_d: 1243 return DAG.getSetCC(SDLoc(Op), Op->getValueType(0), Op->getOperand(1), 1244 lowerMSASplatImm(Op, 2, DAG), ISD::SETLT); 1245 case Intrinsic::mips_clt_u_b: 1246 case Intrinsic::mips_clt_u_h: 1247 case Intrinsic::mips_clt_u_w: 1248 case Intrinsic::mips_clt_u_d: 1249 return DAG.getSetCC(SDLoc(Op), Op->getValueType(0), Op->getOperand(1), 1250 Op->getOperand(2), ISD::SETULT); 1251 case Intrinsic::mips_clti_u_b: 1252 case Intrinsic::mips_clti_u_h: 1253 case Intrinsic::mips_clti_u_w: 1254 case Intrinsic::mips_clti_u_d: 1255 return DAG.getSetCC(SDLoc(Op), Op->getValueType(0), Op->getOperand(1), 1256 lowerMSASplatImm(Op, 2, DAG), ISD::SETULT); 1257 case Intrinsic::mips_copy_s_b: 1258 case Intrinsic::mips_copy_s_h: 1259 case Intrinsic::mips_copy_s_w: 1260 return lowerMSACopyIntr(Op, DAG, MipsISD::VEXTRACT_SEXT_ELT); 1261 case Intrinsic::mips_copy_u_b: 1262 case Intrinsic::mips_copy_u_h: 1263 case Intrinsic::mips_copy_u_w: 1264 return lowerMSACopyIntr(Op, DAG, MipsISD::VEXTRACT_ZEXT_ELT); 1265 case Intrinsic::mips_div_s_b: 1266 case Intrinsic::mips_div_s_h: 1267 case Intrinsic::mips_div_s_w: 1268 case Intrinsic::mips_div_s_d: 1269 return lowerMSABinaryIntr(Op, DAG, ISD::SDIV); 1270 case Intrinsic::mips_div_u_b: 1271 case Intrinsic::mips_div_u_h: 1272 case Intrinsic::mips_div_u_w: 1273 case Intrinsic::mips_div_u_d: 1274 return lowerMSABinaryIntr(Op, DAG, ISD::UDIV); 1275 case Intrinsic::mips_fadd_w: 1276 case Intrinsic::mips_fadd_d: 1277 return lowerMSABinaryIntr(Op, DAG, ISD::FADD); 1278 // Don't lower mips_fcaf_[wd] since LLVM folds SETFALSE condcodes away 1279 case Intrinsic::mips_fceq_w: 1280 case Intrinsic::mips_fceq_d: 1281 return DAG.getSetCC(SDLoc(Op), Op->getValueType(0), Op->getOperand(1), 1282 Op->getOperand(2), ISD::SETOEQ); 1283 case Intrinsic::mips_fcle_w: 1284 case Intrinsic::mips_fcle_d: 1285 return DAG.getSetCC(SDLoc(Op), Op->getValueType(0), Op->getOperand(1), 1286 Op->getOperand(2), ISD::SETOLE); 1287 case Intrinsic::mips_fclt_w: 1288 case Intrinsic::mips_fclt_d: 1289 return DAG.getSetCC(SDLoc(Op), Op->getValueType(0), Op->getOperand(1), 1290 Op->getOperand(2), ISD::SETOLT); 1291 case Intrinsic::mips_fcne_w: 1292 case Intrinsic::mips_fcne_d: 1293 return DAG.getSetCC(SDLoc(Op), Op->getValueType(0), Op->getOperand(1), 1294 Op->getOperand(2), ISD::SETONE); 1295 case Intrinsic::mips_fcor_w: 1296 case Intrinsic::mips_fcor_d: 1297 return DAG.getSetCC(SDLoc(Op), Op->getValueType(0), Op->getOperand(1), 1298 Op->getOperand(2), ISD::SETO); 1299 case Intrinsic::mips_fcueq_w: 1300 case Intrinsic::mips_fcueq_d: 1301 return DAG.getSetCC(SDLoc(Op), Op->getValueType(0), Op->getOperand(1), 1302 Op->getOperand(2), ISD::SETUEQ); 1303 case Intrinsic::mips_fcule_w: 1304 case Intrinsic::mips_fcule_d: 1305 return DAG.getSetCC(SDLoc(Op), Op->getValueType(0), Op->getOperand(1), 1306 Op->getOperand(2), ISD::SETULE); 1307 case Intrinsic::mips_fcult_w: 1308 case Intrinsic::mips_fcult_d: 1309 return DAG.getSetCC(SDLoc(Op), Op->getValueType(0), Op->getOperand(1), 1310 Op->getOperand(2), ISD::SETULT); 1311 case Intrinsic::mips_fcun_w: 1312 case Intrinsic::mips_fcun_d: 1313 return DAG.getSetCC(SDLoc(Op), Op->getValueType(0), Op->getOperand(1), 1314 Op->getOperand(2), ISD::SETUO); 1315 case Intrinsic::mips_fcune_w: 1316 case Intrinsic::mips_fcune_d: 1317 return DAG.getSetCC(SDLoc(Op), Op->getValueType(0), Op->getOperand(1), 1318 Op->getOperand(2), ISD::SETUNE); 1319 case Intrinsic::mips_fdiv_w: 1320 case Intrinsic::mips_fdiv_d: 1321 return lowerMSABinaryIntr(Op, DAG, ISD::FDIV); 1322 case Intrinsic::mips_fill_b: 1323 case Intrinsic::mips_fill_h: 1324 case Intrinsic::mips_fill_w: { 1325 SmallVector<SDValue, 16> Ops; 1326 EVT ResTy = Op->getValueType(0); 1327 1328 for (unsigned i = 0; i < ResTy.getVectorNumElements(); ++i) 1329 Ops.push_back(Op->getOperand(1)); 1330 1331 return DAG.getNode(ISD::BUILD_VECTOR, SDLoc(Op), ResTy, &Ops[0], 1332 Ops.size()); 1333 } 1334 case Intrinsic::mips_flog2_w: 1335 case Intrinsic::mips_flog2_d: 1336 return lowerMSAUnaryIntr(Op, DAG, ISD::FLOG2); 1337 case Intrinsic::mips_fmul_w: 1338 case Intrinsic::mips_fmul_d: 1339 return lowerMSABinaryIntr(Op, DAG, ISD::FMUL); 1340 case Intrinsic::mips_frint_w: 1341 case Intrinsic::mips_frint_d: 1342 return lowerMSAUnaryIntr(Op, DAG, ISD::FRINT); 1343 case Intrinsic::mips_fsqrt_w: 1344 case Intrinsic::mips_fsqrt_d: 1345 return lowerMSAUnaryIntr(Op, DAG, ISD::FSQRT); 1346 case Intrinsic::mips_fsub_w: 1347 case Intrinsic::mips_fsub_d: 1348 return lowerMSABinaryIntr(Op, DAG, ISD::FSUB); 1349 case Intrinsic::mips_insert_b: 1350 case Intrinsic::mips_insert_h: 1351 case Intrinsic::mips_insert_w: 1352 return lowerMSAInsertIntr(Op, DAG, ISD::INSERT_VECTOR_ELT); 1353 case Intrinsic::mips_ldi_b: 1354 case Intrinsic::mips_ldi_h: 1355 case Intrinsic::mips_ldi_w: 1356 case Intrinsic::mips_ldi_d: 1357 return lowerMSASplatImm(Op, 1, DAG); 1358 case Intrinsic::mips_max_s_b: 1359 case Intrinsic::mips_max_s_h: 1360 case Intrinsic::mips_max_s_w: 1361 case Intrinsic::mips_max_s_d: 1362 return lowerMSABinaryIntr(Op, DAG, MipsISD::VSMAX); 1363 case Intrinsic::mips_max_u_b: 1364 case Intrinsic::mips_max_u_h: 1365 case Intrinsic::mips_max_u_w: 1366 case Intrinsic::mips_max_u_d: 1367 return lowerMSABinaryIntr(Op, DAG, MipsISD::VUMAX); 1368 case Intrinsic::mips_maxi_s_b: 1369 case Intrinsic::mips_maxi_s_h: 1370 case Intrinsic::mips_maxi_s_w: 1371 case Intrinsic::mips_maxi_s_d: 1372 return lowerMSABinaryImmIntr(Op, DAG, MipsISD::VSMAX, 1373 lowerMSASplatImm(Op, 2, DAG)); 1374 case Intrinsic::mips_maxi_u_b: 1375 case Intrinsic::mips_maxi_u_h: 1376 case Intrinsic::mips_maxi_u_w: 1377 case Intrinsic::mips_maxi_u_d: 1378 return lowerMSABinaryImmIntr(Op, DAG, MipsISD::VUMAX, 1379 lowerMSASplatImm(Op, 2, DAG)); 1380 case Intrinsic::mips_min_s_b: 1381 case Intrinsic::mips_min_s_h: 1382 case Intrinsic::mips_min_s_w: 1383 case Intrinsic::mips_min_s_d: 1384 return lowerMSABinaryIntr(Op, DAG, MipsISD::VSMIN); 1385 case Intrinsic::mips_min_u_b: 1386 case Intrinsic::mips_min_u_h: 1387 case Intrinsic::mips_min_u_w: 1388 case Intrinsic::mips_min_u_d: 1389 return lowerMSABinaryIntr(Op, DAG, MipsISD::VUMIN); 1390 case Intrinsic::mips_mini_s_b: 1391 case Intrinsic::mips_mini_s_h: 1392 case Intrinsic::mips_mini_s_w: 1393 case Intrinsic::mips_mini_s_d: 1394 return lowerMSABinaryImmIntr(Op, DAG, MipsISD::VSMIN, 1395 lowerMSASplatImm(Op, 2, DAG)); 1396 case Intrinsic::mips_mini_u_b: 1397 case Intrinsic::mips_mini_u_h: 1398 case Intrinsic::mips_mini_u_w: 1399 case Intrinsic::mips_mini_u_d: 1400 return lowerMSABinaryImmIntr(Op, DAG, MipsISD::VUMIN, 1401 lowerMSASplatImm(Op, 2, DAG)); 1402 case Intrinsic::mips_mulv_b: 1403 case Intrinsic::mips_mulv_h: 1404 case Intrinsic::mips_mulv_w: 1405 case Intrinsic::mips_mulv_d: 1406 return lowerMSABinaryIntr(Op, DAG, ISD::MUL); 1407 case Intrinsic::mips_nlzc_b: 1408 case Intrinsic::mips_nlzc_h: 1409 case Intrinsic::mips_nlzc_w: 1410 case Intrinsic::mips_nlzc_d: 1411 return lowerMSAUnaryIntr(Op, DAG, ISD::CTLZ); 1412 case Intrinsic::mips_nor_v: { 1413 SDValue Res = lowerMSABinaryIntr(Op, DAG, ISD::OR); 1414 return DAG.getNOT(SDLoc(Op), Res, Res->getValueType(0)); 1415 } 1416 case Intrinsic::mips_nori_b: { 1417 SDValue Res = lowerMSABinaryImmIntr(Op, DAG, ISD::OR, 1418 lowerMSASplatImm(Op, 2, DAG)); 1419 return DAG.getNOT(SDLoc(Op), Res, Res->getValueType(0)); 1420 } 1421 case Intrinsic::mips_or_v: 1422 return lowerMSABinaryIntr(Op, DAG, ISD::OR); 1423 case Intrinsic::mips_ori_b: 1424 return lowerMSABinaryImmIntr(Op, DAG, ISD::OR, 1425 lowerMSASplatImm(Op, 2, DAG)); 1426 case Intrinsic::mips_pcnt_b: 1427 case Intrinsic::mips_pcnt_h: 1428 case Intrinsic::mips_pcnt_w: 1429 case Intrinsic::mips_pcnt_d: 1430 return lowerMSAUnaryIntr(Op, DAG, ISD::CTPOP); 1431 case Intrinsic::mips_sll_b: 1432 case Intrinsic::mips_sll_h: 1433 case Intrinsic::mips_sll_w: 1434 case Intrinsic::mips_sll_d: 1435 return lowerMSABinaryIntr(Op, DAG, ISD::SHL); 1436 case Intrinsic::mips_slli_b: 1437 case Intrinsic::mips_slli_h: 1438 case Intrinsic::mips_slli_w: 1439 case Intrinsic::mips_slli_d: 1440 return lowerMSABinaryImmIntr(Op, DAG, ISD::SHL, 1441 lowerMSASplatImm(Op, 2, DAG)); 1442 case Intrinsic::mips_sra_b: 1443 case Intrinsic::mips_sra_h: 1444 case Intrinsic::mips_sra_w: 1445 case Intrinsic::mips_sra_d: 1446 return lowerMSABinaryIntr(Op, DAG, ISD::SRA); 1447 case Intrinsic::mips_srai_b: 1448 case Intrinsic::mips_srai_h: 1449 case Intrinsic::mips_srai_w: 1450 case Intrinsic::mips_srai_d: 1451 return lowerMSABinaryImmIntr(Op, DAG, ISD::SRA, 1452 lowerMSASplatImm(Op, 2, DAG)); 1453 case Intrinsic::mips_srl_b: 1454 case Intrinsic::mips_srl_h: 1455 case Intrinsic::mips_srl_w: 1456 case Intrinsic::mips_srl_d: 1457 return lowerMSABinaryIntr(Op, DAG, ISD::SRL); 1458 case Intrinsic::mips_srli_b: 1459 case Intrinsic::mips_srli_h: 1460 case Intrinsic::mips_srli_w: 1461 case Intrinsic::mips_srli_d: 1462 return lowerMSABinaryImmIntr(Op, DAG, ISD::SRL, 1463 lowerMSASplatImm(Op, 2, DAG)); 1464 case Intrinsic::mips_subv_b: 1465 case Intrinsic::mips_subv_h: 1466 case Intrinsic::mips_subv_w: 1467 case Intrinsic::mips_subv_d: 1468 return lowerMSABinaryIntr(Op, DAG, ISD::SUB); 1469 case Intrinsic::mips_subvi_b: 1470 case Intrinsic::mips_subvi_h: 1471 case Intrinsic::mips_subvi_w: 1472 case Intrinsic::mips_subvi_d: 1473 return lowerMSABinaryImmIntr(Op, DAG, ISD::SUB, 1474 lowerMSASplatImm(Op, 2, DAG)); 1475 case Intrinsic::mips_vshf_b: 1476 case Intrinsic::mips_vshf_h: 1477 case Intrinsic::mips_vshf_w: 1478 case Intrinsic::mips_vshf_d: 1479 return DAG.getNode(MipsISD::VSHF, SDLoc(Op), Op->getValueType(0), 1480 Op->getOperand(1), Op->getOperand(2), Op->getOperand(3)); 1481 case Intrinsic::mips_xor_v: 1482 return lowerMSABinaryIntr(Op, DAG, ISD::XOR); 1483 case Intrinsic::mips_xori_b: 1484 return lowerMSABinaryImmIntr(Op, DAG, ISD::XOR, 1485 lowerMSASplatImm(Op, 2, DAG)); 1486 } 1487} 1488 1489static SDValue lowerMSALoadIntr(SDValue Op, SelectionDAG &DAG, unsigned Intr) { 1490 SDLoc DL(Op); 1491 SDValue ChainIn = Op->getOperand(0); 1492 SDValue Address = Op->getOperand(2); 1493 SDValue Offset = Op->getOperand(3); 1494 EVT ResTy = Op->getValueType(0); 1495 EVT PtrTy = Address->getValueType(0); 1496 1497 Address = DAG.getNode(ISD::ADD, DL, PtrTy, Address, Offset); 1498 1499 return DAG.getLoad(ResTy, DL, ChainIn, Address, MachinePointerInfo(), false, 1500 false, false, 16); 1501} 1502 1503SDValue MipsSETargetLowering::lowerINTRINSIC_W_CHAIN(SDValue Op, 1504 SelectionDAG &DAG) const { 1505 unsigned Intr = cast<ConstantSDNode>(Op->getOperand(1))->getZExtValue(); 1506 switch (Intr) { 1507 default: 1508 return SDValue(); 1509 case Intrinsic::mips_extp: 1510 return lowerDSPIntr(Op, DAG, MipsISD::EXTP); 1511 case Intrinsic::mips_extpdp: 1512 return lowerDSPIntr(Op, DAG, MipsISD::EXTPDP); 1513 case Intrinsic::mips_extr_w: 1514 return lowerDSPIntr(Op, DAG, MipsISD::EXTR_W); 1515 case Intrinsic::mips_extr_r_w: 1516 return lowerDSPIntr(Op, DAG, MipsISD::EXTR_R_W); 1517 case Intrinsic::mips_extr_rs_w: 1518 return lowerDSPIntr(Op, DAG, MipsISD::EXTR_RS_W); 1519 case Intrinsic::mips_extr_s_h: 1520 return lowerDSPIntr(Op, DAG, MipsISD::EXTR_S_H); 1521 case Intrinsic::mips_mthlip: 1522 return lowerDSPIntr(Op, DAG, MipsISD::MTHLIP); 1523 case Intrinsic::mips_mulsaq_s_w_ph: 1524 return lowerDSPIntr(Op, DAG, MipsISD::MULSAQ_S_W_PH); 1525 case Intrinsic::mips_maq_s_w_phl: 1526 return lowerDSPIntr(Op, DAG, MipsISD::MAQ_S_W_PHL); 1527 case Intrinsic::mips_maq_s_w_phr: 1528 return lowerDSPIntr(Op, DAG, MipsISD::MAQ_S_W_PHR); 1529 case Intrinsic::mips_maq_sa_w_phl: 1530 return lowerDSPIntr(Op, DAG, MipsISD::MAQ_SA_W_PHL); 1531 case Intrinsic::mips_maq_sa_w_phr: 1532 return lowerDSPIntr(Op, DAG, MipsISD::MAQ_SA_W_PHR); 1533 case Intrinsic::mips_dpaq_s_w_ph: 1534 return lowerDSPIntr(Op, DAG, MipsISD::DPAQ_S_W_PH); 1535 case Intrinsic::mips_dpsq_s_w_ph: 1536 return lowerDSPIntr(Op, DAG, MipsISD::DPSQ_S_W_PH); 1537 case Intrinsic::mips_dpaq_sa_l_w: 1538 return lowerDSPIntr(Op, DAG, MipsISD::DPAQ_SA_L_W); 1539 case Intrinsic::mips_dpsq_sa_l_w: 1540 return lowerDSPIntr(Op, DAG, MipsISD::DPSQ_SA_L_W); 1541 case Intrinsic::mips_dpaqx_s_w_ph: 1542 return lowerDSPIntr(Op, DAG, MipsISD::DPAQX_S_W_PH); 1543 case Intrinsic::mips_dpaqx_sa_w_ph: 1544 return lowerDSPIntr(Op, DAG, MipsISD::DPAQX_SA_W_PH); 1545 case Intrinsic::mips_dpsqx_s_w_ph: 1546 return lowerDSPIntr(Op, DAG, MipsISD::DPSQX_S_W_PH); 1547 case Intrinsic::mips_dpsqx_sa_w_ph: 1548 return lowerDSPIntr(Op, DAG, MipsISD::DPSQX_SA_W_PH); 1549 case Intrinsic::mips_ld_b: 1550 case Intrinsic::mips_ld_h: 1551 case Intrinsic::mips_ld_w: 1552 case Intrinsic::mips_ld_d: 1553 case Intrinsic::mips_ldx_b: 1554 case Intrinsic::mips_ldx_h: 1555 case Intrinsic::mips_ldx_w: 1556 case Intrinsic::mips_ldx_d: 1557 return lowerMSALoadIntr(Op, DAG, Intr); 1558 } 1559} 1560 1561static SDValue lowerMSAStoreIntr(SDValue Op, SelectionDAG &DAG, unsigned Intr) { 1562 SDLoc DL(Op); 1563 SDValue ChainIn = Op->getOperand(0); 1564 SDValue Value = Op->getOperand(2); 1565 SDValue Address = Op->getOperand(3); 1566 SDValue Offset = Op->getOperand(4); 1567 EVT PtrTy = Address->getValueType(0); 1568 1569 Address = DAG.getNode(ISD::ADD, DL, PtrTy, Address, Offset); 1570 1571 return DAG.getStore(ChainIn, DL, Value, Address, MachinePointerInfo(), false, 1572 false, 16); 1573} 1574 1575SDValue MipsSETargetLowering::lowerINTRINSIC_VOID(SDValue Op, 1576 SelectionDAG &DAG) const { 1577 unsigned Intr = cast<ConstantSDNode>(Op->getOperand(1))->getZExtValue(); 1578 switch (Intr) { 1579 default: 1580 return SDValue(); 1581 case Intrinsic::mips_st_b: 1582 case Intrinsic::mips_st_h: 1583 case Intrinsic::mips_st_w: 1584 case Intrinsic::mips_st_d: 1585 case Intrinsic::mips_stx_b: 1586 case Intrinsic::mips_stx_h: 1587 case Intrinsic::mips_stx_w: 1588 case Intrinsic::mips_stx_d: 1589 return lowerMSAStoreIntr(Op, DAG, Intr); 1590 } 1591} 1592 1593/// \brief Check if the given BuildVectorSDNode is a splat. 1594/// This method currently relies on DAG nodes being reused when equivalent, 1595/// so it's possible for this to return false even when isConstantSplat returns 1596/// true. 1597static bool isSplatVector(const BuildVectorSDNode *N) { 1598 unsigned int nOps = N->getNumOperands(); 1599 assert(nOps > 1 && "isSplat has 0 or 1 sized build vector"); 1600 1601 SDValue Operand0 = N->getOperand(0); 1602 1603 for (unsigned int i = 1; i < nOps; ++i) { 1604 if (N->getOperand(i) != Operand0) 1605 return false; 1606 } 1607 1608 return true; 1609} 1610 1611// Lower ISD::EXTRACT_VECTOR_ELT into MipsISD::VEXTRACT_SEXT_ELT. 1612// 1613// The non-value bits resulting from ISD::EXTRACT_VECTOR_ELT are undefined. We 1614// choose to sign-extend but we could have equally chosen zero-extend. The 1615// DAGCombiner will fold any sign/zero extension of the ISD::EXTRACT_VECTOR_ELT 1616// result into this node later (possibly changing it to a zero-extend in the 1617// process). 1618SDValue MipsSETargetLowering:: 1619lowerEXTRACT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG) const { 1620 SDLoc DL(Op); 1621 EVT ResTy = Op->getValueType(0); 1622 SDValue Op0 = Op->getOperand(0); 1623 SDValue Op1 = Op->getOperand(1); 1624 EVT EltTy = Op0->getValueType(0).getVectorElementType(); 1625 return DAG.getNode(MipsISD::VEXTRACT_SEXT_ELT, DL, ResTy, Op0, Op1, 1626 DAG.getValueType(EltTy)); 1627} 1628 1629static bool isConstantOrUndef(const SDValue Op) { 1630 if (Op->getOpcode() == ISD::UNDEF) 1631 return true; 1632 if (dyn_cast<ConstantSDNode>(Op)) 1633 return true; 1634 if (dyn_cast<ConstantFPSDNode>(Op)) 1635 return true; 1636 return false; 1637} 1638 1639static bool isConstantOrUndefBUILD_VECTOR(const BuildVectorSDNode *Op) { 1640 for (unsigned i = 0; i < Op->getNumOperands(); ++i) 1641 if (isConstantOrUndef(Op->getOperand(i))) 1642 return true; 1643 return false; 1644} 1645 1646// Lowers ISD::BUILD_VECTOR into appropriate SelectionDAG nodes for the 1647// backend. 1648// 1649// Lowers according to the following rules: 1650// - Constant splats are legal as-is as long as the SplatBitSize is a power of 1651// 2 less than or equal to 64 and the value fits into a signed 10-bit 1652// immediate 1653// - Constant splats are lowered to bitconverted BUILD_VECTORs if SplatBitSize 1654// is a power of 2 less than or equal to 64 and the value does not fit into a 1655// signed 10-bit immediate 1656// - Non-constant splats are legal as-is. 1657// - Non-constant non-splats are lowered to sequences of INSERT_VECTOR_ELT. 1658// - All others are illegal and must be expanded. 1659SDValue MipsSETargetLowering::lowerBUILD_VECTOR(SDValue Op, 1660 SelectionDAG &DAG) const { 1661 BuildVectorSDNode *Node = cast<BuildVectorSDNode>(Op); 1662 EVT ResTy = Op->getValueType(0); 1663 SDLoc DL(Op); 1664 APInt SplatValue, SplatUndef; 1665 unsigned SplatBitSize; 1666 bool HasAnyUndefs; 1667 1668 if (!Subtarget->hasMSA() || !ResTy.is128BitVector()) 1669 return SDValue(); 1670 1671 if (Node->isConstantSplat(SplatValue, SplatUndef, SplatBitSize, 1672 HasAnyUndefs, 8, 1673 !Subtarget->isLittle()) && SplatBitSize <= 64) { 1674 // We can only cope with 8, 16, 32, or 64-bit elements 1675 if (SplatBitSize != 8 && SplatBitSize != 16 && SplatBitSize != 32 && 1676 SplatBitSize != 64) 1677 return SDValue(); 1678 1679 // If the value fits into a simm10 then we can use ldi.[bhwd] 1680 if (SplatValue.isSignedIntN(10)) 1681 return Op; 1682 1683 EVT ViaVecTy; 1684 1685 switch (SplatBitSize) { 1686 default: 1687 return SDValue(); 1688 case 8: 1689 ViaVecTy = MVT::v16i8; 1690 break; 1691 case 16: 1692 ViaVecTy = MVT::v8i16; 1693 break; 1694 case 32: 1695 ViaVecTy = MVT::v4i32; 1696 break; 1697 case 64: 1698 // There's no fill.d to fall back on for 64-bit values 1699 return SDValue(); 1700 } 1701 1702 SmallVector<SDValue, 16> Ops; 1703 SDValue Constant = DAG.getConstant(SplatValue.sextOrSelf(32), MVT::i32); 1704 1705 for (unsigned i = 0; i < ViaVecTy.getVectorNumElements(); ++i) 1706 Ops.push_back(Constant); 1707 1708 SDValue Result = DAG.getNode(ISD::BUILD_VECTOR, SDLoc(Node), ViaVecTy, 1709 &Ops[0], Ops.size()); 1710 1711 if (ViaVecTy != ResTy) 1712 Result = DAG.getNode(ISD::BITCAST, SDLoc(Node), ResTy, Result); 1713 1714 return Result; 1715 } else if (isSplatVector(Node)) 1716 return Op; 1717 else if (!isConstantOrUndefBUILD_VECTOR(Node)) { 1718 // Use INSERT_VECTOR_ELT operations rather than expand to stores. 1719 // The resulting code is the same length as the expansion, but it doesn't 1720 // use memory operations 1721 EVT ResTy = Node->getValueType(0); 1722 1723 assert(ResTy.isVector()); 1724 1725 unsigned NumElts = ResTy.getVectorNumElements(); 1726 SDValue Vector = DAG.getUNDEF(ResTy); 1727 for (unsigned i = 0; i < NumElts; ++i) { 1728 Vector = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, ResTy, Vector, 1729 Node->getOperand(i), 1730 DAG.getConstant(i, MVT::i32)); 1731 } 1732 return Vector; 1733 } 1734 1735 return SDValue(); 1736} 1737 1738// Lower VECTOR_SHUFFLE into VSHF. 1739// 1740// This mostly consists of converting the shuffle indices in Indices into a 1741// BUILD_VECTOR and adding it as an operand to the resulting VSHF. There is 1742// also code to eliminate unused operands of the VECTOR_SHUFFLE. For example, 1743// if the type is v8i16 and all the indices are less than 8 then the second 1744// operand is unused and can be replaced with anything. We choose to replace it 1745// with the used operand since this reduces the number of instructions overall. 1746static SDValue lowerVECTOR_SHUFFLE_VSHF(SDValue Op, EVT ResTy, 1747 SmallVector<int, 16> Indices, 1748 SelectionDAG &DAG) { 1749 SmallVector<SDValue, 16> Ops; 1750 SDValue Op0; 1751 SDValue Op1; 1752 EVT MaskVecTy = ResTy.changeVectorElementTypeToInteger(); 1753 EVT MaskEltTy = MaskVecTy.getVectorElementType(); 1754 bool Using1stVec = false; 1755 bool Using2ndVec = false; 1756 SDLoc DL(Op); 1757 int ResTyNumElts = ResTy.getVectorNumElements(); 1758 1759 for (int i = 0; i < ResTyNumElts; ++i) { 1760 // Idx == -1 means UNDEF 1761 int Idx = Indices[i]; 1762 1763 if (0 <= Idx && Idx < ResTyNumElts) 1764 Using1stVec = true; 1765 if (ResTyNumElts <= Idx && Idx < ResTyNumElts * 2) 1766 Using2ndVec = true; 1767 } 1768 1769 for (SmallVector<int, 16>::iterator I = Indices.begin(); I != Indices.end(); 1770 ++I) 1771 Ops.push_back(DAG.getTargetConstant(*I, MaskEltTy)); 1772 1773 SDValue MaskVec = DAG.getNode(ISD::BUILD_VECTOR, DL, MaskVecTy, &Ops[0], 1774 Ops.size()); 1775 1776 if (Using1stVec && Using2ndVec) { 1777 Op0 = Op->getOperand(0); 1778 Op1 = Op->getOperand(1); 1779 } else if (Using1stVec) 1780 Op0 = Op1 = Op->getOperand(0); 1781 else if (Using2ndVec) 1782 Op0 = Op1 = Op->getOperand(1); 1783 else 1784 llvm_unreachable("shuffle vector mask references neither vector operand?"); 1785 1786 return DAG.getNode(MipsISD::VSHF, DL, ResTy, MaskVec, Op0, Op1); 1787} 1788 1789// Lower VECTOR_SHUFFLE into one of a number of instructions depending on the 1790// indices in the shuffle. 1791SDValue MipsSETargetLowering::lowerVECTOR_SHUFFLE(SDValue Op, 1792 SelectionDAG &DAG) const { 1793 ShuffleVectorSDNode *Node = cast<ShuffleVectorSDNode>(Op); 1794 EVT ResTy = Op->getValueType(0); 1795 1796 if (!ResTy.is128BitVector()) 1797 return SDValue(); 1798 1799 int ResTyNumElts = ResTy.getVectorNumElements(); 1800 SmallVector<int, 16> Indices; 1801 1802 for (int i = 0; i < ResTyNumElts; ++i) 1803 Indices.push_back(Node->getMaskElt(i)); 1804 1805 return lowerVECTOR_SHUFFLE_VSHF(Op, ResTy, Indices, DAG); 1806} 1807 1808MachineBasicBlock * MipsSETargetLowering:: 1809emitBPOSGE32(MachineInstr *MI, MachineBasicBlock *BB) const{ 1810 // $bb: 1811 // bposge32_pseudo $vr0 1812 // => 1813 // $bb: 1814 // bposge32 $tbb 1815 // $fbb: 1816 // li $vr2, 0 1817 // b $sink 1818 // $tbb: 1819 // li $vr1, 1 1820 // $sink: 1821 // $vr0 = phi($vr2, $fbb, $vr1, $tbb) 1822 1823 MachineRegisterInfo &RegInfo = BB->getParent()->getRegInfo(); 1824 const TargetInstrInfo *TII = getTargetMachine().getInstrInfo(); 1825 const TargetRegisterClass *RC = &Mips::GPR32RegClass; 1826 DebugLoc DL = MI->getDebugLoc(); 1827 const BasicBlock *LLVM_BB = BB->getBasicBlock(); 1828 MachineFunction::iterator It = llvm::next(MachineFunction::iterator(BB)); 1829 MachineFunction *F = BB->getParent(); 1830 MachineBasicBlock *FBB = F->CreateMachineBasicBlock(LLVM_BB); 1831 MachineBasicBlock *TBB = F->CreateMachineBasicBlock(LLVM_BB); 1832 MachineBasicBlock *Sink = F->CreateMachineBasicBlock(LLVM_BB); 1833 F->insert(It, FBB); 1834 F->insert(It, TBB); 1835 F->insert(It, Sink); 1836 1837 // Transfer the remainder of BB and its successor edges to Sink. 1838 Sink->splice(Sink->begin(), BB, llvm::next(MachineBasicBlock::iterator(MI)), 1839 BB->end()); 1840 Sink->transferSuccessorsAndUpdatePHIs(BB); 1841 1842 // Add successors. 1843 BB->addSuccessor(FBB); 1844 BB->addSuccessor(TBB); 1845 FBB->addSuccessor(Sink); 1846 TBB->addSuccessor(Sink); 1847 1848 // Insert the real bposge32 instruction to $BB. 1849 BuildMI(BB, DL, TII->get(Mips::BPOSGE32)).addMBB(TBB); 1850 1851 // Fill $FBB. 1852 unsigned VR2 = RegInfo.createVirtualRegister(RC); 1853 BuildMI(*FBB, FBB->end(), DL, TII->get(Mips::ADDiu), VR2) 1854 .addReg(Mips::ZERO).addImm(0); 1855 BuildMI(*FBB, FBB->end(), DL, TII->get(Mips::B)).addMBB(Sink); 1856 1857 // Fill $TBB. 1858 unsigned VR1 = RegInfo.createVirtualRegister(RC); 1859 BuildMI(*TBB, TBB->end(), DL, TII->get(Mips::ADDiu), VR1) 1860 .addReg(Mips::ZERO).addImm(1); 1861 1862 // Insert phi function to $Sink. 1863 BuildMI(*Sink, Sink->begin(), DL, TII->get(Mips::PHI), 1864 MI->getOperand(0).getReg()) 1865 .addReg(VR2).addMBB(FBB).addReg(VR1).addMBB(TBB); 1866 1867 MI->eraseFromParent(); // The pseudo instruction is gone now. 1868 return Sink; 1869} 1870 1871MachineBasicBlock * MipsSETargetLowering:: 1872emitMSACBranchPseudo(MachineInstr *MI, MachineBasicBlock *BB, 1873 unsigned BranchOp) const{ 1874 // $bb: 1875 // vany_nonzero $rd, $ws 1876 // => 1877 // $bb: 1878 // bnz.b $ws, $tbb 1879 // b $fbb 1880 // $fbb: 1881 // li $rd1, 0 1882 // b $sink 1883 // $tbb: 1884 // li $rd2, 1 1885 // $sink: 1886 // $rd = phi($rd1, $fbb, $rd2, $tbb) 1887 1888 MachineRegisterInfo &RegInfo = BB->getParent()->getRegInfo(); 1889 const TargetInstrInfo *TII = getTargetMachine().getInstrInfo(); 1890 const TargetRegisterClass *RC = &Mips::GPR32RegClass; 1891 DebugLoc DL = MI->getDebugLoc(); 1892 const BasicBlock *LLVM_BB = BB->getBasicBlock(); 1893 MachineFunction::iterator It = llvm::next(MachineFunction::iterator(BB)); 1894 MachineFunction *F = BB->getParent(); 1895 MachineBasicBlock *FBB = F->CreateMachineBasicBlock(LLVM_BB); 1896 MachineBasicBlock *TBB = F->CreateMachineBasicBlock(LLVM_BB); 1897 MachineBasicBlock *Sink = F->CreateMachineBasicBlock(LLVM_BB); 1898 F->insert(It, FBB); 1899 F->insert(It, TBB); 1900 F->insert(It, Sink); 1901 1902 // Transfer the remainder of BB and its successor edges to Sink. 1903 Sink->splice(Sink->begin(), BB, llvm::next(MachineBasicBlock::iterator(MI)), 1904 BB->end()); 1905 Sink->transferSuccessorsAndUpdatePHIs(BB); 1906 1907 // Add successors. 1908 BB->addSuccessor(FBB); 1909 BB->addSuccessor(TBB); 1910 FBB->addSuccessor(Sink); 1911 TBB->addSuccessor(Sink); 1912 1913 // Insert the real bnz.b instruction to $BB. 1914 BuildMI(BB, DL, TII->get(BranchOp)) 1915 .addReg(MI->getOperand(1).getReg()) 1916 .addMBB(TBB); 1917 1918 // Fill $FBB. 1919 unsigned RD1 = RegInfo.createVirtualRegister(RC); 1920 BuildMI(*FBB, FBB->end(), DL, TII->get(Mips::ADDiu), RD1) 1921 .addReg(Mips::ZERO).addImm(0); 1922 BuildMI(*FBB, FBB->end(), DL, TII->get(Mips::B)).addMBB(Sink); 1923 1924 // Fill $TBB. 1925 unsigned RD2 = RegInfo.createVirtualRegister(RC); 1926 BuildMI(*TBB, TBB->end(), DL, TII->get(Mips::ADDiu), RD2) 1927 .addReg(Mips::ZERO).addImm(1); 1928 1929 // Insert phi function to $Sink. 1930 BuildMI(*Sink, Sink->begin(), DL, TII->get(Mips::PHI), 1931 MI->getOperand(0).getReg()) 1932 .addReg(RD1).addMBB(FBB).addReg(RD2).addMBB(TBB); 1933 1934 MI->eraseFromParent(); // The pseudo instruction is gone now. 1935 return Sink; 1936} 1937