SIISelLowering.cpp revision 311ea66db107917789c00f2ae11ec6b98eb64e59
1//===-- SIISelLowering.cpp - SI DAG Lowering Implementation ---------------===// 2// 3// The LLVM Compiler Infrastructure 4// 5// This file is distributed under the University of Illinois Open Source 6// License. See LICENSE.TXT for details. 7// 8//===----------------------------------------------------------------------===// 9// 10/// \file 11/// \brief Custom DAG lowering for SI 12// 13//===----------------------------------------------------------------------===// 14 15#include "SIISelLowering.h" 16#include "AMDIL.h" 17#include "AMDILIntrinsicInfo.h" 18#include "SIInstrInfo.h" 19#include "SIMachineFunctionInfo.h" 20#include "SIRegisterInfo.h" 21#include "llvm/CodeGen/MachineInstrBuilder.h" 22#include "llvm/CodeGen/MachineRegisterInfo.h" 23#include "llvm/CodeGen/SelectionDAG.h" 24 25using namespace llvm; 26 27SITargetLowering::SITargetLowering(TargetMachine &TM) : 28 AMDGPUTargetLowering(TM), 29 TII(static_cast<const SIInstrInfo*>(TM.getInstrInfo())) { 30 addRegisterClass(MVT::v4f32, &AMDGPU::VReg_128RegClass); 31 addRegisterClass(MVT::f32, &AMDGPU::VReg_32RegClass); 32 addRegisterClass(MVT::i32, &AMDGPU::VReg_32RegClass); 33 addRegisterClass(MVT::i64, &AMDGPU::SReg_64RegClass); 34 addRegisterClass(MVT::i1, &AMDGPU::SCCRegRegClass); 35 addRegisterClass(MVT::i1, &AMDGPU::VCCRegRegClass); 36 37 addRegisterClass(MVT::v1i32, &AMDGPU::VReg_32RegClass); 38 addRegisterClass(MVT::v2i32, &AMDGPU::VReg_64RegClass); 39 addRegisterClass(MVT::v4i32, &AMDGPU::VReg_128RegClass); 40 addRegisterClass(MVT::v8i32, &AMDGPU::VReg_256RegClass); 41 addRegisterClass(MVT::v16i32, &AMDGPU::VReg_512RegClass); 42 43 computeRegisterProperties(); 44 45 setOperationAction(ISD::AND, MVT::i1, Custom); 46 47 setOperationAction(ISD::ADD, MVT::i64, Legal); 48 setOperationAction(ISD::ADD, MVT::i32, Legal); 49 50 setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::Other, Custom); 51 52 // We need to custom lower loads from the USER_SGPR address space, so we can 53 // add the SGPRs as livein registers. 54 setOperationAction(ISD::LOAD, MVT::i32, Custom); 55 setOperationAction(ISD::LOAD, MVT::i64, Custom); 56 57 setOperationAction(ISD::SELECT_CC, MVT::f32, Custom); 58 setOperationAction(ISD::SELECT_CC, MVT::i32, Custom); 59 60 setOperationAction(ISD::SELECT_CC, MVT::Other, Expand); 61 setTargetDAGCombine(ISD::SELECT_CC); 62 63 setTargetDAGCombine(ISD::SETCC); 64} 65 66MachineBasicBlock * SITargetLowering::EmitInstrWithCustomInserter( 67 MachineInstr * MI, MachineBasicBlock * BB) const { 68 const TargetInstrInfo * TII = getTargetMachine().getInstrInfo(); 69 MachineRegisterInfo & MRI = BB->getParent()->getRegInfo(); 70 MachineBasicBlock::iterator I = MI; 71 72 switch (MI->getOpcode()) { 73 default: 74 return AMDGPUTargetLowering::EmitInstrWithCustomInserter(MI, BB); 75 case AMDGPU::BRANCH: return BB; 76 case AMDGPU::CLAMP_SI: 77 BuildMI(*BB, I, BB->findDebugLoc(I), TII->get(AMDGPU::V_ADD_F32_e64)) 78 .addOperand(MI->getOperand(0)) 79 .addOperand(MI->getOperand(1)) 80 .addReg(AMDGPU::SREG_LIT_0) 81 .addReg(AMDGPU::SREG_LIT_0) 82 .addImm(0) // ABS 83 .addImm(1) // CLAMP 84 .addImm(0) // OMOD 85 .addImm(0); // NEG 86 MI->eraseFromParent(); 87 break; 88 89 case AMDGPU::FABS_SI: 90 BuildMI(*BB, I, BB->findDebugLoc(I), TII->get(AMDGPU::V_ADD_F32_e64)) 91 .addOperand(MI->getOperand(0)) 92 .addOperand(MI->getOperand(1)) 93 .addReg(AMDGPU::SREG_LIT_0) 94 .addReg(AMDGPU::SREG_LIT_0) 95 .addImm(1) // ABS 96 .addImm(0) // CLAMP 97 .addImm(0) // OMOD 98 .addImm(0); // NEG 99 MI->eraseFromParent(); 100 break; 101 102 case AMDGPU::FNEG_SI: 103 BuildMI(*BB, I, BB->findDebugLoc(I), TII->get(AMDGPU::V_ADD_F32_e64)) 104 .addOperand(MI->getOperand(0)) 105 .addOperand(MI->getOperand(1)) 106 .addReg(AMDGPU::SREG_LIT_0) 107 .addReg(AMDGPU::SREG_LIT_0) 108 .addImm(0) // ABS 109 .addImm(0) // CLAMP 110 .addImm(0) // OMOD 111 .addImm(1); // NEG 112 MI->eraseFromParent(); 113 break; 114 case AMDGPU::SHADER_TYPE: 115 BB->getParent()->getInfo<SIMachineFunctionInfo>()->ShaderType = 116 MI->getOperand(0).getImm(); 117 MI->eraseFromParent(); 118 break; 119 120 case AMDGPU::SI_INTERP: 121 LowerSI_INTERP(MI, *BB, I, MRI); 122 break; 123 case AMDGPU::SI_INTERP_CONST: 124 LowerSI_INTERP_CONST(MI, *BB, I, MRI); 125 break; 126 case AMDGPU::SI_WQM: 127 LowerSI_WQM(MI, *BB, I, MRI); 128 break; 129 case AMDGPU::SI_V_CNDLT: 130 LowerSI_V_CNDLT(MI, *BB, I, MRI); 131 break; 132 } 133 return BB; 134} 135 136void SITargetLowering::LowerSI_WQM(MachineInstr *MI, MachineBasicBlock &BB, 137 MachineBasicBlock::iterator I, MachineRegisterInfo & MRI) const { 138 BuildMI(BB, I, BB.findDebugLoc(I), TII->get(AMDGPU::S_WQM_B64), AMDGPU::EXEC) 139 .addReg(AMDGPU::EXEC); 140 141 MI->eraseFromParent(); 142} 143 144void SITargetLowering::LowerSI_INTERP(MachineInstr *MI, MachineBasicBlock &BB, 145 MachineBasicBlock::iterator I, MachineRegisterInfo & MRI) const { 146 unsigned tmp = MRI.createVirtualRegister(&AMDGPU::VReg_32RegClass); 147 unsigned M0 = MRI.createVirtualRegister(&AMDGPU::M0RegRegClass); 148 MachineOperand dst = MI->getOperand(0); 149 MachineOperand iReg = MI->getOperand(1); 150 MachineOperand jReg = MI->getOperand(2); 151 MachineOperand attr_chan = MI->getOperand(3); 152 MachineOperand attr = MI->getOperand(4); 153 MachineOperand params = MI->getOperand(5); 154 155 BuildMI(BB, I, BB.findDebugLoc(I), TII->get(AMDGPU::S_MOV_B32), M0) 156 .addOperand(params); 157 158 BuildMI(BB, I, BB.findDebugLoc(I), TII->get(AMDGPU::V_INTERP_P1_F32), tmp) 159 .addOperand(iReg) 160 .addOperand(attr_chan) 161 .addOperand(attr) 162 .addReg(M0); 163 164 BuildMI(BB, I, BB.findDebugLoc(I), TII->get(AMDGPU::V_INTERP_P2_F32)) 165 .addOperand(dst) 166 .addReg(tmp) 167 .addOperand(jReg) 168 .addOperand(attr_chan) 169 .addOperand(attr) 170 .addReg(M0); 171 172 MI->eraseFromParent(); 173} 174 175void SITargetLowering::LowerSI_INTERP_CONST(MachineInstr *MI, 176 MachineBasicBlock &BB, MachineBasicBlock::iterator I, 177 MachineRegisterInfo &MRI) const { 178 MachineOperand dst = MI->getOperand(0); 179 MachineOperand attr_chan = MI->getOperand(1); 180 MachineOperand attr = MI->getOperand(2); 181 MachineOperand params = MI->getOperand(3); 182 unsigned M0 = MRI.createVirtualRegister(&AMDGPU::M0RegRegClass); 183 184 BuildMI(BB, I, BB.findDebugLoc(I), TII->get(AMDGPU::S_MOV_B32), M0) 185 .addOperand(params); 186 187 BuildMI(BB, I, BB.findDebugLoc(I), TII->get(AMDGPU::V_INTERP_MOV_F32)) 188 .addOperand(dst) 189 .addOperand(attr_chan) 190 .addOperand(attr) 191 .addReg(M0); 192 193 MI->eraseFromParent(); 194} 195 196void SITargetLowering::LowerSI_V_CNDLT(MachineInstr *MI, MachineBasicBlock &BB, 197 MachineBasicBlock::iterator I, MachineRegisterInfo & MRI) const { 198 unsigned VCC = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass); 199 200 BuildMI(BB, I, BB.findDebugLoc(I), 201 TII->get(AMDGPU::V_CMP_GT_F32_e32), 202 VCC) 203 .addReg(AMDGPU::SREG_LIT_0) 204 .addOperand(MI->getOperand(1)); 205 206 BuildMI(BB, I, BB.findDebugLoc(I), TII->get(AMDGPU::V_CNDMASK_B32_e32)) 207 .addOperand(MI->getOperand(0)) 208 .addOperand(MI->getOperand(3)) 209 .addOperand(MI->getOperand(2)) 210 .addReg(VCC); 211 212 MI->eraseFromParent(); 213} 214 215EVT SITargetLowering::getSetCCResultType(EVT VT) const { 216 return MVT::i1; 217} 218 219//===----------------------------------------------------------------------===// 220// Custom DAG Lowering Operations 221//===----------------------------------------------------------------------===// 222 223SDValue SITargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const { 224 switch (Op.getOpcode()) { 225 default: return AMDGPUTargetLowering::LowerOperation(Op, DAG); 226 case ISD::BRCOND: return LowerBRCOND(Op, DAG); 227 case ISD::LOAD: return LowerLOAD(Op, DAG); 228 case ISD::SELECT_CC: return LowerSELECT_CC(Op, DAG); 229 case ISD::AND: return Loweri1ContextSwitch(Op, DAG, ISD::AND); 230 case ISD::INTRINSIC_WO_CHAIN: { 231 unsigned IntrinsicID = 232 cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue(); 233 EVT VT = Op.getValueType(); 234 switch (IntrinsicID) { 235 case AMDGPUIntrinsic::SI_vs_load_buffer_index: 236 return CreateLiveInRegister(DAG, &AMDGPU::VReg_32RegClass, 237 AMDGPU::VGPR0, VT); 238 default: return AMDGPUTargetLowering::LowerOperation(Op, DAG); 239 } 240 break; 241 } 242 } 243 return SDValue(); 244} 245 246/// \brief The function is for lowering i1 operations on the 247/// VCC register. 248/// 249/// In the VALU context, VCC is a one bit register, but in the 250/// SALU context the VCC is a 64-bit register (1-bit per thread). Since only 251/// the SALU can perform operations on the VCC register, we need to promote 252/// the operand types from i1 to i64 in order for tablegen to be able to match 253/// this operation to the correct SALU instruction. We do this promotion by 254/// wrapping the operands in a CopyToReg node. 255/// 256SDValue SITargetLowering::Loweri1ContextSwitch(SDValue Op, 257 SelectionDAG &DAG, 258 unsigned VCCNode) const { 259 DebugLoc DL = Op.getDebugLoc(); 260 261 SDValue OpNode = DAG.getNode(VCCNode, DL, MVT::i64, 262 DAG.getNode(SIISD::VCC_BITCAST, DL, MVT::i64, 263 Op.getOperand(0)), 264 DAG.getNode(SIISD::VCC_BITCAST, DL, MVT::i64, 265 Op.getOperand(1))); 266 267 return DAG.getNode(SIISD::VCC_BITCAST, DL, MVT::i1, OpNode); 268} 269 270/// \brief Helper function for LowerBRCOND 271static SDNode *findUser(SDValue Value, unsigned Opcode) { 272 273 SDNode *Parent = Value.getNode(); 274 for (SDNode::use_iterator I = Parent->use_begin(), E = Parent->use_end(); 275 I != E; ++I) { 276 277 if (I.getUse().get() != Value) 278 continue; 279 280 if (I->getOpcode() == Opcode) 281 return *I; 282 } 283 return 0; 284} 285 286/// This transforms the control flow intrinsics to get the branch destination as 287/// last parameter, also switches branch target with BR if the need arise 288SDValue SITargetLowering::LowerBRCOND(SDValue BRCOND, 289 SelectionDAG &DAG) const { 290 291 DebugLoc DL = BRCOND.getDebugLoc(); 292 293 SDNode *Intr = BRCOND.getOperand(1).getNode(); 294 SDValue Target = BRCOND.getOperand(2); 295 SDNode *BR = 0; 296 297 if (Intr->getOpcode() == ISD::SETCC) { 298 // As long as we negate the condition everything is fine 299 SDNode *SetCC = Intr; 300 assert(SetCC->getConstantOperandVal(1) == 1); 301 assert(cast<CondCodeSDNode>(SetCC->getOperand(2).getNode())->get() == 302 ISD::SETNE); 303 Intr = SetCC->getOperand(0).getNode(); 304 305 } else { 306 // Get the target from BR if we don't negate the condition 307 BR = findUser(BRCOND, ISD::BR); 308 Target = BR->getOperand(1); 309 } 310 311 assert(Intr->getOpcode() == ISD::INTRINSIC_W_CHAIN); 312 313 // Build the result and 314 SmallVector<EVT, 4> Res; 315 for (unsigned i = 1, e = Intr->getNumValues(); i != e; ++i) 316 Res.push_back(Intr->getValueType(i)); 317 318 // operands of the new intrinsic call 319 SmallVector<SDValue, 4> Ops; 320 Ops.push_back(BRCOND.getOperand(0)); 321 for (unsigned i = 1, e = Intr->getNumOperands(); i != e; ++i) 322 Ops.push_back(Intr->getOperand(i)); 323 Ops.push_back(Target); 324 325 // build the new intrinsic call 326 SDNode *Result = DAG.getNode( 327 Res.size() > 1 ? ISD::INTRINSIC_W_CHAIN : ISD::INTRINSIC_VOID, DL, 328 DAG.getVTList(Res.data(), Res.size()), Ops.data(), Ops.size()).getNode(); 329 330 if (BR) { 331 // Give the branch instruction our target 332 SDValue Ops[] = { 333 BR->getOperand(0), 334 BRCOND.getOperand(2) 335 }; 336 DAG.MorphNodeTo(BR, ISD::BR, BR->getVTList(), Ops, 2); 337 } 338 339 SDValue Chain = SDValue(Result, Result->getNumValues() - 1); 340 341 // Copy the intrinsic results to registers 342 for (unsigned i = 1, e = Intr->getNumValues() - 1; i != e; ++i) { 343 SDNode *CopyToReg = findUser(SDValue(Intr, i), ISD::CopyToReg); 344 if (!CopyToReg) 345 continue; 346 347 Chain = DAG.getCopyToReg( 348 Chain, DL, 349 CopyToReg->getOperand(1), 350 SDValue(Result, i - 1), 351 SDValue()); 352 353 DAG.ReplaceAllUsesWith(SDValue(CopyToReg, 0), CopyToReg->getOperand(0)); 354 } 355 356 // Remove the old intrinsic from the chain 357 DAG.ReplaceAllUsesOfValueWith( 358 SDValue(Intr, Intr->getNumValues() - 1), 359 Intr->getOperand(0)); 360 361 return Chain; 362} 363 364SDValue SITargetLowering::LowerLOAD(SDValue Op, SelectionDAG &DAG) const { 365 EVT VT = Op.getValueType(); 366 LoadSDNode *Ptr = dyn_cast<LoadSDNode>(Op); 367 368 assert(Ptr); 369 370 unsigned AddrSpace = Ptr->getPointerInfo().getAddrSpace(); 371 372 // We only need to lower USER_SGPR address space loads 373 if (AddrSpace != AMDGPUAS::USER_SGPR_ADDRESS) { 374 return SDValue(); 375 } 376 377 // Loads from the USER_SGPR address space can only have constant value 378 // pointers. 379 ConstantSDNode *BasePtr = dyn_cast<ConstantSDNode>(Ptr->getBasePtr()); 380 assert(BasePtr); 381 382 unsigned TypeDwordWidth = VT.getSizeInBits() / 32; 383 const TargetRegisterClass * dstClass; 384 switch (TypeDwordWidth) { 385 default: 386 assert(!"USER_SGPR value size not implemented"); 387 return SDValue(); 388 case 1: 389 dstClass = &AMDGPU::SReg_32RegClass; 390 break; 391 case 2: 392 dstClass = &AMDGPU::SReg_64RegClass; 393 break; 394 } 395 uint64_t Index = BasePtr->getZExtValue(); 396 assert(Index % TypeDwordWidth == 0 && "USER_SGPR not properly aligned"); 397 unsigned SGPRIndex = Index / TypeDwordWidth; 398 unsigned Reg = dstClass->getRegister(SGPRIndex); 399 400 DAG.ReplaceAllUsesOfValueWith(Op, CreateLiveInRegister(DAG, dstClass, Reg, 401 VT)); 402 return SDValue(); 403} 404 405SDValue SITargetLowering::LowerSELECT_CC(SDValue Op, SelectionDAG &DAG) const { 406 SDValue LHS = Op.getOperand(0); 407 SDValue RHS = Op.getOperand(1); 408 SDValue True = Op.getOperand(2); 409 SDValue False = Op.getOperand(3); 410 SDValue CC = Op.getOperand(4); 411 EVT VT = Op.getValueType(); 412 DebugLoc DL = Op.getDebugLoc(); 413 414 // Possible Min/Max pattern 415 SDValue MinMax = LowerMinMax(Op, DAG); 416 if (MinMax.getNode()) { 417 return MinMax; 418 } 419 420 SDValue Cond = DAG.getNode(ISD::SETCC, DL, MVT::i1, LHS, RHS, CC); 421 return DAG.getNode(ISD::SELECT, DL, VT, Cond, True, False); 422} 423 424//===----------------------------------------------------------------------===// 425// Custom DAG optimizations 426//===----------------------------------------------------------------------===// 427 428SDValue SITargetLowering::PerformDAGCombine(SDNode *N, 429 DAGCombinerInfo &DCI) const { 430 SelectionDAG &DAG = DCI.DAG; 431 DebugLoc DL = N->getDebugLoc(); 432 EVT VT = N->getValueType(0); 433 434 switch (N->getOpcode()) { 435 default: break; 436 case ISD::SELECT_CC: { 437 N->dump(); 438 ConstantSDNode *True, *False; 439 // i1 selectcc(l, r, -1, 0, cc) -> i1 setcc(l, r, cc) 440 if ((True = dyn_cast<ConstantSDNode>(N->getOperand(2))) 441 && (False = dyn_cast<ConstantSDNode>(N->getOperand(3))) 442 && True->isAllOnesValue() 443 && False->isNullValue() 444 && VT == MVT::i1) { 445 return DAG.getNode(ISD::SETCC, DL, VT, N->getOperand(0), 446 N->getOperand(1), N->getOperand(4)); 447 448 } 449 break; 450 } 451 case ISD::SETCC: { 452 SDValue Arg0 = N->getOperand(0); 453 SDValue Arg1 = N->getOperand(1); 454 SDValue CC = N->getOperand(2); 455 ConstantSDNode * C = NULL; 456 ISD::CondCode CCOp = dyn_cast<CondCodeSDNode>(CC)->get(); 457 458 // i1 setcc (sext(i1), 0, setne) -> i1 setcc(i1, 0, setne) 459 if (VT == MVT::i1 460 && Arg0.getOpcode() == ISD::SIGN_EXTEND 461 && Arg0.getOperand(0).getValueType() == MVT::i1 462 && (C = dyn_cast<ConstantSDNode>(Arg1)) 463 && C->isNullValue() 464 && CCOp == ISD::SETNE) { 465 return SimplifySetCC(VT, Arg0.getOperand(0), 466 DAG.getConstant(0, MVT::i1), CCOp, true, DCI, DL); 467 } 468 break; 469 } 470 } 471 return SDValue(); 472} 473 474#define NODE_NAME_CASE(node) case SIISD::node: return #node; 475 476const char* SITargetLowering::getTargetNodeName(unsigned Opcode) const { 477 switch (Opcode) { 478 default: return AMDGPUTargetLowering::getTargetNodeName(Opcode); 479 NODE_NAME_CASE(VCC_AND) 480 NODE_NAME_CASE(VCC_BITCAST) 481 } 482} 483