1//===----- R600Packetizer.cpp - VLIW packetizer ---------------------------===// 2// 3// The LLVM Compiler Infrastructure 4// 5// This file is distributed under the University of Illinois Open Source 6// License. See LICENSE.TXT for details. 7// 8//===----------------------------------------------------------------------===// 9// 10/// \file 11/// This pass implements instructions packetization for R600. It unsets isLast 12/// bit of instructions inside a bundle and substitutes src register with 13/// PreviousVector when applicable. 14// 15//===----------------------------------------------------------------------===// 16 17#include "llvm/Support/Debug.h" 18#include "AMDGPU.h" 19#include "AMDGPUSubtarget.h" 20#include "R600InstrInfo.h" 21#include "llvm/CodeGen/DFAPacketizer.h" 22#include "llvm/CodeGen/MachineDominators.h" 23#include "llvm/CodeGen/MachineFunctionPass.h" 24#include "llvm/CodeGen/MachineLoopInfo.h" 25#include "llvm/CodeGen/Passes.h" 26#include "llvm/CodeGen/ScheduleDAG.h" 27#include "llvm/Support/raw_ostream.h" 28 29using namespace llvm; 30 31#define DEBUG_TYPE "packets" 32 33namespace { 34 35class R600Packetizer : public MachineFunctionPass { 36 37public: 38 static char ID; 39 R600Packetizer(const TargetMachine &TM) : MachineFunctionPass(ID) {} 40 41 void getAnalysisUsage(AnalysisUsage &AU) const override { 42 AU.setPreservesCFG(); 43 AU.addRequired<MachineDominatorTree>(); 44 AU.addPreserved<MachineDominatorTree>(); 45 AU.addRequired<MachineLoopInfo>(); 46 AU.addPreserved<MachineLoopInfo>(); 47 MachineFunctionPass::getAnalysisUsage(AU); 48 } 49 50 const char *getPassName() const override { 51 return "R600 Packetizer"; 52 } 53 54 bool runOnMachineFunction(MachineFunction &Fn) override; 55}; 56char R600Packetizer::ID = 0; 57 58class R600PacketizerList : public VLIWPacketizerList { 59 60private: 61 const R600InstrInfo *TII; 62 const R600RegisterInfo &TRI; 63 bool VLIW5; 64 bool ConsideredInstUsesAlreadyWrittenVectorElement; 65 66 unsigned getSlot(const MachineInstr *MI) const { 67 return TRI.getHWRegChan(MI->getOperand(0).getReg()); 68 } 69 70 /// \returns register to PV chan mapping for bundle/single instructions that 71 /// immediately precedes I. 72 DenseMap<unsigned, unsigned> getPreviousVector(MachineBasicBlock::iterator I) 73 const { 74 DenseMap<unsigned, unsigned> Result; 75 I--; 76 if (!TII->isALUInstr(I->getOpcode()) && !I->isBundle()) 77 return Result; 78 MachineBasicBlock::instr_iterator BI = I.getInstrIterator(); 79 if (I->isBundle()) 80 BI++; 81 int LastDstChan = -1; 82 do { 83 bool isTrans = false; 84 int BISlot = getSlot(BI); 85 if (LastDstChan >= BISlot) 86 isTrans = true; 87 LastDstChan = BISlot; 88 if (TII->isPredicated(BI)) 89 continue; 90 int OperandIdx = TII->getOperandIdx(BI->getOpcode(), AMDGPU::OpName::write); 91 if (OperandIdx > -1 && BI->getOperand(OperandIdx).getImm() == 0) 92 continue; 93 int DstIdx = TII->getOperandIdx(BI->getOpcode(), AMDGPU::OpName::dst); 94 if (DstIdx == -1) { 95 continue; 96 } 97 unsigned Dst = BI->getOperand(DstIdx).getReg(); 98 if (isTrans || TII->isTransOnly(BI)) { 99 Result[Dst] = AMDGPU::PS; 100 continue; 101 } 102 if (BI->getOpcode() == AMDGPU::DOT4_r600 || 103 BI->getOpcode() == AMDGPU::DOT4_eg) { 104 Result[Dst] = AMDGPU::PV_X; 105 continue; 106 } 107 if (Dst == AMDGPU::OQAP) { 108 continue; 109 } 110 unsigned PVReg = 0; 111 switch (TRI.getHWRegChan(Dst)) { 112 case 0: 113 PVReg = AMDGPU::PV_X; 114 break; 115 case 1: 116 PVReg = AMDGPU::PV_Y; 117 break; 118 case 2: 119 PVReg = AMDGPU::PV_Z; 120 break; 121 case 3: 122 PVReg = AMDGPU::PV_W; 123 break; 124 default: 125 llvm_unreachable("Invalid Chan"); 126 } 127 Result[Dst] = PVReg; 128 } while ((++BI)->isBundledWithPred()); 129 return Result; 130 } 131 132 void substitutePV(MachineInstr *MI, const DenseMap<unsigned, unsigned> &PVs) 133 const { 134 unsigned Ops[] = { 135 AMDGPU::OpName::src0, 136 AMDGPU::OpName::src1, 137 AMDGPU::OpName::src2 138 }; 139 for (unsigned i = 0; i < 3; i++) { 140 int OperandIdx = TII->getOperandIdx(MI->getOpcode(), Ops[i]); 141 if (OperandIdx < 0) 142 continue; 143 unsigned Src = MI->getOperand(OperandIdx).getReg(); 144 const DenseMap<unsigned, unsigned>::const_iterator It = PVs.find(Src); 145 if (It != PVs.end()) 146 MI->getOperand(OperandIdx).setReg(It->second); 147 } 148 } 149public: 150 // Ctor. 151 R600PacketizerList(MachineFunction &MF, MachineLoopInfo &MLI) 152 : VLIWPacketizerList(MF, MLI, true), 153 TII(static_cast<const R600InstrInfo *>( 154 MF.getSubtarget().getInstrInfo())), 155 TRI(TII->getRegisterInfo()) { 156 VLIW5 = !MF.getSubtarget<AMDGPUSubtarget>().hasCaymanISA(); 157 } 158 159 // initPacketizerState - initialize some internal flags. 160 void initPacketizerState() override { 161 ConsideredInstUsesAlreadyWrittenVectorElement = false; 162 } 163 164 // ignorePseudoInstruction - Ignore bundling of pseudo instructions. 165 bool ignorePseudoInstruction(MachineInstr *MI, 166 MachineBasicBlock *MBB) override { 167 return false; 168 } 169 170 // isSoloInstruction - return true if instruction MI can not be packetized 171 // with any other instruction, which means that MI itself is a packet. 172 bool isSoloInstruction(MachineInstr *MI) override { 173 if (TII->isVector(*MI)) 174 return true; 175 if (!TII->isALUInstr(MI->getOpcode())) 176 return true; 177 if (MI->getOpcode() == AMDGPU::GROUP_BARRIER) 178 return true; 179 // XXX: This can be removed once the packetizer properly handles all the 180 // LDS instruction group restrictions. 181 if (TII->isLDSInstr(MI->getOpcode())) 182 return true; 183 return false; 184 } 185 186 // isLegalToPacketizeTogether - Is it legal to packetize SUI and SUJ 187 // together. 188 bool isLegalToPacketizeTogether(SUnit *SUI, SUnit *SUJ) override { 189 MachineInstr *MII = SUI->getInstr(), *MIJ = SUJ->getInstr(); 190 if (getSlot(MII) == getSlot(MIJ)) 191 ConsideredInstUsesAlreadyWrittenVectorElement = true; 192 // Does MII and MIJ share the same pred_sel ? 193 int OpI = TII->getOperandIdx(MII->getOpcode(), AMDGPU::OpName::pred_sel), 194 OpJ = TII->getOperandIdx(MIJ->getOpcode(), AMDGPU::OpName::pred_sel); 195 unsigned PredI = (OpI > -1)?MII->getOperand(OpI).getReg():0, 196 PredJ = (OpJ > -1)?MIJ->getOperand(OpJ).getReg():0; 197 if (PredI != PredJ) 198 return false; 199 if (SUJ->isSucc(SUI)) { 200 for (unsigned i = 0, e = SUJ->Succs.size(); i < e; ++i) { 201 const SDep &Dep = SUJ->Succs[i]; 202 if (Dep.getSUnit() != SUI) 203 continue; 204 if (Dep.getKind() == SDep::Anti) 205 continue; 206 if (Dep.getKind() == SDep::Output) 207 if (MII->getOperand(0).getReg() != MIJ->getOperand(0).getReg()) 208 continue; 209 return false; 210 } 211 } 212 213 bool ARDef = TII->definesAddressRegister(MII) || 214 TII->definesAddressRegister(MIJ); 215 bool ARUse = TII->usesAddressRegister(MII) || 216 TII->usesAddressRegister(MIJ); 217 if (ARDef && ARUse) 218 return false; 219 220 return true; 221 } 222 223 // isLegalToPruneDependencies - Is it legal to prune dependece between SUI 224 // and SUJ. 225 bool isLegalToPruneDependencies(SUnit *SUI, SUnit *SUJ) override { 226 return false; 227 } 228 229 void setIsLastBit(MachineInstr *MI, unsigned Bit) const { 230 unsigned LastOp = TII->getOperandIdx(MI->getOpcode(), AMDGPU::OpName::last); 231 MI->getOperand(LastOp).setImm(Bit); 232 } 233 234 bool isBundlableWithCurrentPMI(MachineInstr *MI, 235 const DenseMap<unsigned, unsigned> &PV, 236 std::vector<R600InstrInfo::BankSwizzle> &BS, 237 bool &isTransSlot) { 238 isTransSlot = TII->isTransOnly(MI); 239 assert (!isTransSlot || VLIW5); 240 241 // Is the dst reg sequence legal ? 242 if (!isTransSlot && !CurrentPacketMIs.empty()) { 243 if (getSlot(MI) <= getSlot(CurrentPacketMIs.back())) { 244 if (ConsideredInstUsesAlreadyWrittenVectorElement && 245 !TII->isVectorOnly(MI) && VLIW5) { 246 isTransSlot = true; 247 DEBUG(dbgs() << "Considering as Trans Inst :"; MI->dump();); 248 } 249 else 250 return false; 251 } 252 } 253 254 // Are the Constants limitations met ? 255 CurrentPacketMIs.push_back(MI); 256 if (!TII->fitsConstReadLimitations(CurrentPacketMIs)) { 257 DEBUG( 258 dbgs() << "Couldn't pack :\n"; 259 MI->dump(); 260 dbgs() << "with the following packets :\n"; 261 for (unsigned i = 0, e = CurrentPacketMIs.size() - 1; i < e; i++) { 262 CurrentPacketMIs[i]->dump(); 263 dbgs() << "\n"; 264 } 265 dbgs() << "because of Consts read limitations\n"; 266 ); 267 CurrentPacketMIs.pop_back(); 268 return false; 269 } 270 271 // Is there a BankSwizzle set that meet Read Port limitations ? 272 if (!TII->fitsReadPortLimitations(CurrentPacketMIs, 273 PV, BS, isTransSlot)) { 274 DEBUG( 275 dbgs() << "Couldn't pack :\n"; 276 MI->dump(); 277 dbgs() << "with the following packets :\n"; 278 for (unsigned i = 0, e = CurrentPacketMIs.size() - 1; i < e; i++) { 279 CurrentPacketMIs[i]->dump(); 280 dbgs() << "\n"; 281 } 282 dbgs() << "because of Read port limitations\n"; 283 ); 284 CurrentPacketMIs.pop_back(); 285 return false; 286 } 287 288 // We cannot read LDS source registrs from the Trans slot. 289 if (isTransSlot && TII->readsLDSSrcReg(MI)) 290 return false; 291 292 CurrentPacketMIs.pop_back(); 293 return true; 294 } 295 296 MachineBasicBlock::iterator addToPacket(MachineInstr *MI) override { 297 MachineBasicBlock::iterator FirstInBundle = 298 CurrentPacketMIs.empty() ? MI : CurrentPacketMIs.front(); 299 const DenseMap<unsigned, unsigned> &PV = 300 getPreviousVector(FirstInBundle); 301 std::vector<R600InstrInfo::BankSwizzle> BS; 302 bool isTransSlot; 303 304 if (isBundlableWithCurrentPMI(MI, PV, BS, isTransSlot)) { 305 for (unsigned i = 0, e = CurrentPacketMIs.size(); i < e; i++) { 306 MachineInstr *MI = CurrentPacketMIs[i]; 307 unsigned Op = TII->getOperandIdx(MI->getOpcode(), 308 AMDGPU::OpName::bank_swizzle); 309 MI->getOperand(Op).setImm(BS[i]); 310 } 311 unsigned Op = TII->getOperandIdx(MI->getOpcode(), 312 AMDGPU::OpName::bank_swizzle); 313 MI->getOperand(Op).setImm(BS.back()); 314 if (!CurrentPacketMIs.empty()) 315 setIsLastBit(CurrentPacketMIs.back(), 0); 316 substitutePV(MI, PV); 317 MachineBasicBlock::iterator It = VLIWPacketizerList::addToPacket(MI); 318 if (isTransSlot) { 319 endPacket(std::next(It)->getParent(), std::next(It)); 320 } 321 return It; 322 } 323 endPacket(MI->getParent(), MI); 324 if (TII->isTransOnly(MI)) 325 return MI; 326 return VLIWPacketizerList::addToPacket(MI); 327 } 328}; 329 330bool R600Packetizer::runOnMachineFunction(MachineFunction &Fn) { 331 const TargetInstrInfo *TII = Fn.getSubtarget().getInstrInfo(); 332 MachineLoopInfo &MLI = getAnalysis<MachineLoopInfo>(); 333 334 // Instantiate the packetizer. 335 R600PacketizerList Packetizer(Fn, MLI); 336 337 // DFA state table should not be empty. 338 assert(Packetizer.getResourceTracker() && "Empty DFA table!"); 339 340 // 341 // Loop over all basic blocks and remove KILL pseudo-instructions 342 // These instructions confuse the dependence analysis. Consider: 343 // D0 = ... (Insn 0) 344 // R0 = KILL R0, D0 (Insn 1) 345 // R0 = ... (Insn 2) 346 // Here, Insn 1 will result in the dependence graph not emitting an output 347 // dependence between Insn 0 and Insn 2. This can lead to incorrect 348 // packetization 349 // 350 for (MachineFunction::iterator MBB = Fn.begin(), MBBe = Fn.end(); 351 MBB != MBBe; ++MBB) { 352 MachineBasicBlock::iterator End = MBB->end(); 353 MachineBasicBlock::iterator MI = MBB->begin(); 354 while (MI != End) { 355 if (MI->isKill() || MI->getOpcode() == AMDGPU::IMPLICIT_DEF || 356 (MI->getOpcode() == AMDGPU::CF_ALU && !MI->getOperand(8).getImm())) { 357 MachineBasicBlock::iterator DeleteMI = MI; 358 ++MI; 359 MBB->erase(DeleteMI); 360 End = MBB->end(); 361 continue; 362 } 363 ++MI; 364 } 365 } 366 367 // Loop over all of the basic blocks. 368 for (MachineFunction::iterator MBB = Fn.begin(), MBBe = Fn.end(); 369 MBB != MBBe; ++MBB) { 370 // Find scheduling regions and schedule / packetize each region. 371 unsigned RemainingCount = MBB->size(); 372 for(MachineBasicBlock::iterator RegionEnd = MBB->end(); 373 RegionEnd != MBB->begin();) { 374 // The next region starts above the previous region. Look backward in the 375 // instruction stream until we find the nearest boundary. 376 MachineBasicBlock::iterator I = RegionEnd; 377 for(;I != MBB->begin(); --I, --RemainingCount) { 378 if (TII->isSchedulingBoundary(std::prev(I), MBB, Fn)) 379 break; 380 } 381 I = MBB->begin(); 382 383 // Skip empty scheduling regions. 384 if (I == RegionEnd) { 385 RegionEnd = std::prev(RegionEnd); 386 --RemainingCount; 387 continue; 388 } 389 // Skip regions with one instruction. 390 if (I == std::prev(RegionEnd)) { 391 RegionEnd = std::prev(RegionEnd); 392 continue; 393 } 394 395 Packetizer.PacketizeMIs(MBB, I, RegionEnd); 396 RegionEnd = I; 397 } 398 } 399 400 return true; 401 402} 403 404} // end anonymous namespace 405 406llvm::FunctionPass *llvm::createR600Packetizer(TargetMachine &tm) { 407 return new R600Packetizer(tm); 408} 409