TailDuplication.cpp revision 034b94b17006f51722886b0f2283fb6fb19aca1f
1//===-- TailDuplication.cpp - Duplicate blocks into predecessors' tails ---===// 2// 3// The LLVM Compiler Infrastructure 4// 5// This file is distributed under the University of Illinois Open Source 6// License. See LICENSE.TXT for details. 7// 8//===----------------------------------------------------------------------===// 9// 10// This pass duplicates basic blocks ending in unconditional branches into 11// the tails of their predecessors. 12// 13//===----------------------------------------------------------------------===// 14 15#define DEBUG_TYPE "tailduplication" 16#include "llvm/CodeGen/Passes.h" 17#include "llvm/ADT/DenseSet.h" 18#include "llvm/ADT/OwningPtr.h" 19#include "llvm/ADT/SetVector.h" 20#include "llvm/ADT/SmallSet.h" 21#include "llvm/ADT/Statistic.h" 22#include "llvm/CodeGen/MachineFunctionPass.h" 23#include "llvm/CodeGen/MachineInstrBuilder.h" 24#include "llvm/CodeGen/MachineModuleInfo.h" 25#include "llvm/CodeGen/MachineRegisterInfo.h" 26#include "llvm/CodeGen/MachineSSAUpdater.h" 27#include "llvm/CodeGen/RegisterScavenging.h" 28#include "llvm/Function.h" 29#include "llvm/Support/CommandLine.h" 30#include "llvm/Support/Debug.h" 31#include "llvm/Support/ErrorHandling.h" 32#include "llvm/Support/raw_ostream.h" 33#include "llvm/Target/TargetInstrInfo.h" 34#include "llvm/Target/TargetRegisterInfo.h" 35using namespace llvm; 36 37STATISTIC(NumTails , "Number of tails duplicated"); 38STATISTIC(NumTailDups , "Number of tail duplicated blocks"); 39STATISTIC(NumInstrDups , "Additional instructions due to tail duplication"); 40STATISTIC(NumDeadBlocks, "Number of dead blocks removed"); 41STATISTIC(NumAddedPHIs , "Number of phis added"); 42 43// Heuristic for tail duplication. 44static cl::opt<unsigned> 45TailDuplicateSize("tail-dup-size", 46 cl::desc("Maximum instructions to consider tail duplicating"), 47 cl::init(2), cl::Hidden); 48 49static cl::opt<bool> 50TailDupVerify("tail-dup-verify", 51 cl::desc("Verify sanity of PHI instructions during taildup"), 52 cl::init(false), cl::Hidden); 53 54static cl::opt<unsigned> 55TailDupLimit("tail-dup-limit", cl::init(~0U), cl::Hidden); 56 57typedef std::vector<std::pair<MachineBasicBlock*,unsigned> > AvailableValsTy; 58 59namespace { 60 /// TailDuplicatePass - Perform tail duplication. 61 class TailDuplicatePass : public MachineFunctionPass { 62 const TargetInstrInfo *TII; 63 const TargetRegisterInfo *TRI; 64 MachineModuleInfo *MMI; 65 MachineRegisterInfo *MRI; 66 OwningPtr<RegScavenger> RS; 67 bool PreRegAlloc; 68 69 // SSAUpdateVRs - A list of virtual registers for which to update SSA form. 70 SmallVector<unsigned, 16> SSAUpdateVRs; 71 72 // SSAUpdateVals - For each virtual register in SSAUpdateVals keep a list of 73 // source virtual registers. 74 DenseMap<unsigned, AvailableValsTy> SSAUpdateVals; 75 76 public: 77 static char ID; 78 explicit TailDuplicatePass() : 79 MachineFunctionPass(ID), PreRegAlloc(false) {} 80 81 virtual bool runOnMachineFunction(MachineFunction &MF); 82 83 private: 84 void AddSSAUpdateEntry(unsigned OrigReg, unsigned NewReg, 85 MachineBasicBlock *BB); 86 void ProcessPHI(MachineInstr *MI, MachineBasicBlock *TailBB, 87 MachineBasicBlock *PredBB, 88 DenseMap<unsigned, unsigned> &LocalVRMap, 89 SmallVector<std::pair<unsigned,unsigned>, 4> &Copies, 90 const DenseSet<unsigned> &UsedByPhi, 91 bool Remove); 92 void DuplicateInstruction(MachineInstr *MI, 93 MachineBasicBlock *TailBB, 94 MachineBasicBlock *PredBB, 95 MachineFunction &MF, 96 DenseMap<unsigned, unsigned> &LocalVRMap, 97 const DenseSet<unsigned> &UsedByPhi); 98 void UpdateSuccessorsPHIs(MachineBasicBlock *FromBB, bool isDead, 99 SmallVector<MachineBasicBlock*, 8> &TDBBs, 100 SmallSetVector<MachineBasicBlock*, 8> &Succs); 101 bool TailDuplicateBlocks(MachineFunction &MF); 102 bool shouldTailDuplicate(const MachineFunction &MF, 103 bool IsSimple, MachineBasicBlock &TailBB); 104 bool isSimpleBB(MachineBasicBlock *TailBB); 105 bool canCompletelyDuplicateBB(MachineBasicBlock &BB); 106 bool duplicateSimpleBB(MachineBasicBlock *TailBB, 107 SmallVector<MachineBasicBlock*, 8> &TDBBs, 108 const DenseSet<unsigned> &RegsUsedByPhi, 109 SmallVector<MachineInstr*, 16> &Copies); 110 bool TailDuplicate(MachineBasicBlock *TailBB, 111 bool IsSimple, 112 MachineFunction &MF, 113 SmallVector<MachineBasicBlock*, 8> &TDBBs, 114 SmallVector<MachineInstr*, 16> &Copies); 115 bool TailDuplicateAndUpdate(MachineBasicBlock *MBB, 116 bool IsSimple, 117 MachineFunction &MF); 118 119 void RemoveDeadBlock(MachineBasicBlock *MBB); 120 }; 121 122 char TailDuplicatePass::ID = 0; 123} 124 125char &llvm::TailDuplicateID = TailDuplicatePass::ID; 126 127INITIALIZE_PASS(TailDuplicatePass, "tailduplication", "Tail Duplication", 128 false, false) 129 130bool TailDuplicatePass::runOnMachineFunction(MachineFunction &MF) { 131 TII = MF.getTarget().getInstrInfo(); 132 TRI = MF.getTarget().getRegisterInfo(); 133 MRI = &MF.getRegInfo(); 134 MMI = getAnalysisIfAvailable<MachineModuleInfo>(); 135 PreRegAlloc = MRI->isSSA(); 136 RS.reset(); 137 if (MRI->tracksLiveness() && TRI->trackLivenessAfterRegAlloc(MF)) 138 RS.reset(new RegScavenger()); 139 140 bool MadeChange = false; 141 while (TailDuplicateBlocks(MF)) 142 MadeChange = true; 143 144 return MadeChange; 145} 146 147static void VerifyPHIs(MachineFunction &MF, bool CheckExtra) { 148 for (MachineFunction::iterator I = ++MF.begin(), E = MF.end(); I != E; ++I) { 149 MachineBasicBlock *MBB = I; 150 SmallSetVector<MachineBasicBlock*, 8> Preds(MBB->pred_begin(), 151 MBB->pred_end()); 152 MachineBasicBlock::iterator MI = MBB->begin(); 153 while (MI != MBB->end()) { 154 if (!MI->isPHI()) 155 break; 156 for (SmallSetVector<MachineBasicBlock *, 8>::iterator PI = Preds.begin(), 157 PE = Preds.end(); PI != PE; ++PI) { 158 MachineBasicBlock *PredBB = *PI; 159 bool Found = false; 160 for (unsigned i = 1, e = MI->getNumOperands(); i != e; i += 2) { 161 MachineBasicBlock *PHIBB = MI->getOperand(i+1).getMBB(); 162 if (PHIBB == PredBB) { 163 Found = true; 164 break; 165 } 166 } 167 if (!Found) { 168 dbgs() << "Malformed PHI in BB#" << MBB->getNumber() << ": " << *MI; 169 dbgs() << " missing input from predecessor BB#" 170 << PredBB->getNumber() << '\n'; 171 llvm_unreachable(0); 172 } 173 } 174 175 for (unsigned i = 1, e = MI->getNumOperands(); i != e; i += 2) { 176 MachineBasicBlock *PHIBB = MI->getOperand(i+1).getMBB(); 177 if (CheckExtra && !Preds.count(PHIBB)) { 178 dbgs() << "Warning: malformed PHI in BB#" << MBB->getNumber() 179 << ": " << *MI; 180 dbgs() << " extra input from predecessor BB#" 181 << PHIBB->getNumber() << '\n'; 182 llvm_unreachable(0); 183 } 184 if (PHIBB->getNumber() < 0) { 185 dbgs() << "Malformed PHI in BB#" << MBB->getNumber() << ": " << *MI; 186 dbgs() << " non-existing BB#" << PHIBB->getNumber() << '\n'; 187 llvm_unreachable(0); 188 } 189 } 190 ++MI; 191 } 192 } 193} 194 195/// TailDuplicateAndUpdate - Tail duplicate the block and cleanup. 196bool 197TailDuplicatePass::TailDuplicateAndUpdate(MachineBasicBlock *MBB, 198 bool IsSimple, 199 MachineFunction &MF) { 200 // Save the successors list. 201 SmallSetVector<MachineBasicBlock*, 8> Succs(MBB->succ_begin(), 202 MBB->succ_end()); 203 204 SmallVector<MachineBasicBlock*, 8> TDBBs; 205 SmallVector<MachineInstr*, 16> Copies; 206 if (!TailDuplicate(MBB, IsSimple, MF, TDBBs, Copies)) 207 return false; 208 209 ++NumTails; 210 211 SmallVector<MachineInstr*, 8> NewPHIs; 212 MachineSSAUpdater SSAUpdate(MF, &NewPHIs); 213 214 // TailBB's immediate successors are now successors of those predecessors 215 // which duplicated TailBB. Add the predecessors as sources to the PHI 216 // instructions. 217 bool isDead = MBB->pred_empty() && !MBB->hasAddressTaken(); 218 if (PreRegAlloc) 219 UpdateSuccessorsPHIs(MBB, isDead, TDBBs, Succs); 220 221 // If it is dead, remove it. 222 if (isDead) { 223 NumInstrDups -= MBB->size(); 224 RemoveDeadBlock(MBB); 225 ++NumDeadBlocks; 226 } 227 228 // Update SSA form. 229 if (!SSAUpdateVRs.empty()) { 230 for (unsigned i = 0, e = SSAUpdateVRs.size(); i != e; ++i) { 231 unsigned VReg = SSAUpdateVRs[i]; 232 SSAUpdate.Initialize(VReg); 233 234 // If the original definition is still around, add it as an available 235 // value. 236 MachineInstr *DefMI = MRI->getVRegDef(VReg); 237 MachineBasicBlock *DefBB = 0; 238 if (DefMI) { 239 DefBB = DefMI->getParent(); 240 SSAUpdate.AddAvailableValue(DefBB, VReg); 241 } 242 243 // Add the new vregs as available values. 244 DenseMap<unsigned, AvailableValsTy>::iterator LI = 245 SSAUpdateVals.find(VReg); 246 for (unsigned j = 0, ee = LI->second.size(); j != ee; ++j) { 247 MachineBasicBlock *SrcBB = LI->second[j].first; 248 unsigned SrcReg = LI->second[j].second; 249 SSAUpdate.AddAvailableValue(SrcBB, SrcReg); 250 } 251 252 // Rewrite uses that are outside of the original def's block. 253 MachineRegisterInfo::use_iterator UI = MRI->use_begin(VReg); 254 while (UI != MRI->use_end()) { 255 MachineOperand &UseMO = UI.getOperand(); 256 MachineInstr *UseMI = &*UI; 257 ++UI; 258 if (UseMI->isDebugValue()) { 259 // SSAUpdate can replace the use with an undef. That creates 260 // a debug instruction that is a kill. 261 // FIXME: Should it SSAUpdate job to delete debug instructions 262 // instead of replacing the use with undef? 263 UseMI->eraseFromParent(); 264 continue; 265 } 266 if (UseMI->getParent() == DefBB && !UseMI->isPHI()) 267 continue; 268 SSAUpdate.RewriteUse(UseMO); 269 } 270 } 271 272 SSAUpdateVRs.clear(); 273 SSAUpdateVals.clear(); 274 } 275 276 // Eliminate some of the copies inserted by tail duplication to maintain 277 // SSA form. 278 for (unsigned i = 0, e = Copies.size(); i != e; ++i) { 279 MachineInstr *Copy = Copies[i]; 280 if (!Copy->isCopy()) 281 continue; 282 unsigned Dst = Copy->getOperand(0).getReg(); 283 unsigned Src = Copy->getOperand(1).getReg(); 284 if (MRI->hasOneNonDBGUse(Src) && 285 MRI->constrainRegClass(Src, MRI->getRegClass(Dst))) { 286 // Copy is the only use. Do trivial copy propagation here. 287 MRI->replaceRegWith(Dst, Src); 288 Copy->eraseFromParent(); 289 } 290 } 291 292 if (NewPHIs.size()) 293 NumAddedPHIs += NewPHIs.size(); 294 295 return true; 296} 297 298/// TailDuplicateBlocks - Look for small blocks that are unconditionally 299/// branched to and do not fall through. Tail-duplicate their instructions 300/// into their predecessors to eliminate (dynamic) branches. 301bool TailDuplicatePass::TailDuplicateBlocks(MachineFunction &MF) { 302 bool MadeChange = false; 303 304 if (PreRegAlloc && TailDupVerify) { 305 DEBUG(dbgs() << "\n*** Before tail-duplicating\n"); 306 VerifyPHIs(MF, true); 307 } 308 309 for (MachineFunction::iterator I = ++MF.begin(), E = MF.end(); I != E; ) { 310 MachineBasicBlock *MBB = I++; 311 312 if (NumTails == TailDupLimit) 313 break; 314 315 bool IsSimple = isSimpleBB(MBB); 316 317 if (!shouldTailDuplicate(MF, IsSimple, *MBB)) 318 continue; 319 320 MadeChange |= TailDuplicateAndUpdate(MBB, IsSimple, MF); 321 } 322 323 if (PreRegAlloc && TailDupVerify) 324 VerifyPHIs(MF, false); 325 326 return MadeChange; 327} 328 329static bool isDefLiveOut(unsigned Reg, MachineBasicBlock *BB, 330 const MachineRegisterInfo *MRI) { 331 for (MachineRegisterInfo::use_iterator UI = MRI->use_begin(Reg), 332 UE = MRI->use_end(); UI != UE; ++UI) { 333 MachineInstr *UseMI = &*UI; 334 if (UseMI->isDebugValue()) 335 continue; 336 if (UseMI->getParent() != BB) 337 return true; 338 } 339 return false; 340} 341 342static unsigned getPHISrcRegOpIdx(MachineInstr *MI, MachineBasicBlock *SrcBB) { 343 for (unsigned i = 1, e = MI->getNumOperands(); i != e; i += 2) 344 if (MI->getOperand(i+1).getMBB() == SrcBB) 345 return i; 346 return 0; 347} 348 349 350// Remember which registers are used by phis in this block. This is 351// used to determine which registers are liveout while modifying the 352// block (which is why we need to copy the information). 353static void getRegsUsedByPHIs(const MachineBasicBlock &BB, 354 DenseSet<unsigned> *UsedByPhi) { 355 for(MachineBasicBlock::const_iterator I = BB.begin(), E = BB.end(); 356 I != E; ++I) { 357 const MachineInstr &MI = *I; 358 if (!MI.isPHI()) 359 break; 360 for (unsigned i = 1, e = MI.getNumOperands(); i != e; i += 2) { 361 unsigned SrcReg = MI.getOperand(i).getReg(); 362 UsedByPhi->insert(SrcReg); 363 } 364 } 365} 366 367/// AddSSAUpdateEntry - Add a definition and source virtual registers pair for 368/// SSA update. 369void TailDuplicatePass::AddSSAUpdateEntry(unsigned OrigReg, unsigned NewReg, 370 MachineBasicBlock *BB) { 371 DenseMap<unsigned, AvailableValsTy>::iterator LI= SSAUpdateVals.find(OrigReg); 372 if (LI != SSAUpdateVals.end()) 373 LI->second.push_back(std::make_pair(BB, NewReg)); 374 else { 375 AvailableValsTy Vals; 376 Vals.push_back(std::make_pair(BB, NewReg)); 377 SSAUpdateVals.insert(std::make_pair(OrigReg, Vals)); 378 SSAUpdateVRs.push_back(OrigReg); 379 } 380} 381 382/// ProcessPHI - Process PHI node in TailBB by turning it into a copy in PredBB. 383/// Remember the source register that's contributed by PredBB and update SSA 384/// update map. 385void TailDuplicatePass::ProcessPHI(MachineInstr *MI, 386 MachineBasicBlock *TailBB, 387 MachineBasicBlock *PredBB, 388 DenseMap<unsigned, unsigned> &LocalVRMap, 389 SmallVector<std::pair<unsigned,unsigned>, 4> &Copies, 390 const DenseSet<unsigned> &RegsUsedByPhi, 391 bool Remove) { 392 unsigned DefReg = MI->getOperand(0).getReg(); 393 unsigned SrcOpIdx = getPHISrcRegOpIdx(MI, PredBB); 394 assert(SrcOpIdx && "Unable to find matching PHI source?"); 395 unsigned SrcReg = MI->getOperand(SrcOpIdx).getReg(); 396 const TargetRegisterClass *RC = MRI->getRegClass(DefReg); 397 LocalVRMap.insert(std::make_pair(DefReg, SrcReg)); 398 399 // Insert a copy from source to the end of the block. The def register is the 400 // available value liveout of the block. 401 unsigned NewDef = MRI->createVirtualRegister(RC); 402 Copies.push_back(std::make_pair(NewDef, SrcReg)); 403 if (isDefLiveOut(DefReg, TailBB, MRI) || RegsUsedByPhi.count(DefReg)) 404 AddSSAUpdateEntry(DefReg, NewDef, PredBB); 405 406 if (!Remove) 407 return; 408 409 // Remove PredBB from the PHI node. 410 MI->RemoveOperand(SrcOpIdx+1); 411 MI->RemoveOperand(SrcOpIdx); 412 if (MI->getNumOperands() == 1) 413 MI->eraseFromParent(); 414} 415 416/// DuplicateInstruction - Duplicate a TailBB instruction to PredBB and update 417/// the source operands due to earlier PHI translation. 418void TailDuplicatePass::DuplicateInstruction(MachineInstr *MI, 419 MachineBasicBlock *TailBB, 420 MachineBasicBlock *PredBB, 421 MachineFunction &MF, 422 DenseMap<unsigned, unsigned> &LocalVRMap, 423 const DenseSet<unsigned> &UsedByPhi) { 424 MachineInstr *NewMI = TII->duplicate(MI, MF); 425 for (unsigned i = 0, e = NewMI->getNumOperands(); i != e; ++i) { 426 MachineOperand &MO = NewMI->getOperand(i); 427 if (!MO.isReg()) 428 continue; 429 unsigned Reg = MO.getReg(); 430 if (!TargetRegisterInfo::isVirtualRegister(Reg)) 431 continue; 432 if (MO.isDef()) { 433 const TargetRegisterClass *RC = MRI->getRegClass(Reg); 434 unsigned NewReg = MRI->createVirtualRegister(RC); 435 MO.setReg(NewReg); 436 LocalVRMap.insert(std::make_pair(Reg, NewReg)); 437 if (isDefLiveOut(Reg, TailBB, MRI) || UsedByPhi.count(Reg)) 438 AddSSAUpdateEntry(Reg, NewReg, PredBB); 439 } else { 440 DenseMap<unsigned, unsigned>::iterator VI = LocalVRMap.find(Reg); 441 if (VI != LocalVRMap.end()) { 442 MO.setReg(VI->second); 443 MRI->constrainRegClass(VI->second, MRI->getRegClass(Reg)); 444 } 445 } 446 } 447 PredBB->insert(PredBB->instr_end(), NewMI); 448} 449 450/// UpdateSuccessorsPHIs - After FromBB is tail duplicated into its predecessor 451/// blocks, the successors have gained new predecessors. Update the PHI 452/// instructions in them accordingly. 453void 454TailDuplicatePass::UpdateSuccessorsPHIs(MachineBasicBlock *FromBB, bool isDead, 455 SmallVector<MachineBasicBlock*, 8> &TDBBs, 456 SmallSetVector<MachineBasicBlock*,8> &Succs) { 457 for (SmallSetVector<MachineBasicBlock*, 8>::iterator SI = Succs.begin(), 458 SE = Succs.end(); SI != SE; ++SI) { 459 MachineBasicBlock *SuccBB = *SI; 460 for (MachineBasicBlock::iterator II = SuccBB->begin(), EE = SuccBB->end(); 461 II != EE; ++II) { 462 if (!II->isPHI()) 463 break; 464 unsigned Idx = 0; 465 for (unsigned i = 1, e = II->getNumOperands(); i != e; i += 2) { 466 MachineOperand &MO = II->getOperand(i+1); 467 if (MO.getMBB() == FromBB) { 468 Idx = i; 469 break; 470 } 471 } 472 473 assert(Idx != 0); 474 MachineOperand &MO0 = II->getOperand(Idx); 475 unsigned Reg = MO0.getReg(); 476 if (isDead) { 477 // Folded into the previous BB. 478 // There could be duplicate phi source entries. FIXME: Should sdisel 479 // or earlier pass fixed this? 480 for (unsigned i = II->getNumOperands()-2; i != Idx; i -= 2) { 481 MachineOperand &MO = II->getOperand(i+1); 482 if (MO.getMBB() == FromBB) { 483 II->RemoveOperand(i+1); 484 II->RemoveOperand(i); 485 } 486 } 487 } else 488 Idx = 0; 489 490 // If Idx is set, the operands at Idx and Idx+1 must be removed. 491 // We reuse the location to avoid expensive RemoveOperand calls. 492 493 DenseMap<unsigned,AvailableValsTy>::iterator LI=SSAUpdateVals.find(Reg); 494 if (LI != SSAUpdateVals.end()) { 495 // This register is defined in the tail block. 496 for (unsigned j = 0, ee = LI->second.size(); j != ee; ++j) { 497 MachineBasicBlock *SrcBB = LI->second[j].first; 498 // If we didn't duplicate a bb into a particular predecessor, we 499 // might still have added an entry to SSAUpdateVals to correcly 500 // recompute SSA. If that case, avoid adding a dummy extra argument 501 // this PHI. 502 if (!SrcBB->isSuccessor(SuccBB)) 503 continue; 504 505 unsigned SrcReg = LI->second[j].second; 506 if (Idx != 0) { 507 II->getOperand(Idx).setReg(SrcReg); 508 II->getOperand(Idx+1).setMBB(SrcBB); 509 Idx = 0; 510 } else { 511 II->addOperand(MachineOperand::CreateReg(SrcReg, false)); 512 II->addOperand(MachineOperand::CreateMBB(SrcBB)); 513 } 514 } 515 } else { 516 // Live in tail block, must also be live in predecessors. 517 for (unsigned j = 0, ee = TDBBs.size(); j != ee; ++j) { 518 MachineBasicBlock *SrcBB = TDBBs[j]; 519 if (Idx != 0) { 520 II->getOperand(Idx).setReg(Reg); 521 II->getOperand(Idx+1).setMBB(SrcBB); 522 Idx = 0; 523 } else { 524 II->addOperand(MachineOperand::CreateReg(Reg, false)); 525 II->addOperand(MachineOperand::CreateMBB(SrcBB)); 526 } 527 } 528 } 529 if (Idx != 0) { 530 II->RemoveOperand(Idx+1); 531 II->RemoveOperand(Idx); 532 } 533 } 534 } 535} 536 537/// shouldTailDuplicate - Determine if it is profitable to duplicate this block. 538bool 539TailDuplicatePass::shouldTailDuplicate(const MachineFunction &MF, 540 bool IsSimple, 541 MachineBasicBlock &TailBB) { 542 // Only duplicate blocks that end with unconditional branches. 543 if (TailBB.canFallThrough()) 544 return false; 545 546 // Don't try to tail-duplicate single-block loops. 547 if (TailBB.isSuccessor(&TailBB)) 548 return false; 549 550 // Set the limit on the cost to duplicate. When optimizing for size, 551 // duplicate only one, because one branch instruction can be eliminated to 552 // compensate for the duplication. 553 unsigned MaxDuplicateCount; 554 if (TailDuplicateSize.getNumOccurrences() == 0 && 555 MF.getFunction()->getFnAttributes(). 556 hasAttribute(Attribute::OptimizeForSize)) 557 MaxDuplicateCount = 1; 558 else 559 MaxDuplicateCount = TailDuplicateSize; 560 561 // If the target has hardware branch prediction that can handle indirect 562 // branches, duplicating them can often make them predictable when there 563 // are common paths through the code. The limit needs to be high enough 564 // to allow undoing the effects of tail merging and other optimizations 565 // that rearrange the predecessors of the indirect branch. 566 567 bool HasIndirectbr = false; 568 if (!TailBB.empty()) 569 HasIndirectbr = TailBB.back().isIndirectBranch(); 570 571 if (HasIndirectbr && PreRegAlloc) 572 MaxDuplicateCount = 20; 573 574 // Check the instructions in the block to determine whether tail-duplication 575 // is invalid or unlikely to be profitable. 576 unsigned InstrCount = 0; 577 for (MachineBasicBlock::iterator I = TailBB.begin(); I != TailBB.end(); ++I) { 578 // Non-duplicable things shouldn't be tail-duplicated. 579 if (I->isNotDuplicable()) 580 return false; 581 582 // Do not duplicate 'return' instructions if this is a pre-regalloc run. 583 // A return may expand into a lot more instructions (e.g. reload of callee 584 // saved registers) after PEI. 585 if (PreRegAlloc && I->isReturn()) 586 return false; 587 588 // Avoid duplicating calls before register allocation. Calls presents a 589 // barrier to register allocation so duplicating them may end up increasing 590 // spills. 591 if (PreRegAlloc && I->isCall()) 592 return false; 593 594 if (!I->isPHI() && !I->isDebugValue()) 595 InstrCount += 1; 596 597 if (InstrCount > MaxDuplicateCount) 598 return false; 599 } 600 601 if (HasIndirectbr && PreRegAlloc) 602 return true; 603 604 if (IsSimple) 605 return true; 606 607 if (!PreRegAlloc) 608 return true; 609 610 return canCompletelyDuplicateBB(TailBB); 611} 612 613/// isSimpleBB - True if this BB has only one unconditional jump. 614bool 615TailDuplicatePass::isSimpleBB(MachineBasicBlock *TailBB) { 616 if (TailBB->succ_size() != 1) 617 return false; 618 if (TailBB->pred_empty()) 619 return false; 620 MachineBasicBlock::iterator I = TailBB->begin(); 621 MachineBasicBlock::iterator E = TailBB->end(); 622 while (I != E && I->isDebugValue()) 623 ++I; 624 if (I == E) 625 return true; 626 return I->isUnconditionalBranch(); 627} 628 629static bool 630bothUsedInPHI(const MachineBasicBlock &A, 631 SmallPtrSet<MachineBasicBlock*, 8> SuccsB) { 632 for (MachineBasicBlock::const_succ_iterator SI = A.succ_begin(), 633 SE = A.succ_end(); SI != SE; ++SI) { 634 MachineBasicBlock *BB = *SI; 635 if (SuccsB.count(BB) && !BB->empty() && BB->begin()->isPHI()) 636 return true; 637 } 638 639 return false; 640} 641 642bool 643TailDuplicatePass::canCompletelyDuplicateBB(MachineBasicBlock &BB) { 644 SmallPtrSet<MachineBasicBlock*, 8> Succs(BB.succ_begin(), BB.succ_end()); 645 646 for (MachineBasicBlock::pred_iterator PI = BB.pred_begin(), 647 PE = BB.pred_end(); PI != PE; ++PI) { 648 MachineBasicBlock *PredBB = *PI; 649 650 if (PredBB->succ_size() > 1) 651 return false; 652 653 MachineBasicBlock *PredTBB = NULL, *PredFBB = NULL; 654 SmallVector<MachineOperand, 4> PredCond; 655 if (TII->AnalyzeBranch(*PredBB, PredTBB, PredFBB, PredCond, true)) 656 return false; 657 658 if (!PredCond.empty()) 659 return false; 660 } 661 return true; 662} 663 664bool 665TailDuplicatePass::duplicateSimpleBB(MachineBasicBlock *TailBB, 666 SmallVector<MachineBasicBlock*, 8> &TDBBs, 667 const DenseSet<unsigned> &UsedByPhi, 668 SmallVector<MachineInstr*, 16> &Copies) { 669 SmallPtrSet<MachineBasicBlock*, 8> Succs(TailBB->succ_begin(), 670 TailBB->succ_end()); 671 SmallVector<MachineBasicBlock*, 8> Preds(TailBB->pred_begin(), 672 TailBB->pred_end()); 673 bool Changed = false; 674 for (SmallSetVector<MachineBasicBlock *, 8>::iterator PI = Preds.begin(), 675 PE = Preds.end(); PI != PE; ++PI) { 676 MachineBasicBlock *PredBB = *PI; 677 678 if (PredBB->getLandingPadSuccessor()) 679 continue; 680 681 if (bothUsedInPHI(*PredBB, Succs)) 682 continue; 683 684 MachineBasicBlock *PredTBB = NULL, *PredFBB = NULL; 685 SmallVector<MachineOperand, 4> PredCond; 686 if (TII->AnalyzeBranch(*PredBB, PredTBB, PredFBB, PredCond, true)) 687 continue; 688 689 Changed = true; 690 DEBUG(dbgs() << "\nTail-duplicating into PredBB: " << *PredBB 691 << "From simple Succ: " << *TailBB); 692 693 MachineBasicBlock *NewTarget = *TailBB->succ_begin(); 694 MachineBasicBlock *NextBB = llvm::next(MachineFunction::iterator(PredBB)); 695 696 // Make PredFBB explicit. 697 if (PredCond.empty()) 698 PredFBB = PredTBB; 699 700 // Make fall through explicit. 701 if (!PredTBB) 702 PredTBB = NextBB; 703 if (!PredFBB) 704 PredFBB = NextBB; 705 706 // Redirect 707 if (PredFBB == TailBB) 708 PredFBB = NewTarget; 709 if (PredTBB == TailBB) 710 PredTBB = NewTarget; 711 712 // Make the branch unconditional if possible 713 if (PredTBB == PredFBB) { 714 PredCond.clear(); 715 PredFBB = NULL; 716 } 717 718 // Avoid adding fall through branches. 719 if (PredFBB == NextBB) 720 PredFBB = NULL; 721 if (PredTBB == NextBB && PredFBB == NULL) 722 PredTBB = NULL; 723 724 TII->RemoveBranch(*PredBB); 725 726 if (PredTBB) 727 TII->InsertBranch(*PredBB, PredTBB, PredFBB, PredCond, DebugLoc()); 728 729 PredBB->removeSuccessor(TailBB); 730 unsigned NumSuccessors = PredBB->succ_size(); 731 assert(NumSuccessors <= 1); 732 if (NumSuccessors == 0 || *PredBB->succ_begin() != NewTarget) 733 PredBB->addSuccessor(NewTarget); 734 735 TDBBs.push_back(PredBB); 736 } 737 return Changed; 738} 739 740/// TailDuplicate - If it is profitable, duplicate TailBB's contents in each 741/// of its predecessors. 742bool 743TailDuplicatePass::TailDuplicate(MachineBasicBlock *TailBB, 744 bool IsSimple, 745 MachineFunction &MF, 746 SmallVector<MachineBasicBlock*, 8> &TDBBs, 747 SmallVector<MachineInstr*, 16> &Copies) { 748 DEBUG(dbgs() << "\n*** Tail-duplicating BB#" << TailBB->getNumber() << '\n'); 749 750 DenseSet<unsigned> UsedByPhi; 751 getRegsUsedByPHIs(*TailBB, &UsedByPhi); 752 753 if (IsSimple) 754 return duplicateSimpleBB(TailBB, TDBBs, UsedByPhi, Copies); 755 756 // Iterate through all the unique predecessors and tail-duplicate this 757 // block into them, if possible. Copying the list ahead of time also 758 // avoids trouble with the predecessor list reallocating. 759 bool Changed = false; 760 SmallSetVector<MachineBasicBlock*, 8> Preds(TailBB->pred_begin(), 761 TailBB->pred_end()); 762 for (SmallSetVector<MachineBasicBlock *, 8>::iterator PI = Preds.begin(), 763 PE = Preds.end(); PI != PE; ++PI) { 764 MachineBasicBlock *PredBB = *PI; 765 766 assert(TailBB != PredBB && 767 "Single-block loop should have been rejected earlier!"); 768 // EH edges are ignored by AnalyzeBranch. 769 if (PredBB->succ_size() > 1) 770 continue; 771 772 MachineBasicBlock *PredTBB, *PredFBB; 773 SmallVector<MachineOperand, 4> PredCond; 774 if (TII->AnalyzeBranch(*PredBB, PredTBB, PredFBB, PredCond, true)) 775 continue; 776 if (!PredCond.empty()) 777 continue; 778 // Don't duplicate into a fall-through predecessor (at least for now). 779 if (PredBB->isLayoutSuccessor(TailBB) && PredBB->canFallThrough()) 780 continue; 781 782 DEBUG(dbgs() << "\nTail-duplicating into PredBB: " << *PredBB 783 << "From Succ: " << *TailBB); 784 785 TDBBs.push_back(PredBB); 786 787 // Remove PredBB's unconditional branch. 788 TII->RemoveBranch(*PredBB); 789 790 if (RS && !TailBB->livein_empty()) { 791 // Update PredBB livein. 792 RS->enterBasicBlock(PredBB); 793 if (!PredBB->empty()) 794 RS->forward(prior(PredBB->end())); 795 BitVector RegsLiveAtExit(TRI->getNumRegs()); 796 RS->getRegsUsed(RegsLiveAtExit, false); 797 for (MachineBasicBlock::livein_iterator I = TailBB->livein_begin(), 798 E = TailBB->livein_end(); I != E; ++I) { 799 if (!RegsLiveAtExit[*I]) 800 // If a register is previously livein to the tail but it's not live 801 // at the end of predecessor BB, then it should be added to its 802 // livein list. 803 PredBB->addLiveIn(*I); 804 } 805 } 806 807 // Clone the contents of TailBB into PredBB. 808 DenseMap<unsigned, unsigned> LocalVRMap; 809 SmallVector<std::pair<unsigned,unsigned>, 4> CopyInfos; 810 // Use instr_iterator here to properly handle bundles, e.g. 811 // ARM Thumb2 IT block. 812 MachineBasicBlock::instr_iterator I = TailBB->instr_begin(); 813 while (I != TailBB->instr_end()) { 814 MachineInstr *MI = &*I; 815 ++I; 816 if (MI->isPHI()) { 817 // Replace the uses of the def of the PHI with the register coming 818 // from PredBB. 819 ProcessPHI(MI, TailBB, PredBB, LocalVRMap, CopyInfos, UsedByPhi, true); 820 } else { 821 // Replace def of virtual registers with new registers, and update 822 // uses with PHI source register or the new registers. 823 DuplicateInstruction(MI, TailBB, PredBB, MF, LocalVRMap, UsedByPhi); 824 } 825 } 826 MachineBasicBlock::iterator Loc = PredBB->getFirstTerminator(); 827 for (unsigned i = 0, e = CopyInfos.size(); i != e; ++i) { 828 Copies.push_back(BuildMI(*PredBB, Loc, DebugLoc(), 829 TII->get(TargetOpcode::COPY), 830 CopyInfos[i].first).addReg(CopyInfos[i].second)); 831 } 832 833 // Simplify 834 TII->AnalyzeBranch(*PredBB, PredTBB, PredFBB, PredCond, true); 835 836 NumInstrDups += TailBB->size() - 1; // subtract one for removed branch 837 838 // Update the CFG. 839 PredBB->removeSuccessor(PredBB->succ_begin()); 840 assert(PredBB->succ_empty() && 841 "TailDuplicate called on block with multiple successors!"); 842 for (MachineBasicBlock::succ_iterator I = TailBB->succ_begin(), 843 E = TailBB->succ_end(); I != E; ++I) 844 PredBB->addSuccessor(*I); 845 846 Changed = true; 847 ++NumTailDups; 848 } 849 850 // If TailBB was duplicated into all its predecessors except for the prior 851 // block, which falls through unconditionally, move the contents of this 852 // block into the prior block. 853 MachineBasicBlock *PrevBB = prior(MachineFunction::iterator(TailBB)); 854 MachineBasicBlock *PriorTBB = 0, *PriorFBB = 0; 855 SmallVector<MachineOperand, 4> PriorCond; 856 // This has to check PrevBB->succ_size() because EH edges are ignored by 857 // AnalyzeBranch. 858 if (PrevBB->succ_size() == 1 && 859 !TII->AnalyzeBranch(*PrevBB, PriorTBB, PriorFBB, PriorCond, true) && 860 PriorCond.empty() && !PriorTBB && TailBB->pred_size() == 1 && 861 !TailBB->hasAddressTaken()) { 862 DEBUG(dbgs() << "\nMerging into block: " << *PrevBB 863 << "From MBB: " << *TailBB); 864 if (PreRegAlloc) { 865 DenseMap<unsigned, unsigned> LocalVRMap; 866 SmallVector<std::pair<unsigned,unsigned>, 4> CopyInfos; 867 MachineBasicBlock::iterator I = TailBB->begin(); 868 // Process PHI instructions first. 869 while (I != TailBB->end() && I->isPHI()) { 870 // Replace the uses of the def of the PHI with the register coming 871 // from PredBB. 872 MachineInstr *MI = &*I++; 873 ProcessPHI(MI, TailBB, PrevBB, LocalVRMap, CopyInfos, UsedByPhi, true); 874 if (MI->getParent()) 875 MI->eraseFromParent(); 876 } 877 878 // Now copy the non-PHI instructions. 879 while (I != TailBB->end()) { 880 // Replace def of virtual registers with new registers, and update 881 // uses with PHI source register or the new registers. 882 MachineInstr *MI = &*I++; 883 assert(!MI->isBundle() && "Not expecting bundles before regalloc!"); 884 DuplicateInstruction(MI, TailBB, PrevBB, MF, LocalVRMap, UsedByPhi); 885 MI->eraseFromParent(); 886 } 887 MachineBasicBlock::iterator Loc = PrevBB->getFirstTerminator(); 888 for (unsigned i = 0, e = CopyInfos.size(); i != e; ++i) { 889 Copies.push_back(BuildMI(*PrevBB, Loc, DebugLoc(), 890 TII->get(TargetOpcode::COPY), 891 CopyInfos[i].first) 892 .addReg(CopyInfos[i].second)); 893 } 894 } else { 895 // No PHIs to worry about, just splice the instructions over. 896 PrevBB->splice(PrevBB->end(), TailBB, TailBB->begin(), TailBB->end()); 897 } 898 PrevBB->removeSuccessor(PrevBB->succ_begin()); 899 assert(PrevBB->succ_empty()); 900 PrevBB->transferSuccessors(TailBB); 901 TDBBs.push_back(PrevBB); 902 Changed = true; 903 } 904 905 // If this is after register allocation, there are no phis to fix. 906 if (!PreRegAlloc) 907 return Changed; 908 909 // If we made no changes so far, we are safe. 910 if (!Changed) 911 return Changed; 912 913 914 // Handle the nasty case in that we duplicated a block that is part of a loop 915 // into some but not all of its predecessors. For example: 916 // 1 -> 2 <-> 3 | 917 // \ | 918 // \---> rest | 919 // if we duplicate 2 into 1 but not into 3, we end up with 920 // 12 -> 3 <-> 2 -> rest | 921 // \ / | 922 // \----->-----/ | 923 // If there was a "var = phi(1, 3)" in 2, it has to be ultimately replaced 924 // with a phi in 3 (which now dominates 2). 925 // What we do here is introduce a copy in 3 of the register defined by the 926 // phi, just like when we are duplicating 2 into 3, but we don't copy any 927 // real instructions or remove the 3 -> 2 edge from the phi in 2. 928 for (SmallSetVector<MachineBasicBlock *, 8>::iterator PI = Preds.begin(), 929 PE = Preds.end(); PI != PE; ++PI) { 930 MachineBasicBlock *PredBB = *PI; 931 if (std::find(TDBBs.begin(), TDBBs.end(), PredBB) != TDBBs.end()) 932 continue; 933 934 // EH edges 935 if (PredBB->succ_size() != 1) 936 continue; 937 938 DenseMap<unsigned, unsigned> LocalVRMap; 939 SmallVector<std::pair<unsigned,unsigned>, 4> CopyInfos; 940 MachineBasicBlock::iterator I = TailBB->begin(); 941 // Process PHI instructions first. 942 while (I != TailBB->end() && I->isPHI()) { 943 // Replace the uses of the def of the PHI with the register coming 944 // from PredBB. 945 MachineInstr *MI = &*I++; 946 ProcessPHI(MI, TailBB, PredBB, LocalVRMap, CopyInfos, UsedByPhi, false); 947 } 948 MachineBasicBlock::iterator Loc = PredBB->getFirstTerminator(); 949 for (unsigned i = 0, e = CopyInfos.size(); i != e; ++i) { 950 Copies.push_back(BuildMI(*PredBB, Loc, DebugLoc(), 951 TII->get(TargetOpcode::COPY), 952 CopyInfos[i].first).addReg(CopyInfos[i].second)); 953 } 954 } 955 956 return Changed; 957} 958 959/// RemoveDeadBlock - Remove the specified dead machine basic block from the 960/// function, updating the CFG. 961void TailDuplicatePass::RemoveDeadBlock(MachineBasicBlock *MBB) { 962 assert(MBB->pred_empty() && "MBB must be dead!"); 963 DEBUG(dbgs() << "\nRemoving MBB: " << *MBB); 964 965 // Remove all successors. 966 while (!MBB->succ_empty()) 967 MBB->removeSuccessor(MBB->succ_end()-1); 968 969 // Remove the block. 970 MBB->eraseFromParent(); 971} 972