IndVarSimplify.cpp revision cda9ca5a4fed09ea3788b572dbddabf2a5a7a5d9
1//===- IndVarSimplify.cpp - Induction Variable Elimination ----------------===// 2// 3// The LLVM Compiler Infrastructure 4// 5// This file was developed by the LLVM research group and is distributed under 6// the University of Illinois Open Source License. See LICENSE.TXT for details. 7// 8//===----------------------------------------------------------------------===// 9// 10// This transformation analyzes and transforms the induction variables (and 11// computations derived from them) into simpler forms suitable for subsequent 12// analysis and transformation. 13// 14// This transformation make the following changes to each loop with an 15// identifiable induction variable: 16// 1. All loops are transformed to have a SINGLE canonical induction variable 17// which starts at zero and steps by one. 18// 2. The canonical induction variable is guaranteed to be the first PHI node 19// in the loop header block. 20// 3. Any pointer arithmetic recurrences are raised to use array subscripts. 21// 22// If the trip count of a loop is computable, this pass also makes the following 23// changes: 24// 1. The exit condition for the loop is canonicalized to compare the 25// induction value against the exit value. This turns loops like: 26// 'for (i = 7; i*i < 1000; ++i)' into 'for (i = 0; i != 25; ++i)' 27// 2. Any use outside of the loop of an expression derived from the indvar 28// is changed to compute the derived value outside of the loop, eliminating 29// the dependence on the exit value of the induction variable. If the only 30// purpose of the loop is to compute the exit value of some derived 31// expression, this transformation will make the loop dead. 32// 33// This transformation should be followed by strength reduction after all of the 34// desired loop transformations have been performed. Additionally, on targets 35// where it is profitable, the loop could be transformed to count down to zero 36// (the "do loop" optimization). 37// 38//===----------------------------------------------------------------------===// 39 40#include "llvm/Transforms/Scalar.h" 41#include "llvm/BasicBlock.h" 42#include "llvm/Constants.h" 43#include "llvm/Instructions.h" 44#include "llvm/Type.h" 45#include "llvm/Analysis/ScalarEvolutionExpander.h" 46#include "llvm/Analysis/LoopInfo.h" 47#include "llvm/Support/CFG.h" 48#include "llvm/Support/GetElementPtrTypeIterator.h" 49#include "llvm/Transforms/Utils/Local.h" 50#include "llvm/Support/CommandLine.h" 51#include "llvm/ADT/Statistic.h" 52using namespace llvm; 53 54namespace { 55 Statistic<> NumRemoved ("indvars", "Number of aux indvars removed"); 56 Statistic<> NumPointer ("indvars", "Number of pointer indvars promoted"); 57 Statistic<> NumInserted("indvars", "Number of canonical indvars added"); 58 Statistic<> NumReplaced("indvars", "Number of exit values replaced"); 59 Statistic<> NumLFTR ("indvars", "Number of loop exit tests replaced"); 60 61 class IndVarSimplify : public FunctionPass { 62 LoopInfo *LI; 63 ScalarEvolution *SE; 64 bool Changed; 65 public: 66 virtual bool runOnFunction(Function &) { 67 LI = &getAnalysis<LoopInfo>(); 68 SE = &getAnalysis<ScalarEvolution>(); 69 Changed = false; 70 71 // Induction Variables live in the header nodes of loops 72 for (LoopInfo::iterator I = LI->begin(), E = LI->end(); I != E; ++I) 73 runOnLoop(*I); 74 return Changed; 75 } 76 77 virtual void getAnalysisUsage(AnalysisUsage &AU) const { 78 AU.addRequiredID(LoopSimplifyID); 79 AU.addRequired<ScalarEvolution>(); 80 AU.addRequired<LoopInfo>(); 81 AU.addPreservedID(LoopSimplifyID); 82 AU.setPreservesCFG(); 83 } 84 private: 85 void runOnLoop(Loop *L); 86 void EliminatePointerRecurrence(PHINode *PN, BasicBlock *Preheader, 87 std::set<Instruction*> &DeadInsts); 88 void LinearFunctionTestReplace(Loop *L, SCEV *IterationCount, 89 SCEVExpander &RW); 90 void RewriteLoopExitValues(Loop *L); 91 92 void DeleteTriviallyDeadInstructions(std::set<Instruction*> &Insts); 93 }; 94 RegisterOpt<IndVarSimplify> X("indvars", "Canonicalize Induction Variables"); 95} 96 97FunctionPass *llvm::createIndVarSimplifyPass() { 98 return new IndVarSimplify(); 99} 100 101/// DeleteTriviallyDeadInstructions - If any of the instructions is the 102/// specified set are trivially dead, delete them and see if this makes any of 103/// their operands subsequently dead. 104void IndVarSimplify:: 105DeleteTriviallyDeadInstructions(std::set<Instruction*> &Insts) { 106 while (!Insts.empty()) { 107 Instruction *I = *Insts.begin(); 108 Insts.erase(Insts.begin()); 109 if (isInstructionTriviallyDead(I)) { 110 for (unsigned i = 0, e = I->getNumOperands(); i != e; ++i) 111 if (Instruction *U = dyn_cast<Instruction>(I->getOperand(i))) 112 Insts.insert(U); 113 SE->deleteInstructionFromRecords(I); 114 I->eraseFromParent(); 115 Changed = true; 116 } 117 } 118} 119 120 121/// EliminatePointerRecurrence - Check to see if this is a trivial GEP pointer 122/// recurrence. If so, change it into an integer recurrence, permitting 123/// analysis by the SCEV routines. 124void IndVarSimplify::EliminatePointerRecurrence(PHINode *PN, 125 BasicBlock *Preheader, 126 std::set<Instruction*> &DeadInsts) { 127 assert(PN->getNumIncomingValues() == 2 && "Noncanonicalized loop!"); 128 unsigned PreheaderIdx = PN->getBasicBlockIndex(Preheader); 129 unsigned BackedgeIdx = PreheaderIdx^1; 130 if (GetElementPtrInst *GEPI = 131 dyn_cast<GetElementPtrInst>(PN->getIncomingValue(BackedgeIdx))) 132 if (GEPI->getOperand(0) == PN) { 133 assert(GEPI->getNumOperands() == 2 && "GEP types must match!"); 134 135 // Okay, we found a pointer recurrence. Transform this pointer 136 // recurrence into an integer recurrence. Compute the value that gets 137 // added to the pointer at every iteration. 138 Value *AddedVal = GEPI->getOperand(1); 139 140 // Insert a new integer PHI node into the top of the block. 141 PHINode *NewPhi = new PHINode(AddedVal->getType(), 142 PN->getName()+".rec", PN); 143 NewPhi->addIncoming(Constant::getNullValue(NewPhi->getType()), Preheader); 144 145 // Create the new add instruction. 146 Value *NewAdd = BinaryOperator::createAdd(NewPhi, AddedVal, 147 GEPI->getName()+".rec", GEPI); 148 NewPhi->addIncoming(NewAdd, PN->getIncomingBlock(BackedgeIdx)); 149 150 // Update the existing GEP to use the recurrence. 151 GEPI->setOperand(0, PN->getIncomingValue(PreheaderIdx)); 152 153 // Update the GEP to use the new recurrence we just inserted. 154 GEPI->setOperand(1, NewAdd); 155 156 // If the incoming value is a constant expr GEP, try peeling out the array 157 // 0 index if possible to make things simpler. 158 if (ConstantExpr *CE = dyn_cast<ConstantExpr>(GEPI->getOperand(0))) 159 if (CE->getOpcode() == Instruction::GetElementPtr) { 160 unsigned NumOps = CE->getNumOperands(); 161 assert(NumOps > 1 && "CE folding didn't work!"); 162 if (CE->getOperand(NumOps-1)->isNullValue()) { 163 // Check to make sure the last index really is an array index. 164 gep_type_iterator GTI = gep_type_begin(GEPI); 165 for (unsigned i = 1, e = GEPI->getNumOperands()-1; 166 i != e; ++i, ++GTI) 167 /*empty*/; 168 if (isa<SequentialType>(*GTI)) { 169 // Pull the last index out of the constant expr GEP. 170 std::vector<Value*> CEIdxs(CE->op_begin()+1, CE->op_end()-1); 171 Constant *NCE = ConstantExpr::getGetElementPtr(CE->getOperand(0), 172 CEIdxs); 173 GetElementPtrInst *NGEPI = 174 new GetElementPtrInst(NCE, Constant::getNullValue(Type::IntTy), 175 NewAdd, GEPI->getName(), GEPI); 176 GEPI->replaceAllUsesWith(NGEPI); 177 GEPI->eraseFromParent(); 178 GEPI = NGEPI; 179 } 180 } 181 } 182 183 184 // Finally, if there are any other users of the PHI node, we must 185 // insert a new GEP instruction that uses the pre-incremented version 186 // of the induction amount. 187 if (!PN->use_empty()) { 188 BasicBlock::iterator InsertPos = PN; ++InsertPos; 189 while (isa<PHINode>(InsertPos)) ++InsertPos; 190 std::string Name = PN->getName(); PN->setName(""); 191 Value *PreInc = 192 new GetElementPtrInst(PN->getIncomingValue(PreheaderIdx), 193 std::vector<Value*>(1, NewPhi), Name, 194 InsertPos); 195 PN->replaceAllUsesWith(PreInc); 196 } 197 198 // Delete the old PHI for sure, and the GEP if its otherwise unused. 199 DeadInsts.insert(PN); 200 201 ++NumPointer; 202 Changed = true; 203 } 204} 205 206/// LinearFunctionTestReplace - This method rewrites the exit condition of the 207/// loop to be a canonical != comparison against the incremented loop induction 208/// variable. This pass is able to rewrite the exit tests of any loop where the 209/// SCEV analysis can determine a loop-invariant trip count of the loop, which 210/// is actually a much broader range than just linear tests. 211void IndVarSimplify::LinearFunctionTestReplace(Loop *L, SCEV *IterationCount, 212 SCEVExpander &RW) { 213 // Find the exit block for the loop. We can currently only handle loops with 214 // a single exit. 215 std::vector<BasicBlock*> ExitBlocks; 216 L->getExitBlocks(ExitBlocks); 217 if (ExitBlocks.size() != 1) return; 218 BasicBlock *ExitBlock = ExitBlocks[0]; 219 220 // Make sure there is only one predecessor block in the loop. 221 BasicBlock *ExitingBlock = 0; 222 for (pred_iterator PI = pred_begin(ExitBlock), PE = pred_end(ExitBlock); 223 PI != PE; ++PI) 224 if (L->contains(*PI)) { 225 if (ExitingBlock == 0) 226 ExitingBlock = *PI; 227 else 228 return; // Multiple exits from loop to this block. 229 } 230 assert(ExitingBlock && "Loop info is broken"); 231 232 if (!isa<BranchInst>(ExitingBlock->getTerminator())) 233 return; // Can't rewrite non-branch yet 234 BranchInst *BI = cast<BranchInst>(ExitingBlock->getTerminator()); 235 assert(BI->isConditional() && "Must be conditional to be part of loop!"); 236 237 std::set<Instruction*> InstructionsToDelete; 238 if (Instruction *Cond = dyn_cast<Instruction>(BI->getCondition())) 239 InstructionsToDelete.insert(Cond); 240 241 // If the exiting block is not the same as the backedge block, we must compare 242 // against the preincremented value, otherwise we prefer to compare against 243 // the post-incremented value. 244 BasicBlock *Header = L->getHeader(); 245 pred_iterator HPI = pred_begin(Header); 246 assert(HPI != pred_end(Header) && "Loop with zero preds???"); 247 if (!L->contains(*HPI)) ++HPI; 248 assert(HPI != pred_end(Header) && L->contains(*HPI) && 249 "No backedge in loop?"); 250 251 SCEVHandle TripCount = IterationCount; 252 Value *IndVar; 253 if (*HPI == ExitingBlock) { 254 // The IterationCount expression contains the number of times that the 255 // backedge actually branches to the loop header. This is one less than the 256 // number of times the loop executes, so add one to it. 257 Constant *OneC = ConstantInt::get(IterationCount->getType(), 1); 258 TripCount = SCEVAddExpr::get(IterationCount, SCEVUnknown::get(OneC)); 259 IndVar = L->getCanonicalInductionVariableIncrement(); 260 } else { 261 // We have to use the preincremented value... 262 IndVar = L->getCanonicalInductionVariable(); 263 } 264 265 // Expand the code for the iteration count into the preheader of the loop. 266 BasicBlock *Preheader = L->getLoopPreheader(); 267 Value *ExitCnt = RW.expandCodeFor(TripCount, Preheader->getTerminator(), 268 IndVar->getType()); 269 270 // Insert a new setne or seteq instruction before the branch. 271 Instruction::BinaryOps Opcode; 272 if (L->contains(BI->getSuccessor(0))) 273 Opcode = Instruction::SetNE; 274 else 275 Opcode = Instruction::SetEQ; 276 277 Value *Cond = new SetCondInst(Opcode, IndVar, ExitCnt, "exitcond", BI); 278 BI->setCondition(Cond); 279 ++NumLFTR; 280 Changed = true; 281 282 DeleteTriviallyDeadInstructions(InstructionsToDelete); 283} 284 285 286/// RewriteLoopExitValues - Check to see if this loop has a computable 287/// loop-invariant execution count. If so, this means that we can compute the 288/// final value of any expressions that are recurrent in the loop, and 289/// substitute the exit values from the loop into any instructions outside of 290/// the loop that use the final values of the current expressions. 291void IndVarSimplify::RewriteLoopExitValues(Loop *L) { 292 BasicBlock *Preheader = L->getLoopPreheader(); 293 294 // Scan all of the instructions in the loop, looking at those that have 295 // extra-loop users and which are recurrences. 296 SCEVExpander Rewriter(*SE, *LI); 297 298 // We insert the code into the preheader of the loop if the loop contains 299 // multiple exit blocks, or in the exit block if there is exactly one. 300 BasicBlock *BlockToInsertInto; 301 std::vector<BasicBlock*> ExitBlocks; 302 L->getExitBlocks(ExitBlocks); 303 if (ExitBlocks.size() == 1) 304 BlockToInsertInto = ExitBlocks[0]; 305 else 306 BlockToInsertInto = Preheader; 307 BasicBlock::iterator InsertPt = BlockToInsertInto->begin(); 308 while (isa<PHINode>(InsertPt)) ++InsertPt; 309 310 bool HasConstantItCount = isa<SCEVConstant>(SE->getIterationCount(L)); 311 312 std::set<Instruction*> InstructionsToDelete; 313 314 for (unsigned i = 0, e = L->getBlocks().size(); i != e; ++i) 315 if (LI->getLoopFor(L->getBlocks()[i]) == L) { // Not in a subloop... 316 BasicBlock *BB = L->getBlocks()[i]; 317 for (BasicBlock::iterator I = BB->begin(), E = BB->end(); I != E;) { 318 if (I->getType()->isInteger()) { // Is an integer instruction 319 SCEVHandle SH = SE->getSCEV(I); 320 if (SH->hasComputableLoopEvolution(L) || // Varies predictably 321 HasConstantItCount) { 322 // Find out if this predictably varying value is actually used 323 // outside of the loop. "extra" as opposed to "intra". 324 std::vector<User*> ExtraLoopUsers; 325 for (Value::use_iterator UI = I->use_begin(), E = I->use_end(); 326 UI != E; ++UI) 327 if (!L->contains(cast<Instruction>(*UI)->getParent())) 328 ExtraLoopUsers.push_back(*UI); 329 if (!ExtraLoopUsers.empty()) { 330 // Okay, this instruction has a user outside of the current loop 331 // and varies predictably in this loop. Evaluate the value it 332 // contains when the loop exits, and insert code for it. 333 SCEVHandle ExitValue = SE->getSCEVAtScope(I, L->getParentLoop()); 334 if (!isa<SCEVCouldNotCompute>(ExitValue)) { 335 Changed = true; 336 ++NumReplaced; 337 // Remember the next instruction. The rewriter can move code 338 // around in some cases. 339 BasicBlock::iterator NextI = I; ++NextI; 340 341 Value *NewVal = Rewriter.expandCodeFor(ExitValue, InsertPt, 342 I->getType()); 343 344 // Rewrite any users of the computed value outside of the loop 345 // with the newly computed value. 346 for (unsigned i = 0, e = ExtraLoopUsers.size(); i != e; ++i) 347 ExtraLoopUsers[i]->replaceUsesOfWith(I, NewVal); 348 349 // If this instruction is dead now, schedule it to be removed. 350 if (I->use_empty()) 351 InstructionsToDelete.insert(I); 352 I = NextI; 353 continue; // Skip the ++I 354 } 355 } 356 } 357 } 358 359 // Next instruction. Continue instruction skips this. 360 ++I; 361 } 362 } 363 364 DeleteTriviallyDeadInstructions(InstructionsToDelete); 365} 366 367 368void IndVarSimplify::runOnLoop(Loop *L) { 369 // First step. Check to see if there are any trivial GEP pointer recurrences. 370 // If there are, change them into integer recurrences, permitting analysis by 371 // the SCEV routines. 372 // 373 BasicBlock *Header = L->getHeader(); 374 BasicBlock *Preheader = L->getLoopPreheader(); 375 376 std::set<Instruction*> DeadInsts; 377 for (BasicBlock::iterator I = Header->begin(); isa<PHINode>(I); ++I) { 378 PHINode *PN = cast<PHINode>(I); 379 if (isa<PointerType>(PN->getType())) 380 EliminatePointerRecurrence(PN, Preheader, DeadInsts); 381 } 382 383 if (!DeadInsts.empty()) 384 DeleteTriviallyDeadInstructions(DeadInsts); 385 386 387 // Next, transform all loops nesting inside of this loop. 388 for (LoopInfo::iterator I = L->begin(), E = L->end(); I != E; ++I) 389 runOnLoop(*I); 390 391 // Check to see if this loop has a computable loop-invariant execution count. 392 // If so, this means that we can compute the final value of any expressions 393 // that are recurrent in the loop, and substitute the exit values from the 394 // loop into any instructions outside of the loop that use the final values of 395 // the current expressions. 396 // 397 SCEVHandle IterationCount = SE->getIterationCount(L); 398 if (!isa<SCEVCouldNotCompute>(IterationCount)) 399 RewriteLoopExitValues(L); 400 401 // Next, analyze all of the induction variables in the loop, canonicalizing 402 // auxillary induction variables. 403 std::vector<std::pair<PHINode*, SCEVHandle> > IndVars; 404 405 for (BasicBlock::iterator I = Header->begin(); isa<PHINode>(I); ++I) { 406 PHINode *PN = cast<PHINode>(I); 407 if (PN->getType()->isInteger()) { // FIXME: when we have fast-math, enable! 408 SCEVHandle SCEV = SE->getSCEV(PN); 409 if (SCEV->hasComputableLoopEvolution(L)) 410 // FIXME: It is an extremely bad idea to indvar substitute anything more 411 // complex than affine induction variables. Doing so will put expensive 412 // polynomial evaluations inside of the loop, and the str reduction pass 413 // currently can only reduce affine polynomials. For now just disable 414 // indvar subst on anything more complex than an affine addrec. 415 if (SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(SCEV)) 416 if (AR->isAffine()) 417 IndVars.push_back(std::make_pair(PN, SCEV)); 418 } 419 } 420 421 // If there are no induction variables in the loop, there is nothing more to 422 // do. 423 if (IndVars.empty()) { 424 // Actually, if we know how many times the loop iterates, lets insert a 425 // canonical induction variable to help subsequent passes. 426 if (!isa<SCEVCouldNotCompute>(IterationCount)) { 427 SCEVExpander Rewriter(*SE, *LI); 428 Rewriter.getOrInsertCanonicalInductionVariable(L, 429 IterationCount->getType()); 430 LinearFunctionTestReplace(L, IterationCount, Rewriter); 431 } 432 return; 433 } 434 435 // Compute the type of the largest recurrence expression. 436 // 437 const Type *LargestType = IndVars[0].first->getType(); 438 bool DifferingSizes = false; 439 for (unsigned i = 1, e = IndVars.size(); i != e; ++i) { 440 const Type *Ty = IndVars[i].first->getType(); 441 DifferingSizes |= Ty->getPrimitiveSize() != LargestType->getPrimitiveSize(); 442 if (Ty->getPrimitiveSize() > LargestType->getPrimitiveSize()) 443 LargestType = Ty; 444 } 445 446 // Create a rewriter object which we'll use to transform the code with. 447 SCEVExpander Rewriter(*SE, *LI); 448 449 // Now that we know the largest of of the induction variables in this loop, 450 // insert a canonical induction variable of the largest size. 451 LargestType = LargestType->getUnsignedVersion(); 452 Value *IndVar = Rewriter.getOrInsertCanonicalInductionVariable(L,LargestType); 453 ++NumInserted; 454 Changed = true; 455 456 if (!isa<SCEVCouldNotCompute>(IterationCount)) 457 LinearFunctionTestReplace(L, IterationCount, Rewriter); 458 459 // Now that we have a canonical induction variable, we can rewrite any 460 // recurrences in terms of the induction variable. Start with the auxillary 461 // induction variables, and recursively rewrite any of their uses. 462 BasicBlock::iterator InsertPt = Header->begin(); 463 while (isa<PHINode>(InsertPt)) ++InsertPt; 464 465 // If there were induction variables of other sizes, cast the primary 466 // induction variable to the right size for them, avoiding the need for the 467 // code evaluation methods to insert induction variables of different sizes. 468 if (DifferingSizes) { 469 bool InsertedSizes[17] = { false }; 470 InsertedSizes[LargestType->getPrimitiveSize()] = true; 471 for (unsigned i = 0, e = IndVars.size(); i != e; ++i) 472 if (!InsertedSizes[IndVars[i].first->getType()->getPrimitiveSize()]) { 473 PHINode *PN = IndVars[i].first; 474 InsertedSizes[PN->getType()->getPrimitiveSize()] = true; 475 Instruction *New = new CastInst(IndVar, 476 PN->getType()->getUnsignedVersion(), 477 "indvar", InsertPt); 478 Rewriter.addInsertedValue(New, SE->getSCEV(New)); 479 } 480 } 481 482 // If there were induction variables of other sizes, cast the primary 483 // induction variable to the right size for them, avoiding the need for the 484 // code evaluation methods to insert induction variables of different sizes. 485 std::map<unsigned, Value*> InsertedSizes; 486 while (!IndVars.empty()) { 487 PHINode *PN = IndVars.back().first; 488 Value *NewVal = Rewriter.expandCodeFor(IndVars.back().second, InsertPt, 489 PN->getType()); 490 std::string Name = PN->getName(); 491 PN->setName(""); 492 NewVal->setName(Name); 493 494 // Replace the old PHI Node with the inserted computation. 495 PN->replaceAllUsesWith(NewVal); 496 DeadInsts.insert(PN); 497 IndVars.pop_back(); 498 ++NumRemoved; 499 Changed = true; 500 } 501 502#if 0 503 // Now replace all derived expressions in the loop body with simpler 504 // expressions. 505 for (unsigned i = 0, e = L->getBlocks().size(); i != e; ++i) 506 if (LI->getLoopFor(L->getBlocks()[i]) == L) { // Not in a subloop... 507 BasicBlock *BB = L->getBlocks()[i]; 508 for (BasicBlock::iterator I = BB->begin(), E = BB->end(); I != E; ++I) 509 if (I->getType()->isInteger() && // Is an integer instruction 510 !I->use_empty() && 511 !Rewriter.isInsertedInstruction(I)) { 512 SCEVHandle SH = SE->getSCEV(I); 513 Value *V = Rewriter.expandCodeFor(SH, I, I->getType()); 514 if (V != I) { 515 if (isa<Instruction>(V)) { 516 std::string Name = I->getName(); 517 I->setName(""); 518 V->setName(Name); 519 } 520 I->replaceAllUsesWith(V); 521 DeadInsts.insert(I); 522 ++NumRemoved; 523 Changed = true; 524 } 525 } 526 } 527#endif 528 529 DeleteTriviallyDeadInstructions(DeadInsts); 530} 531