TailRecursionElimination.cpp revision b5d84d13bc338efc4eeed87d19c49dfaed38e036
1//===- TailRecursionElimination.cpp - Eliminate Tail Calls ----------------===// 2// 3// The LLVM Compiler Infrastructure 4// 5// This file is distributed under the University of Illinois Open Source 6// License. See LICENSE.TXT for details. 7// 8//===----------------------------------------------------------------------===// 9// 10// This file transforms calls of the current function (self recursion) followed 11// by a return instruction with a branch to the entry of the function, creating 12// a loop. This pass also implements the following extensions to the basic 13// algorithm: 14// 15// 1. Trivial instructions between the call and return do not prevent the 16// transformation from taking place, though currently the analysis cannot 17// support moving any really useful instructions (only dead ones). 18// 2. This pass transforms functions that are prevented from being tail 19// recursive by an associative and commutative expression to use an 20// accumulator variable, thus compiling the typical naive factorial or 21// 'fib' implementation into efficient code. 22// 3. TRE is performed if the function returns void, if the return 23// returns the result returned by the call, or if the function returns a 24// run-time constant on all exits from the function. It is possible, though 25// unlikely, that the return returns something else (like constant 0), and 26// can still be TRE'd. It can be TRE'd if ALL OTHER return instructions in 27// the function return the exact same value. 28// 4. If it can prove that callees do not access their caller stack frame, 29// they are marked as eligible for tail call elimination (by the code 30// generator). 31// 32// There are several improvements that could be made: 33// 34// 1. If the function has any alloca instructions, these instructions will be 35// moved out of the entry block of the function, causing them to be 36// evaluated each time through the tail recursion. Safely keeping allocas 37// in the entry block requires analysis to proves that the tail-called 38// function does not read or write the stack object. 39// 2. Tail recursion is only performed if the call immediately preceeds the 40// return instruction. It's possible that there could be a jump between 41// the call and the return. 42// 3. There can be intervening operations between the call and the return that 43// prevent the TRE from occurring. For example, there could be GEP's and 44// stores to memory that will not be read or written by the call. This 45// requires some substantial analysis (such as with DSA) to prove safe to 46// move ahead of the call, but doing so could allow many more TREs to be 47// performed, for example in TreeAdd/TreeAlloc from the treeadd benchmark. 48// 4. The algorithm we use to detect if callees access their caller stack 49// frames is very primitive. 50// 51//===----------------------------------------------------------------------===// 52 53#define DEBUG_TYPE "tailcallelim" 54#include "llvm/Transforms/Scalar.h" 55#include "llvm/Transforms/Utils/Local.h" 56#include "llvm/Constants.h" 57#include "llvm/DerivedTypes.h" 58#include "llvm/Function.h" 59#include "llvm/Instructions.h" 60#include "llvm/Pass.h" 61#include "llvm/Analysis/CaptureTracking.h" 62#include "llvm/Analysis/InlineCost.h" 63#include "llvm/Analysis/Loads.h" 64#include "llvm/Support/CallSite.h" 65#include "llvm/Support/CFG.h" 66#include "llvm/ADT/Statistic.h" 67using namespace llvm; 68 69STATISTIC(NumEliminated, "Number of tail calls removed"); 70STATISTIC(NumAccumAdded, "Number of accumulators introduced"); 71 72namespace { 73 struct TailCallElim : public FunctionPass { 74 static char ID; // Pass identification, replacement for typeid 75 TailCallElim() : FunctionPass(ID) {} 76 77 virtual bool runOnFunction(Function &F); 78 79 private: 80 bool ProcessReturningBlock(ReturnInst *RI, BasicBlock *&OldEntry, 81 bool &TailCallsAreMarkedTail, 82 SmallVector<PHINode*, 8> &ArgumentPHIs, 83 bool CannotTailCallElimCallsMarkedTail); 84 bool CanMoveAboveCall(Instruction *I, CallInst *CI); 85 Value *CanTransformAccumulatorRecursion(Instruction *I, CallInst *CI); 86 }; 87} 88 89char TailCallElim::ID = 0; 90INITIALIZE_PASS(TailCallElim, "tailcallelim", 91 "Tail Call Elimination", false, false); 92 93// Public interface to the TailCallElimination pass 94FunctionPass *llvm::createTailCallEliminationPass() { 95 return new TailCallElim(); 96} 97 98/// AllocaMightEscapeToCalls - Return true if this alloca may be accessed by 99/// callees of this function. We only do very simple analysis right now, this 100/// could be expanded in the future to use mod/ref information for particular 101/// call sites if desired. 102static bool AllocaMightEscapeToCalls(AllocaInst *AI) { 103 // FIXME: do simple 'address taken' analysis. 104 return true; 105} 106 107/// CheckForEscapingAllocas - Scan the specified basic block for alloca 108/// instructions. If it contains any that might be accessed by calls, return 109/// true. 110static bool CheckForEscapingAllocas(BasicBlock *BB, 111 bool &CannotTCETailMarkedCall) { 112 bool RetVal = false; 113 for (BasicBlock::iterator I = BB->begin(), E = BB->end(); I != E; ++I) 114 if (AllocaInst *AI = dyn_cast<AllocaInst>(I)) { 115 RetVal |= AllocaMightEscapeToCalls(AI); 116 117 // If this alloca is in the body of the function, or if it is a variable 118 // sized allocation, we cannot tail call eliminate calls marked 'tail' 119 // with this mechanism. 120 if (BB != &BB->getParent()->getEntryBlock() || 121 !isa<ConstantInt>(AI->getArraySize())) 122 CannotTCETailMarkedCall = true; 123 } 124 return RetVal; 125} 126 127bool TailCallElim::runOnFunction(Function &F) { 128 // If this function is a varargs function, we won't be able to PHI the args 129 // right, so don't even try to convert it... 130 if (F.getFunctionType()->isVarArg()) return false; 131 132 BasicBlock *OldEntry = 0; 133 bool TailCallsAreMarkedTail = false; 134 SmallVector<PHINode*, 8> ArgumentPHIs; 135 bool MadeChange = false; 136 137 bool FunctionContainsEscapingAllocas = false; 138 139 // CannotTCETailMarkedCall - If true, we cannot perform TCE on tail calls 140 // marked with the 'tail' attribute, because doing so would cause the stack 141 // size to increase (real TCE would deallocate variable sized allocas, TCE 142 // doesn't). 143 bool CannotTCETailMarkedCall = false; 144 145 // Loop over the function, looking for any returning blocks, and keeping track 146 // of whether this function has any non-trivially used allocas. 147 for (Function::iterator BB = F.begin(), E = F.end(); BB != E; ++BB) { 148 if (FunctionContainsEscapingAllocas && CannotTCETailMarkedCall) 149 break; 150 151 FunctionContainsEscapingAllocas |= 152 CheckForEscapingAllocas(BB, CannotTCETailMarkedCall); 153 } 154 155 /// FIXME: The code generator produces really bad code when an 'escaping 156 /// alloca' is changed from being a static alloca to being a dynamic alloca. 157 /// Until this is resolved, disable this transformation if that would ever 158 /// happen. This bug is PR962. 159 if (FunctionContainsEscapingAllocas) 160 return false; 161 162 // Second pass, change any tail calls to loops. 163 for (Function::iterator BB = F.begin(), E = F.end(); BB != E; ++BB) 164 if (ReturnInst *Ret = dyn_cast<ReturnInst>(BB->getTerminator())) 165 MadeChange |= ProcessReturningBlock(Ret, OldEntry, TailCallsAreMarkedTail, 166 ArgumentPHIs,CannotTCETailMarkedCall); 167 168 // If we eliminated any tail recursions, it's possible that we inserted some 169 // silly PHI nodes which just merge an initial value (the incoming operand) 170 // with themselves. Check to see if we did and clean up our mess if so. This 171 // occurs when a function passes an argument straight through to its tail 172 // call. 173 if (!ArgumentPHIs.empty()) { 174 for (unsigned i = 0, e = ArgumentPHIs.size(); i != e; ++i) { 175 PHINode *PN = ArgumentPHIs[i]; 176 177 // If the PHI Node is a dynamic constant, replace it with the value it is. 178 if (Value *PNV = PN->hasConstantValue()) { 179 PN->replaceAllUsesWith(PNV); 180 PN->eraseFromParent(); 181 } 182 } 183 } 184 185 // Finally, if this function contains no non-escaping allocas, mark all calls 186 // in the function as eligible for tail calls (there is no stack memory for 187 // them to access). 188 if (!FunctionContainsEscapingAllocas) 189 for (Function::iterator BB = F.begin(), E = F.end(); BB != E; ++BB) 190 for (BasicBlock::iterator I = BB->begin(), E = BB->end(); I != E; ++I) 191 if (CallInst *CI = dyn_cast<CallInst>(I)) { 192 CI->setTailCall(); 193 MadeChange = true; 194 } 195 196 return MadeChange; 197} 198 199 200/// CanMoveAboveCall - Return true if it is safe to move the specified 201/// instruction from after the call to before the call, assuming that all 202/// instructions between the call and this instruction are movable. 203/// 204bool TailCallElim::CanMoveAboveCall(Instruction *I, CallInst *CI) { 205 // FIXME: We can move load/store/call/free instructions above the call if the 206 // call does not mod/ref the memory location being processed. 207 if (I->mayHaveSideEffects()) // This also handles volatile loads. 208 return false; 209 210 if (LoadInst *L = dyn_cast<LoadInst>(I)) { 211 // Loads may always be moved above calls without side effects. 212 if (CI->mayHaveSideEffects()) { 213 // Non-volatile loads may be moved above a call with side effects if it 214 // does not write to memory and the load provably won't trap. 215 // FIXME: Writes to memory only matter if they may alias the pointer 216 // being loaded from. 217 if (CI->mayWriteToMemory() || 218 !isSafeToLoadUnconditionally(L->getPointerOperand(), L, 219 L->getAlignment())) 220 return false; 221 } 222 } 223 224 // Otherwise, if this is a side-effect free instruction, check to make sure 225 // that it does not use the return value of the call. If it doesn't use the 226 // return value of the call, it must only use things that are defined before 227 // the call, or movable instructions between the call and the instruction 228 // itself. 229 for (unsigned i = 0, e = I->getNumOperands(); i != e; ++i) 230 if (I->getOperand(i) == CI) 231 return false; 232 return true; 233} 234 235// isDynamicConstant - Return true if the specified value is the same when the 236// return would exit as it was when the initial iteration of the recursive 237// function was executed. 238// 239// We currently handle static constants and arguments that are not modified as 240// part of the recursion. 241// 242static bool isDynamicConstant(Value *V, CallInst *CI, ReturnInst *RI) { 243 if (isa<Constant>(V)) return true; // Static constants are always dyn consts 244 245 // Check to see if this is an immutable argument, if so, the value 246 // will be available to initialize the accumulator. 247 if (Argument *Arg = dyn_cast<Argument>(V)) { 248 // Figure out which argument number this is... 249 unsigned ArgNo = 0; 250 Function *F = CI->getParent()->getParent(); 251 for (Function::arg_iterator AI = F->arg_begin(); &*AI != Arg; ++AI) 252 ++ArgNo; 253 254 // If we are passing this argument into call as the corresponding 255 // argument operand, then the argument is dynamically constant. 256 // Otherwise, we cannot transform this function safely. 257 if (CI->getArgOperand(ArgNo) == Arg) 258 return true; 259 } 260 261 // Switch cases are always constant integers. If the value is being switched 262 // on and the return is only reachable from one of its cases, it's 263 // effectively constant. 264 if (BasicBlock *UniquePred = RI->getParent()->getUniquePredecessor()) 265 if (SwitchInst *SI = dyn_cast<SwitchInst>(UniquePred->getTerminator())) 266 if (SI->getCondition() == V) 267 return SI->getDefaultDest() != RI->getParent(); 268 269 // Not a constant or immutable argument, we can't safely transform. 270 return false; 271} 272 273// getCommonReturnValue - Check to see if the function containing the specified 274// tail call consistently returns the same runtime-constant value at all exit 275// points except for IgnoreRI. If so, return the returned value. 276// 277static Value *getCommonReturnValue(ReturnInst *IgnoreRI, CallInst *CI) { 278 Function *F = CI->getParent()->getParent(); 279 Value *ReturnedValue = 0; 280 281 for (Function::iterator BBI = F->begin(), E = F->end(); BBI != E; ++BBI) { 282 ReturnInst *RI = dyn_cast<ReturnInst>(BBI->getTerminator()); 283 if (RI == 0 || RI == IgnoreRI) continue; 284 285 // We can only perform this transformation if the value returned is 286 // evaluatable at the start of the initial invocation of the function, 287 // instead of at the end of the evaluation. 288 // 289 Value *RetOp = RI->getOperand(0); 290 if (!isDynamicConstant(RetOp, CI, RI)) 291 return 0; 292 293 if (ReturnedValue && RetOp != ReturnedValue) 294 return 0; // Cannot transform if differing values are returned. 295 ReturnedValue = RetOp; 296 } 297 return ReturnedValue; 298} 299 300/// CanTransformAccumulatorRecursion - If the specified instruction can be 301/// transformed using accumulator recursion elimination, return the constant 302/// which is the start of the accumulator value. Otherwise return null. 303/// 304Value *TailCallElim::CanTransformAccumulatorRecursion(Instruction *I, 305 CallInst *CI) { 306 if (!I->isAssociative() || !I->isCommutative()) return 0; 307 assert(I->getNumOperands() == 2 && 308 "Associative/commutative operations should have 2 args!"); 309 310 // Exactly one operand should be the result of the call instruction. 311 if ((I->getOperand(0) == CI && I->getOperand(1) == CI) || 312 (I->getOperand(0) != CI && I->getOperand(1) != CI)) 313 return 0; 314 315 // The only user of this instruction we allow is a single return instruction. 316 if (!I->hasOneUse() || !isa<ReturnInst>(I->use_back())) 317 return 0; 318 319 // Ok, now we have to check all of the other return instructions in this 320 // function. If they return non-constants or differing values, then we cannot 321 // transform the function safely. 322 return getCommonReturnValue(cast<ReturnInst>(I->use_back()), CI); 323} 324 325bool TailCallElim::ProcessReturningBlock(ReturnInst *Ret, BasicBlock *&OldEntry, 326 bool &TailCallsAreMarkedTail, 327 SmallVector<PHINode*, 8> &ArgumentPHIs, 328 bool CannotTailCallElimCallsMarkedTail) { 329 BasicBlock *BB = Ret->getParent(); 330 Function *F = BB->getParent(); 331 332 if (&BB->front() == Ret) // Make sure there is something before the ret... 333 return false; 334 335 // Scan backwards from the return, checking to see if there is a tail call in 336 // this block. If so, set CI to it. 337 CallInst *CI; 338 BasicBlock::iterator BBI = Ret; 339 while (1) { 340 CI = dyn_cast<CallInst>(BBI); 341 if (CI && CI->getCalledFunction() == F) 342 break; 343 344 if (BBI == BB->begin()) 345 return false; // Didn't find a potential tail call. 346 --BBI; 347 } 348 349 // If this call is marked as a tail call, and if there are dynamic allocas in 350 // the function, we cannot perform this optimization. 351 if (CI->isTailCall() && CannotTailCallElimCallsMarkedTail) 352 return false; 353 354 // As a special case, detect code like this: 355 // double fabs(double f) { return __builtin_fabs(f); } // a 'fabs' call 356 // and disable this xform in this case, because the code generator will 357 // lower the call to fabs into inline code. 358 if (BB == &F->getEntryBlock() && 359 &BB->front() == CI && &*++BB->begin() == Ret && 360 callIsSmall(F)) { 361 // A single-block function with just a call and a return. Check that 362 // the arguments match. 363 CallSite::arg_iterator I = CallSite(CI).arg_begin(), 364 E = CallSite(CI).arg_end(); 365 Function::arg_iterator FI = F->arg_begin(), 366 FE = F->arg_end(); 367 for (; I != E && FI != FE; ++I, ++FI) 368 if (*I != &*FI) break; 369 if (I == E && FI == FE) 370 return false; 371 } 372 373 // If we are introducing accumulator recursion to eliminate operations after 374 // the call instruction that are both associative and commutative, the initial 375 // value for the accumulator is placed in this variable. If this value is set 376 // then we actually perform accumulator recursion elimination instead of 377 // simple tail recursion elimination. If the operation is an LLVM instruction 378 // (eg: "add") then it is recorded in AccumulatorRecursionInstr. If not, then 379 // we are handling the case when the return instruction returns a constant C 380 // which is different to the constant returned by other return instructions 381 // (which is recorded in AccumulatorRecursionEliminationInitVal). This is a 382 // special case of accumulator recursion, the operation being "return C". 383 Value *AccumulatorRecursionEliminationInitVal = 0; 384 Instruction *AccumulatorRecursionInstr = 0; 385 386 // Ok, we found a potential tail call. We can currently only transform the 387 // tail call if all of the instructions between the call and the return are 388 // movable to above the call itself, leaving the call next to the return. 389 // Check that this is the case now. 390 for (BBI = CI, ++BBI; &*BBI != Ret; ++BBI) { 391 if (CanMoveAboveCall(BBI, CI)) continue; 392 393 // If we can't move the instruction above the call, it might be because it 394 // is an associative and commutative operation that could be tranformed 395 // using accumulator recursion elimination. Check to see if this is the 396 // case, and if so, remember the initial accumulator value for later. 397 if ((AccumulatorRecursionEliminationInitVal = 398 CanTransformAccumulatorRecursion(BBI, CI))) { 399 // Yes, this is accumulator recursion. Remember which instruction 400 // accumulates. 401 AccumulatorRecursionInstr = BBI; 402 } else { 403 return false; // Otherwise, we cannot eliminate the tail recursion! 404 } 405 } 406 407 // We can only transform call/return pairs that either ignore the return value 408 // of the call and return void, ignore the value of the call and return a 409 // constant, return the value returned by the tail call, or that are being 410 // accumulator recursion variable eliminated. 411 if (Ret->getNumOperands() == 1 && Ret->getReturnValue() != CI && 412 !isa<UndefValue>(Ret->getReturnValue()) && 413 AccumulatorRecursionEliminationInitVal == 0 && 414 !getCommonReturnValue(0, CI)) { 415 // One case remains that we are able to handle: the current return 416 // instruction returns a constant, and all other return instructions 417 // return a different constant. 418 if (!isDynamicConstant(Ret->getReturnValue(), CI, Ret)) 419 return false; // Current return instruction does not return a constant. 420 // Check that all other return instructions return a common constant. If 421 // so, record it in AccumulatorRecursionEliminationInitVal. 422 AccumulatorRecursionEliminationInitVal = getCommonReturnValue(Ret, CI); 423 if (!AccumulatorRecursionEliminationInitVal) 424 return false; 425 } 426 427 // OK! We can transform this tail call. If this is the first one found, 428 // create the new entry block, allowing us to branch back to the old entry. 429 if (OldEntry == 0) { 430 OldEntry = &F->getEntryBlock(); 431 BasicBlock *NewEntry = BasicBlock::Create(F->getContext(), "", F, OldEntry); 432 NewEntry->takeName(OldEntry); 433 OldEntry->setName("tailrecurse"); 434 BranchInst::Create(OldEntry, NewEntry); 435 436 // If this tail call is marked 'tail' and if there are any allocas in the 437 // entry block, move them up to the new entry block. 438 TailCallsAreMarkedTail = CI->isTailCall(); 439 if (TailCallsAreMarkedTail) 440 // Move all fixed sized allocas from OldEntry to NewEntry. 441 for (BasicBlock::iterator OEBI = OldEntry->begin(), E = OldEntry->end(), 442 NEBI = NewEntry->begin(); OEBI != E; ) 443 if (AllocaInst *AI = dyn_cast<AllocaInst>(OEBI++)) 444 if (isa<ConstantInt>(AI->getArraySize())) 445 AI->moveBefore(NEBI); 446 447 // Now that we have created a new block, which jumps to the entry 448 // block, insert a PHI node for each argument of the function. 449 // For now, we initialize each PHI to only have the real arguments 450 // which are passed in. 451 Instruction *InsertPos = OldEntry->begin(); 452 for (Function::arg_iterator I = F->arg_begin(), E = F->arg_end(); 453 I != E; ++I) { 454 PHINode *PN = PHINode::Create(I->getType(), 455 I->getName() + ".tr", InsertPos); 456 I->replaceAllUsesWith(PN); // Everyone use the PHI node now! 457 PN->addIncoming(I, NewEntry); 458 ArgumentPHIs.push_back(PN); 459 } 460 } 461 462 // If this function has self recursive calls in the tail position where some 463 // are marked tail and some are not, only transform one flavor or another. We 464 // have to choose whether we move allocas in the entry block to the new entry 465 // block or not, so we can't make a good choice for both. NOTE: We could do 466 // slightly better here in the case that the function has no entry block 467 // allocas. 468 if (TailCallsAreMarkedTail && !CI->isTailCall()) 469 return false; 470 471 // Ok, now that we know we have a pseudo-entry block WITH all of the 472 // required PHI nodes, add entries into the PHI node for the actual 473 // parameters passed into the tail-recursive call. 474 for (unsigned i = 0, e = CI->getNumArgOperands(); i != e; ++i) 475 ArgumentPHIs[i]->addIncoming(CI->getArgOperand(i), BB); 476 477 // If we are introducing an accumulator variable to eliminate the recursion, 478 // do so now. Note that we _know_ that no subsequent tail recursion 479 // eliminations will happen on this function because of the way the 480 // accumulator recursion predicate is set up. 481 // 482 if (AccumulatorRecursionEliminationInitVal) { 483 Instruction *AccRecInstr = AccumulatorRecursionInstr; 484 // Start by inserting a new PHI node for the accumulator. 485 PHINode *AccPN = 486 PHINode::Create(AccumulatorRecursionEliminationInitVal->getType(), 487 "accumulator.tr", OldEntry->begin()); 488 489 // Loop over all of the predecessors of the tail recursion block. For the 490 // real entry into the function we seed the PHI with the initial value, 491 // computed earlier. For any other existing branches to this block (due to 492 // other tail recursions eliminated) the accumulator is not modified. 493 // Because we haven't added the branch in the current block to OldEntry yet, 494 // it will not show up as a predecessor. 495 for (pred_iterator PI = pred_begin(OldEntry), PE = pred_end(OldEntry); 496 PI != PE; ++PI) { 497 BasicBlock *P = *PI; 498 if (P == &F->getEntryBlock()) 499 AccPN->addIncoming(AccumulatorRecursionEliminationInitVal, P); 500 else 501 AccPN->addIncoming(AccPN, P); 502 } 503 504 if (AccRecInstr) { 505 // Add an incoming argument for the current block, which is computed by 506 // our associative and commutative accumulator instruction. 507 AccPN->addIncoming(AccRecInstr, BB); 508 509 // Next, rewrite the accumulator recursion instruction so that it does not 510 // use the result of the call anymore, instead, use the PHI node we just 511 // inserted. 512 AccRecInstr->setOperand(AccRecInstr->getOperand(0) != CI, AccPN); 513 } else { 514 // Add an incoming argument for the current block, which is just the 515 // constant returned by the current return instruction. 516 AccPN->addIncoming(Ret->getReturnValue(), BB); 517 } 518 519 // Finally, rewrite any return instructions in the program to return the PHI 520 // node instead of the "initval" that they do currently. This loop will 521 // actually rewrite the return value we are destroying, but that's ok. 522 for (Function::iterator BBI = F->begin(), E = F->end(); BBI != E; ++BBI) 523 if (ReturnInst *RI = dyn_cast<ReturnInst>(BBI->getTerminator())) 524 RI->setOperand(0, AccPN); 525 ++NumAccumAdded; 526 } 527 528 // Now that all of the PHI nodes are in place, remove the call and 529 // ret instructions, replacing them with an unconditional branch. 530 BranchInst::Create(OldEntry, Ret); 531 BB->getInstList().erase(Ret); // Remove return. 532 BB->getInstList().erase(CI); // Remove call. 533 ++NumEliminated; 534 return true; 535} 536