Inliner.cpp revision 54970c032815edadb1b2988ea33f5a1173e5b29c
1cf5933a716e7eb6bd5ff49aa62f3e76379ebaf51Chris Lattner//===- Inliner.cpp - Code common to all inliners --------------------------===// 2b576c94c15af9a440f69d9d03c2afead7971118cJohn Criswell// 3b576c94c15af9a440f69d9d03c2afead7971118cJohn Criswell// The LLVM Compiler Infrastructure 4b576c94c15af9a440f69d9d03c2afead7971118cJohn Criswell// 5b576c94c15af9a440f69d9d03c2afead7971118cJohn Criswell// This file was developed by the LLVM research group and is distributed under 6b576c94c15af9a440f69d9d03c2afead7971118cJohn Criswell// the University of Illinois Open Source License. See LICENSE.TXT for details. 7b576c94c15af9a440f69d9d03c2afead7971118cJohn Criswell// 8b576c94c15af9a440f69d9d03c2afead7971118cJohn Criswell//===----------------------------------------------------------------------===// 9237ef567f6764f24a47c63121cc0a599ddc8f56dChris Lattner// 10befa499d45ffcc32bd9902518aec18589464e47cChris Lattner// This file implements the mechanics required to implement inlining without 11befa499d45ffcc32bd9902518aec18589464e47cChris Lattner// missing any calls and updating the call graph. The decisions of which calls 12befa499d45ffcc32bd9902518aec18589464e47cChris Lattner// are profitable to inline are implemented elsewhere. 13237ef567f6764f24a47c63121cc0a599ddc8f56dChris Lattner// 14237ef567f6764f24a47c63121cc0a599ddc8f56dChris Lattner//===----------------------------------------------------------------------===// 15237ef567f6764f24a47c63121cc0a599ddc8f56dChris Lattner 16237ef567f6764f24a47c63121cc0a599ddc8f56dChris Lattner#include "Inliner.h" 17237ef567f6764f24a47c63121cc0a599ddc8f56dChris Lattner#include "llvm/Module.h" 1847b14a4a6a455c7be169cfd312fcbe796f0ad426Misha Brukman#include "llvm/Instructions.h" 19237ef567f6764f24a47c63121cc0a599ddc8f56dChris Lattner#include "llvm/Analysis/CallGraph.h" 20237ef567f6764f24a47c63121cc0a599ddc8f56dChris Lattner#include "llvm/Support/CallSite.h" 21237ef567f6764f24a47c63121cc0a599ddc8f56dChris Lattner#include "llvm/Transforms/Utils/Cloning.h" 22237ef567f6764f24a47c63121cc0a599ddc8f56dChris Lattner#include "Support/CommandLine.h" 23237ef567f6764f24a47c63121cc0a599ddc8f56dChris Lattner#include "Support/Debug.h" 24237ef567f6764f24a47c63121cc0a599ddc8f56dChris Lattner#include "Support/Statistic.h" 25befa499d45ffcc32bd9902518aec18589464e47cChris Lattner#include <set> 26a51bcb50b0c74adc741361824ef81dbefb715c53Chris Lattnerusing namespace llvm; 27d0fde30ce850b78371fd1386338350591f9ff494Brian Gaeke 28237ef567f6764f24a47c63121cc0a599ddc8f56dChris Lattnernamespace { 29237ef567f6764f24a47c63121cc0a599ddc8f56dChris Lattner Statistic<> NumInlined("inline", "Number of functions inlined"); 30237ef567f6764f24a47c63121cc0a599ddc8f56dChris Lattner Statistic<> NumDeleted("inline", "Number of functions deleted because all callers found"); 31237ef567f6764f24a47c63121cc0a599ddc8f56dChris Lattner cl::opt<unsigned> // FIXME: 200 is VERY conservative 32237ef567f6764f24a47c63121cc0a599ddc8f56dChris Lattner InlineLimit("inline-threshold", cl::Hidden, cl::init(200), 33237ef567f6764f24a47c63121cc0a599ddc8f56dChris Lattner cl::desc("Control the amount of inlining to perform (default = 200)")); 34237ef567f6764f24a47c63121cc0a599ddc8f56dChris Lattner} 35237ef567f6764f24a47c63121cc0a599ddc8f56dChris Lattner 36237ef567f6764f24a47c63121cc0a599ddc8f56dChris LattnerInliner::Inliner() : InlineThreshold(InlineLimit) {} 37237ef567f6764f24a47c63121cc0a599ddc8f56dChris Lattner 38befa499d45ffcc32bd9902518aec18589464e47cChris Lattner// InlineCallIfPossible - If it is possible to inline the specified call site, 39befa499d45ffcc32bd9902518aec18589464e47cChris Lattner// do so and update the CallGraph for this operation. 40befa499d45ffcc32bd9902518aec18589464e47cChris Lattnerstatic bool InlineCallIfPossible(CallSite CS, CallGraph &CG, 41befa499d45ffcc32bd9902518aec18589464e47cChris Lattner const std::set<Function*> &SCCFunctions) { 42befa499d45ffcc32bd9902518aec18589464e47cChris Lattner Function *Caller = CS.getInstruction()->getParent()->getParent(); 43befa499d45ffcc32bd9902518aec18589464e47cChris Lattner Function *Callee = CS.getCalledFunction(); 44befa499d45ffcc32bd9902518aec18589464e47cChris Lattner if (!InlineFunction(CS)) return false; 45befa499d45ffcc32bd9902518aec18589464e47cChris Lattner 46befa499d45ffcc32bd9902518aec18589464e47cChris Lattner // Update the call graph by deleting the edge from Callee to Caller 47befa499d45ffcc32bd9902518aec18589464e47cChris Lattner CallGraphNode *CalleeNode = CG[Callee]; 48befa499d45ffcc32bd9902518aec18589464e47cChris Lattner CallGraphNode *CallerNode = CG[Caller]; 49befa499d45ffcc32bd9902518aec18589464e47cChris Lattner CallerNode->removeCallEdgeTo(CalleeNode); 50befa499d45ffcc32bd9902518aec18589464e47cChris Lattner 51befa499d45ffcc32bd9902518aec18589464e47cChris Lattner // Since we inlined all uninlined call sites in the callee into the caller, 52befa499d45ffcc32bd9902518aec18589464e47cChris Lattner // add edges from the caller to all of the callees of the callee. 53befa499d45ffcc32bd9902518aec18589464e47cChris Lattner for (CallGraphNode::iterator I = CalleeNode->begin(), 54befa499d45ffcc32bd9902518aec18589464e47cChris Lattner E = CalleeNode->end(); I != E; ++I) 55befa499d45ffcc32bd9902518aec18589464e47cChris Lattner CallerNode->addCalledFunction(*I); 56befa499d45ffcc32bd9902518aec18589464e47cChris Lattner 5754970c032815edadb1b2988ea33f5a1173e5b29cChris Lattner // If we inlined the last possible call site to the function, delete the 5854970c032815edadb1b2988ea33f5a1173e5b29cChris Lattner // function body now. 59befa499d45ffcc32bd9902518aec18589464e47cChris Lattner if (Callee->use_empty() && Callee->hasInternalLinkage() && 60befa499d45ffcc32bd9902518aec18589464e47cChris Lattner !SCCFunctions.count(Callee)) { 61befa499d45ffcc32bd9902518aec18589464e47cChris Lattner DEBUG(std::cerr << " -> Deleting dead function: " 62befa499d45ffcc32bd9902518aec18589464e47cChris Lattner << Callee->getName() << "\n"); 63befa499d45ffcc32bd9902518aec18589464e47cChris Lattner 64befa499d45ffcc32bd9902518aec18589464e47cChris Lattner // Remove any call graph edges from the callee to its callees. 65befa499d45ffcc32bd9902518aec18589464e47cChris Lattner while (CalleeNode->begin() != CalleeNode->end()) 66befa499d45ffcc32bd9902518aec18589464e47cChris Lattner CalleeNode->removeCallEdgeTo(*(CalleeNode->end()-1)); 6754970c032815edadb1b2988ea33f5a1173e5b29cChris Lattner 68befa499d45ffcc32bd9902518aec18589464e47cChris Lattner // Removing the node for callee from the call graph and delete it. 69befa499d45ffcc32bd9902518aec18589464e47cChris Lattner delete CG.removeFunctionFromModule(CalleeNode); 70befa499d45ffcc32bd9902518aec18589464e47cChris Lattner ++NumDeleted; 71befa499d45ffcc32bd9902518aec18589464e47cChris Lattner } 72befa499d45ffcc32bd9902518aec18589464e47cChris Lattner return true; 73237ef567f6764f24a47c63121cc0a599ddc8f56dChris Lattner} 74237ef567f6764f24a47c63121cc0a599ddc8f56dChris Lattner 75237ef567f6764f24a47c63121cc0a599ddc8f56dChris Lattnerbool Inliner::runOnSCC(const std::vector<CallGraphNode*> &SCC) { 76237ef567f6764f24a47c63121cc0a599ddc8f56dChris Lattner CallGraph &CG = getAnalysis<CallGraph>(); 77237ef567f6764f24a47c63121cc0a599ddc8f56dChris Lattner 78237ef567f6764f24a47c63121cc0a599ddc8f56dChris Lattner std::set<Function*> SCCFunctions; 79237ef567f6764f24a47c63121cc0a599ddc8f56dChris Lattner DEBUG(std::cerr << "Inliner visiting SCC:"); 80237ef567f6764f24a47c63121cc0a599ddc8f56dChris Lattner for (unsigned i = 0, e = SCC.size(); i != e; ++i) { 81befa499d45ffcc32bd9902518aec18589464e47cChris Lattner Function *F = SCC[i]->getFunction(); 82befa499d45ffcc32bd9902518aec18589464e47cChris Lattner if (F) SCCFunctions.insert(F); 83befa499d45ffcc32bd9902518aec18589464e47cChris Lattner DEBUG(std::cerr << " " << (F ? F->getName() : "INDIRECTNODE")); 84237ef567f6764f24a47c63121cc0a599ddc8f56dChris Lattner } 85237ef567f6764f24a47c63121cc0a599ddc8f56dChris Lattner 86befa499d45ffcc32bd9902518aec18589464e47cChris Lattner // Scan through and identify all call sites ahead of time so that we only 87befa499d45ffcc32bd9902518aec18589464e47cChris Lattner // inline call sites in the original functions, not call sites that result 88befa499d45ffcc32bd9902518aec18589464e47cChris Lattner // from inlining other functions. 89befa499d45ffcc32bd9902518aec18589464e47cChris Lattner std::vector<CallSite> CallSites; 90befa499d45ffcc32bd9902518aec18589464e47cChris Lattner 91cf5933a716e7eb6bd5ff49aa62f3e76379ebaf51Chris Lattner for (unsigned i = 0, e = SCC.size(); i != e; ++i) 92cf5933a716e7eb6bd5ff49aa62f3e76379ebaf51Chris Lattner if (Function *F = SCC[i]->getFunction()) 93befa499d45ffcc32bd9902518aec18589464e47cChris Lattner for (Function::iterator BB = F->begin(), E = F->end(); BB != E; ++BB) 94befa499d45ffcc32bd9902518aec18589464e47cChris Lattner for (BasicBlock::iterator I = BB->begin(); I != BB->end(); ++I) { 95befa499d45ffcc32bd9902518aec18589464e47cChris Lattner CallSite CS = CallSite::get(I); 96befa499d45ffcc32bd9902518aec18589464e47cChris Lattner if (CS.getInstruction() && (!CS.getCalledFunction() || 97befa499d45ffcc32bd9902518aec18589464e47cChris Lattner !CS.getCalledFunction()->isExternal())) 98befa499d45ffcc32bd9902518aec18589464e47cChris Lattner CallSites.push_back(CS); 99befa499d45ffcc32bd9902518aec18589464e47cChris Lattner } 100237ef567f6764f24a47c63121cc0a599ddc8f56dChris Lattner 101befa499d45ffcc32bd9902518aec18589464e47cChris Lattner DEBUG(std::cerr << ": " << CallSites.size() << " call sites.\n"); 102befa499d45ffcc32bd9902518aec18589464e47cChris Lattner 103befa499d45ffcc32bd9902518aec18589464e47cChris Lattner // Now that we have all of the call sites, move the ones to functions in the 104befa499d45ffcc32bd9902518aec18589464e47cChris Lattner // current SCC to the end of the list. 105befa499d45ffcc32bd9902518aec18589464e47cChris Lattner unsigned FirstCallInSCC = CallSites.size(); 106befa499d45ffcc32bd9902518aec18589464e47cChris Lattner for (unsigned i = 0; i < FirstCallInSCC; ++i) 107befa499d45ffcc32bd9902518aec18589464e47cChris Lattner if (Function *F = CallSites[i].getCalledFunction()) 108befa499d45ffcc32bd9902518aec18589464e47cChris Lattner if (SCCFunctions.count(F)) 109befa499d45ffcc32bd9902518aec18589464e47cChris Lattner std::swap(CallSites[i--], CallSites[--FirstCallInSCC]); 110befa499d45ffcc32bd9902518aec18589464e47cChris Lattner 111befa499d45ffcc32bd9902518aec18589464e47cChris Lattner // Now that we have all of the call sites, loop over them and inline them if 112befa499d45ffcc32bd9902518aec18589464e47cChris Lattner // it looks profitable to do so. 113befa499d45ffcc32bd9902518aec18589464e47cChris Lattner bool Changed = false; 114befa499d45ffcc32bd9902518aec18589464e47cChris Lattner bool LocalChange; 115befa499d45ffcc32bd9902518aec18589464e47cChris Lattner do { 116befa499d45ffcc32bd9902518aec18589464e47cChris Lattner LocalChange = false; 117befa499d45ffcc32bd9902518aec18589464e47cChris Lattner // Iterate over the outer loop because inlining functions can cause indirect 118befa499d45ffcc32bd9902518aec18589464e47cChris Lattner // calls to become direct calls. 119befa499d45ffcc32bd9902518aec18589464e47cChris Lattner for (unsigned CSi = 0; CSi != CallSites.size(); ++CSi) 120befa499d45ffcc32bd9902518aec18589464e47cChris Lattner if (Function *Callee = CallSites[CSi].getCalledFunction()) { 121befa499d45ffcc32bd9902518aec18589464e47cChris Lattner // Calls to external functions are never inlinable. 122befa499d45ffcc32bd9902518aec18589464e47cChris Lattner if (Callee->isExternal() || 123befa499d45ffcc32bd9902518aec18589464e47cChris Lattner CallSites[CSi].getInstruction()->getParent()->getParent() ==Callee){ 124befa499d45ffcc32bd9902518aec18589464e47cChris Lattner std::swap(CallSites[CSi], CallSites.back()); 125b11a99bd39ec12733527bde330302b435b8d9f1aChris Lattner CallSites.pop_back(); 126befa499d45ffcc32bd9902518aec18589464e47cChris Lattner --CSi; 127befa499d45ffcc32bd9902518aec18589464e47cChris Lattner continue; 128befa499d45ffcc32bd9902518aec18589464e47cChris Lattner } 129befa499d45ffcc32bd9902518aec18589464e47cChris Lattner 130befa499d45ffcc32bd9902518aec18589464e47cChris Lattner // If the policy determines that we should inline this function, 131befa499d45ffcc32bd9902518aec18589464e47cChris Lattner // try to do so. 132befa499d45ffcc32bd9902518aec18589464e47cChris Lattner CallSite CS = CallSites[CSi]; 133befa499d45ffcc32bd9902518aec18589464e47cChris Lattner int InlineCost = getInlineCost(CS); 134befa499d45ffcc32bd9902518aec18589464e47cChris Lattner if (InlineCost >= (int)InlineThreshold) { 135befa499d45ffcc32bd9902518aec18589464e47cChris Lattner DEBUG(std::cerr << " NOT Inlining: cost=" << InlineCost 136befa499d45ffcc32bd9902518aec18589464e47cChris Lattner << ", Call: " << *CS.getInstruction()); 137befa499d45ffcc32bd9902518aec18589464e47cChris Lattner } else { 138befa499d45ffcc32bd9902518aec18589464e47cChris Lattner DEBUG(std::cerr << " Inlining: cost=" << InlineCost 139befa499d45ffcc32bd9902518aec18589464e47cChris Lattner << ", Call: " << *CS.getInstruction()); 140befa499d45ffcc32bd9902518aec18589464e47cChris Lattner 141befa499d45ffcc32bd9902518aec18589464e47cChris Lattner Function *Caller = CS.getInstruction()->getParent()->getParent(); 142befa499d45ffcc32bd9902518aec18589464e47cChris Lattner 143befa499d45ffcc32bd9902518aec18589464e47cChris Lattner // Attempt to inline the function... 144befa499d45ffcc32bd9902518aec18589464e47cChris Lattner if (InlineCallIfPossible(CS, CG, SCCFunctions)) { 145befa499d45ffcc32bd9902518aec18589464e47cChris Lattner // Remove this call site from the list. 146befa499d45ffcc32bd9902518aec18589464e47cChris Lattner std::swap(CallSites[CSi], CallSites.back()); 147befa499d45ffcc32bd9902518aec18589464e47cChris Lattner CallSites.pop_back(); 148befa499d45ffcc32bd9902518aec18589464e47cChris Lattner --CSi; 149befa499d45ffcc32bd9902518aec18589464e47cChris Lattner 150befa499d45ffcc32bd9902518aec18589464e47cChris Lattner ++NumInlined; 151befa499d45ffcc32bd9902518aec18589464e47cChris Lattner Changed = true; 152befa499d45ffcc32bd9902518aec18589464e47cChris Lattner LocalChange = true; 153775cbdd51a3b33dd5eb343689f65ab5cc8ac7118Chris Lattner } 154237ef567f6764f24a47c63121cc0a599ddc8f56dChris Lattner } 155237ef567f6764f24a47c63121cc0a599ddc8f56dChris Lattner } 156befa499d45ffcc32bd9902518aec18589464e47cChris Lattner } while (LocalChange); 157237ef567f6764f24a47c63121cc0a599ddc8f56dChris Lattner 158775cbdd51a3b33dd5eb343689f65ab5cc8ac7118Chris Lattner return Changed; 159237ef567f6764f24a47c63121cc0a599ddc8f56dChris Lattner} 160d0fde30ce850b78371fd1386338350591f9ff494Brian Gaeke 16168d57e7ae80044401efd889270a12c71b3efb9abChris Lattner// doFinalization - Remove now-dead linkonce functions at the end of 16268d57e7ae80044401efd889270a12c71b3efb9abChris Lattner// processing to avoid breaking the SCC traversal. 16368d57e7ae80044401efd889270a12c71b3efb9abChris Lattnerbool Inliner::doFinalization(CallGraph &CG) { 1643e1358a9fa1ebd3f51c94eb69da55d693895fe7cChris Lattner std::set<CallGraphNode*> FunctionsToRemove; 1653e1358a9fa1ebd3f51c94eb69da55d693895fe7cChris Lattner 1663e1358a9fa1ebd3f51c94eb69da55d693895fe7cChris Lattner // Scan for all of the functions, looking for ones that should now be removed 1673e1358a9fa1ebd3f51c94eb69da55d693895fe7cChris Lattner // from the program. Insert the dead ones in the FunctionsToRemove set. 1683e1358a9fa1ebd3f51c94eb69da55d693895fe7cChris Lattner for (CallGraph::iterator I = CG.begin(), E = CG.end(); I != E; ++I) { 1693e1358a9fa1ebd3f51c94eb69da55d693895fe7cChris Lattner CallGraphNode *CGN = I->second; 17054970c032815edadb1b2988ea33f5a1173e5b29cChris Lattner if (Function *F = CGN ? CGN->getFunction() : 0) { 17154970c032815edadb1b2988ea33f5a1173e5b29cChris Lattner // If the only remaining users of the function are dead constants, 17254970c032815edadb1b2988ea33f5a1173e5b29cChris Lattner // remove them. 17354970c032815edadb1b2988ea33f5a1173e5b29cChris Lattner bool HadDeadConstantUsers = !F->use_empty(); 17454970c032815edadb1b2988ea33f5a1173e5b29cChris Lattner F->removeDeadConstantUsers(); 17554970c032815edadb1b2988ea33f5a1173e5b29cChris Lattner 17654970c032815edadb1b2988ea33f5a1173e5b29cChris Lattner if ((F->hasLinkOnceLinkage() || F->hasInternalLinkage()) && 17754970c032815edadb1b2988ea33f5a1173e5b29cChris Lattner F->use_empty()) { 17854970c032815edadb1b2988ea33f5a1173e5b29cChris Lattner // Remove any call graph edges from the function to its callees. 17954970c032815edadb1b2988ea33f5a1173e5b29cChris Lattner while (CGN->begin() != CGN->end()) 18054970c032815edadb1b2988ea33f5a1173e5b29cChris Lattner CGN->removeCallEdgeTo(*(CGN->end()-1)); 18154970c032815edadb1b2988ea33f5a1173e5b29cChris Lattner 18254970c032815edadb1b2988ea33f5a1173e5b29cChris Lattner // If the function has external linkage (basically if it's a linkonce 18354970c032815edadb1b2988ea33f5a1173e5b29cChris Lattner // function) remove the edge from the external node to the callee 18454970c032815edadb1b2988ea33f5a1173e5b29cChris Lattner // node. 18554970c032815edadb1b2988ea33f5a1173e5b29cChris Lattner if (!F->hasInternalLinkage() || HadDeadConstantUsers) 18654970c032815edadb1b2988ea33f5a1173e5b29cChris Lattner CG.getExternalCallingNode()->removeCallEdgeTo(CGN); 18754970c032815edadb1b2988ea33f5a1173e5b29cChris Lattner 18854970c032815edadb1b2988ea33f5a1173e5b29cChris Lattner // Removing the node for callee from the call graph and delete it. 18954970c032815edadb1b2988ea33f5a1173e5b29cChris Lattner FunctionsToRemove.insert(CGN); 19054970c032815edadb1b2988ea33f5a1173e5b29cChris Lattner } 19168d57e7ae80044401efd889270a12c71b3efb9abChris Lattner } 19268d57e7ae80044401efd889270a12c71b3efb9abChris Lattner } 1933e1358a9fa1ebd3f51c94eb69da55d693895fe7cChris Lattner 1943e1358a9fa1ebd3f51c94eb69da55d693895fe7cChris Lattner // Now that we know which functions to delete, do so. We didn't want to do 1953e1358a9fa1ebd3f51c94eb69da55d693895fe7cChris Lattner // this inline, because that would invalidate our CallGraph::iterator 1963e1358a9fa1ebd3f51c94eb69da55d693895fe7cChris Lattner // objects. :( 1973e1358a9fa1ebd3f51c94eb69da55d693895fe7cChris Lattner bool Changed = false; 1983e1358a9fa1ebd3f51c94eb69da55d693895fe7cChris Lattner for (std::set<CallGraphNode*>::iterator I = FunctionsToRemove.begin(), 1993e1358a9fa1ebd3f51c94eb69da55d693895fe7cChris Lattner E = FunctionsToRemove.end(); I != E; ++I) { 2003e1358a9fa1ebd3f51c94eb69da55d693895fe7cChris Lattner delete CG.removeFunctionFromModule(*I); 2013e1358a9fa1ebd3f51c94eb69da55d693895fe7cChris Lattner ++NumDeleted; 2023e1358a9fa1ebd3f51c94eb69da55d693895fe7cChris Lattner Changed = true; 2033e1358a9fa1ebd3f51c94eb69da55d693895fe7cChris Lattner } 2043e1358a9fa1ebd3f51c94eb69da55d693895fe7cChris Lattner 20568d57e7ae80044401efd889270a12c71b3efb9abChris Lattner return Changed; 20668d57e7ae80044401efd889270a12c71b3efb9abChris Lattner} 207