Inliner.cpp revision 5cbf985dcbc89fba3208e7baf8b6f488b06d3ec9
1cf5933a716e7eb6bd5ff49aa62f3e76379ebaf51Chris Lattner//===- Inliner.cpp - Code common to all inliners --------------------------===// 2fd93908ae8b9684fe71c239e3c6cfe13ff6a2663Misha Brukman// 3b576c94c15af9a440f69d9d03c2afead7971118cJohn Criswell// The LLVM Compiler Infrastructure 4b576c94c15af9a440f69d9d03c2afead7971118cJohn Criswell// 5b576c94c15af9a440f69d9d03c2afead7971118cJohn Criswell// This file was developed by the LLVM research group and is distributed under 6b576c94c15af9a440f69d9d03c2afead7971118cJohn Criswell// the University of Illinois Open Source License. See LICENSE.TXT for details. 7fd93908ae8b9684fe71c239e3c6cfe13ff6a2663Misha Brukman// 8b576c94c15af9a440f69d9d03c2afead7971118cJohn Criswell//===----------------------------------------------------------------------===// 9237ef567f6764f24a47c63121cc0a599ddc8f56dChris Lattner// 10befa499d45ffcc32bd9902518aec18589464e47cChris Lattner// This file implements the mechanics required to implement inlining without 11befa499d45ffcc32bd9902518aec18589464e47cChris Lattner// missing any calls and updating the call graph. The decisions of which calls 12befa499d45ffcc32bd9902518aec18589464e47cChris Lattner// are profitable to inline are implemented elsewhere. 13237ef567f6764f24a47c63121cc0a599ddc8f56dChris Lattner// 14237ef567f6764f24a47c63121cc0a599ddc8f56dChris Lattner//===----------------------------------------------------------------------===// 15237ef567f6764f24a47c63121cc0a599ddc8f56dChris Lattner 1686453c52ba02e743d29c08456e51006500041456Chris Lattner#define DEBUG_TYPE "inline" 17237ef567f6764f24a47c63121cc0a599ddc8f56dChris Lattner#include "Inliner.h" 18237ef567f6764f24a47c63121cc0a599ddc8f56dChris Lattner#include "llvm/Module.h" 1947b14a4a6a455c7be169cfd312fcbe796f0ad426Misha Brukman#include "llvm/Instructions.h" 20237ef567f6764f24a47c63121cc0a599ddc8f56dChris Lattner#include "llvm/Analysis/CallGraph.h" 21237ef567f6764f24a47c63121cc0a599ddc8f56dChris Lattner#include "llvm/Support/CallSite.h" 22237ef567f6764f24a47c63121cc0a599ddc8f56dChris Lattner#include "llvm/Transforms/Utils/Cloning.h" 23551ccae044b0ff658fe629dd67edd5ffe75d10e8Reid Spencer#include "llvm/Support/CommandLine.h" 24551ccae044b0ff658fe629dd67edd5ffe75d10e8Reid Spencer#include "llvm/Support/Debug.h" 25551ccae044b0ff658fe629dd67edd5ffe75d10e8Reid Spencer#include "llvm/ADT/Statistic.h" 26befa499d45ffcc32bd9902518aec18589464e47cChris Lattner#include <set> 27a51bcb50b0c74adc741361824ef81dbefb715c53Chris Lattnerusing namespace llvm; 28d0fde30ce850b78371fd1386338350591f9ff494Brian Gaeke 2986453c52ba02e743d29c08456e51006500041456Chris LattnerSTATISTIC(NumInlined, "Number of functions inlined"); 3086453c52ba02e743d29c08456e51006500041456Chris LattnerSTATISTIC(NumDeleted, "Number of functions deleted because all callers found"); 3186453c52ba02e743d29c08456e51006500041456Chris Lattner 32237ef567f6764f24a47c63121cc0a599ddc8f56dChris Lattnernamespace { 33237ef567f6764f24a47c63121cc0a599ddc8f56dChris Lattner cl::opt<unsigned> // FIXME: 200 is VERY conservative 34237ef567f6764f24a47c63121cc0a599ddc8f56dChris Lattner InlineLimit("inline-threshold", cl::Hidden, cl::init(200), 358acb249725e3304aafe91d5357f69722957c51b1Chris Lattner cl::desc("Control the amount of inlining to perform (default = 200)")); 36237ef567f6764f24a47c63121cc0a599ddc8f56dChris Lattner} 37237ef567f6764f24a47c63121cc0a599ddc8f56dChris Lattner 38237ef567f6764f24a47c63121cc0a599ddc8f56dChris LattnerInliner::Inliner() : InlineThreshold(InlineLimit) {} 39237ef567f6764f24a47c63121cc0a599ddc8f56dChris Lattner 40befa499d45ffcc32bd9902518aec18589464e47cChris Lattner// InlineCallIfPossible - If it is possible to inline the specified call site, 41befa499d45ffcc32bd9902518aec18589464e47cChris Lattner// do so and update the CallGraph for this operation. 42befa499d45ffcc32bd9902518aec18589464e47cChris Lattnerstatic bool InlineCallIfPossible(CallSite CS, CallGraph &CG, 43befa499d45ffcc32bd9902518aec18589464e47cChris Lattner const std::set<Function*> &SCCFunctions) { 44befa499d45ffcc32bd9902518aec18589464e47cChris Lattner Function *Callee = CS.getCalledFunction(); 45432a205769d22e7744dd8b0b45efd687aa6998e3Chris Lattner if (!InlineFunction(CS, &CG)) return false; 46fd93908ae8b9684fe71c239e3c6cfe13ff6a2663Misha Brukman 4754970c032815edadb1b2988ea33f5a1173e5b29cChris Lattner // If we inlined the last possible call site to the function, delete the 4854970c032815edadb1b2988ea33f5a1173e5b29cChris Lattner // function body now. 49befa499d45ffcc32bd9902518aec18589464e47cChris Lattner if (Callee->use_empty() && Callee->hasInternalLinkage() && 50befa499d45ffcc32bd9902518aec18589464e47cChris Lattner !SCCFunctions.count(Callee)) { 510a81aac4b46eed130d20714af5a1c01b05d0275eBill Wendling DOUT << " -> Deleting dead function: " << Callee->getName() << "\n"; 52fd93908ae8b9684fe71c239e3c6cfe13ff6a2663Misha Brukman 53befa499d45ffcc32bd9902518aec18589464e47cChris Lattner // Remove any call graph edges from the callee to its callees. 54432a205769d22e7744dd8b0b45efd687aa6998e3Chris Lattner CallGraphNode *CalleeNode = CG[Callee]; 55befa499d45ffcc32bd9902518aec18589464e47cChris Lattner while (CalleeNode->begin() != CalleeNode->end()) 56d85340f4ec587e22b0239617f3b747a6df113894Chris Lattner CalleeNode->removeCallEdgeTo((CalleeNode->end()-1)->second); 57fd93908ae8b9684fe71c239e3c6cfe13ff6a2663Misha Brukman 58befa499d45ffcc32bd9902518aec18589464e47cChris Lattner // Removing the node for callee from the call graph and delete it. 59befa499d45ffcc32bd9902518aec18589464e47cChris Lattner delete CG.removeFunctionFromModule(CalleeNode); 60befa499d45ffcc32bd9902518aec18589464e47cChris Lattner ++NumDeleted; 61befa499d45ffcc32bd9902518aec18589464e47cChris Lattner } 62befa499d45ffcc32bd9902518aec18589464e47cChris Lattner return true; 63237ef567f6764f24a47c63121cc0a599ddc8f56dChris Lattner} 64237ef567f6764f24a47c63121cc0a599ddc8f56dChris Lattner 65237ef567f6764f24a47c63121cc0a599ddc8f56dChris Lattnerbool Inliner::runOnSCC(const std::vector<CallGraphNode*> &SCC) { 66237ef567f6764f24a47c63121cc0a599ddc8f56dChris Lattner CallGraph &CG = getAnalysis<CallGraph>(); 67237ef567f6764f24a47c63121cc0a599ddc8f56dChris Lattner 68237ef567f6764f24a47c63121cc0a599ddc8f56dChris Lattner std::set<Function*> SCCFunctions; 690a81aac4b46eed130d20714af5a1c01b05d0275eBill Wendling DOUT << "Inliner visiting SCC:"; 70237ef567f6764f24a47c63121cc0a599ddc8f56dChris Lattner for (unsigned i = 0, e = SCC.size(); i != e; ++i) { 71befa499d45ffcc32bd9902518aec18589464e47cChris Lattner Function *F = SCC[i]->getFunction(); 72befa499d45ffcc32bd9902518aec18589464e47cChris Lattner if (F) SCCFunctions.insert(F); 730a81aac4b46eed130d20714af5a1c01b05d0275eBill Wendling DOUT << " " << (F ? F->getName() : "INDIRECTNODE"); 74237ef567f6764f24a47c63121cc0a599ddc8f56dChris Lattner } 75237ef567f6764f24a47c63121cc0a599ddc8f56dChris Lattner 76befa499d45ffcc32bd9902518aec18589464e47cChris Lattner // Scan through and identify all call sites ahead of time so that we only 77befa499d45ffcc32bd9902518aec18589464e47cChris Lattner // inline call sites in the original functions, not call sites that result 78befa499d45ffcc32bd9902518aec18589464e47cChris Lattner // from inlining other functions. 79befa499d45ffcc32bd9902518aec18589464e47cChris Lattner std::vector<CallSite> CallSites; 80befa499d45ffcc32bd9902518aec18589464e47cChris Lattner 81cf5933a716e7eb6bd5ff49aa62f3e76379ebaf51Chris Lattner for (unsigned i = 0, e = SCC.size(); i != e; ++i) 82cf5933a716e7eb6bd5ff49aa62f3e76379ebaf51Chris Lattner if (Function *F = SCC[i]->getFunction()) 83befa499d45ffcc32bd9902518aec18589464e47cChris Lattner for (Function::iterator BB = F->begin(), E = F->end(); BB != E; ++BB) 84befa499d45ffcc32bd9902518aec18589464e47cChris Lattner for (BasicBlock::iterator I = BB->begin(); I != BB->end(); ++I) { 85befa499d45ffcc32bd9902518aec18589464e47cChris Lattner CallSite CS = CallSite::get(I); 86befa499d45ffcc32bd9902518aec18589464e47cChris Lattner if (CS.getInstruction() && (!CS.getCalledFunction() || 875cbf985dcbc89fba3208e7baf8b6f488b06d3ec9Reid Spencer !CS.getCalledFunction()->isDeclaration())) 88befa499d45ffcc32bd9902518aec18589464e47cChris Lattner CallSites.push_back(CS); 89befa499d45ffcc32bd9902518aec18589464e47cChris Lattner } 90237ef567f6764f24a47c63121cc0a599ddc8f56dChris Lattner 910a81aac4b46eed130d20714af5a1c01b05d0275eBill Wendling DOUT << ": " << CallSites.size() << " call sites.\n"; 92fd93908ae8b9684fe71c239e3c6cfe13ff6a2663Misha Brukman 93befa499d45ffcc32bd9902518aec18589464e47cChris Lattner // Now that we have all of the call sites, move the ones to functions in the 94befa499d45ffcc32bd9902518aec18589464e47cChris Lattner // current SCC to the end of the list. 95befa499d45ffcc32bd9902518aec18589464e47cChris Lattner unsigned FirstCallInSCC = CallSites.size(); 96befa499d45ffcc32bd9902518aec18589464e47cChris Lattner for (unsigned i = 0; i < FirstCallInSCC; ++i) 97befa499d45ffcc32bd9902518aec18589464e47cChris Lattner if (Function *F = CallSites[i].getCalledFunction()) 98befa499d45ffcc32bd9902518aec18589464e47cChris Lattner if (SCCFunctions.count(F)) 99befa499d45ffcc32bd9902518aec18589464e47cChris Lattner std::swap(CallSites[i--], CallSites[--FirstCallInSCC]); 100fd93908ae8b9684fe71c239e3c6cfe13ff6a2663Misha Brukman 101befa499d45ffcc32bd9902518aec18589464e47cChris Lattner // Now that we have all of the call sites, loop over them and inline them if 102befa499d45ffcc32bd9902518aec18589464e47cChris Lattner // it looks profitable to do so. 103befa499d45ffcc32bd9902518aec18589464e47cChris Lattner bool Changed = false; 104befa499d45ffcc32bd9902518aec18589464e47cChris Lattner bool LocalChange; 105befa499d45ffcc32bd9902518aec18589464e47cChris Lattner do { 106befa499d45ffcc32bd9902518aec18589464e47cChris Lattner LocalChange = false; 107befa499d45ffcc32bd9902518aec18589464e47cChris Lattner // Iterate over the outer loop because inlining functions can cause indirect 108befa499d45ffcc32bd9902518aec18589464e47cChris Lattner // calls to become direct calls. 109befa499d45ffcc32bd9902518aec18589464e47cChris Lattner for (unsigned CSi = 0; CSi != CallSites.size(); ++CSi) 110befa499d45ffcc32bd9902518aec18589464e47cChris Lattner if (Function *Callee = CallSites[CSi].getCalledFunction()) { 111befa499d45ffcc32bd9902518aec18589464e47cChris Lattner // Calls to external functions are never inlinable. 1125cbf985dcbc89fba3208e7baf8b6f488b06d3ec9Reid Spencer if (Callee->isDeclaration() || 113befa499d45ffcc32bd9902518aec18589464e47cChris Lattner CallSites[CSi].getInstruction()->getParent()->getParent() ==Callee){ 11408ff1480ffcb22e946c7bb6c7d66c5d977ae3d6eChris Lattner if (SCC.size() == 1) { 11508ff1480ffcb22e946c7bb6c7d66c5d977ae3d6eChris Lattner std::swap(CallSites[CSi], CallSites.back()); 11608ff1480ffcb22e946c7bb6c7d66c5d977ae3d6eChris Lattner CallSites.pop_back(); 11708ff1480ffcb22e946c7bb6c7d66c5d977ae3d6eChris Lattner } else { 11808ff1480ffcb22e946c7bb6c7d66c5d977ae3d6eChris Lattner // Keep the 'in SCC / not in SCC' boundary correct. 11908ff1480ffcb22e946c7bb6c7d66c5d977ae3d6eChris Lattner CallSites.erase(CallSites.begin()+CSi); 12008ff1480ffcb22e946c7bb6c7d66c5d977ae3d6eChris Lattner } 121befa499d45ffcc32bd9902518aec18589464e47cChris Lattner --CSi; 122befa499d45ffcc32bd9902518aec18589464e47cChris Lattner continue; 123befa499d45ffcc32bd9902518aec18589464e47cChris Lattner } 124befa499d45ffcc32bd9902518aec18589464e47cChris Lattner 125befa499d45ffcc32bd9902518aec18589464e47cChris Lattner // If the policy determines that we should inline this function, 126befa499d45ffcc32bd9902518aec18589464e47cChris Lattner // try to do so. 127befa499d45ffcc32bd9902518aec18589464e47cChris Lattner CallSite CS = CallSites[CSi]; 128befa499d45ffcc32bd9902518aec18589464e47cChris Lattner int InlineCost = getInlineCost(CS); 129befa499d45ffcc32bd9902518aec18589464e47cChris Lattner if (InlineCost >= (int)InlineThreshold) { 1300a81aac4b46eed130d20714af5a1c01b05d0275eBill Wendling DOUT << " NOT Inlining: cost=" << InlineCost 1310a81aac4b46eed130d20714af5a1c01b05d0275eBill Wendling << ", Call: " << *CS.getInstruction(); 132befa499d45ffcc32bd9902518aec18589464e47cChris Lattner } else { 1330a81aac4b46eed130d20714af5a1c01b05d0275eBill Wendling DOUT << " Inlining: cost=" << InlineCost 1340a81aac4b46eed130d20714af5a1c01b05d0275eBill Wendling << ", Call: " << *CS.getInstruction(); 135fd93908ae8b9684fe71c239e3c6cfe13ff6a2663Misha Brukman 136befa499d45ffcc32bd9902518aec18589464e47cChris Lattner // Attempt to inline the function... 137befa499d45ffcc32bd9902518aec18589464e47cChris Lattner if (InlineCallIfPossible(CS, CG, SCCFunctions)) { 13808ff1480ffcb22e946c7bb6c7d66c5d977ae3d6eChris Lattner // Remove this call site from the list. If possible, use 13908ff1480ffcb22e946c7bb6c7d66c5d977ae3d6eChris Lattner // swap/pop_back for efficiency, but do not use it if doing so would 14008ff1480ffcb22e946c7bb6c7d66c5d977ae3d6eChris Lattner // move a call site to a function in this SCC before the 14108ff1480ffcb22e946c7bb6c7d66c5d977ae3d6eChris Lattner // 'FirstCallInSCC' barrier. 14208ff1480ffcb22e946c7bb6c7d66c5d977ae3d6eChris Lattner if (SCC.size() == 1) { 14308ff1480ffcb22e946c7bb6c7d66c5d977ae3d6eChris Lattner std::swap(CallSites[CSi], CallSites.back()); 14408ff1480ffcb22e946c7bb6c7d66c5d977ae3d6eChris Lattner CallSites.pop_back(); 14508ff1480ffcb22e946c7bb6c7d66c5d977ae3d6eChris Lattner } else { 14608ff1480ffcb22e946c7bb6c7d66c5d977ae3d6eChris Lattner CallSites.erase(CallSites.begin()+CSi); 14708ff1480ffcb22e946c7bb6c7d66c5d977ae3d6eChris Lattner } 148befa499d45ffcc32bd9902518aec18589464e47cChris Lattner --CSi; 149befa499d45ffcc32bd9902518aec18589464e47cChris Lattner 150befa499d45ffcc32bd9902518aec18589464e47cChris Lattner ++NumInlined; 151befa499d45ffcc32bd9902518aec18589464e47cChris Lattner Changed = true; 152befa499d45ffcc32bd9902518aec18589464e47cChris Lattner LocalChange = true; 153775cbdd51a3b33dd5eb343689f65ab5cc8ac7118Chris Lattner } 154237ef567f6764f24a47c63121cc0a599ddc8f56dChris Lattner } 155237ef567f6764f24a47c63121cc0a599ddc8f56dChris Lattner } 156befa499d45ffcc32bd9902518aec18589464e47cChris Lattner } while (LocalChange); 157237ef567f6764f24a47c63121cc0a599ddc8f56dChris Lattner 158775cbdd51a3b33dd5eb343689f65ab5cc8ac7118Chris Lattner return Changed; 159237ef567f6764f24a47c63121cc0a599ddc8f56dChris Lattner} 160d0fde30ce850b78371fd1386338350591f9ff494Brian Gaeke 16168d57e7ae80044401efd889270a12c71b3efb9abChris Lattner// doFinalization - Remove now-dead linkonce functions at the end of 16268d57e7ae80044401efd889270a12c71b3efb9abChris Lattner// processing to avoid breaking the SCC traversal. 16368d57e7ae80044401efd889270a12c71b3efb9abChris Lattnerbool Inliner::doFinalization(CallGraph &CG) { 1643e1358a9fa1ebd3f51c94eb69da55d693895fe7cChris Lattner std::set<CallGraphNode*> FunctionsToRemove; 1653e1358a9fa1ebd3f51c94eb69da55d693895fe7cChris Lattner 1663e1358a9fa1ebd3f51c94eb69da55d693895fe7cChris Lattner // Scan for all of the functions, looking for ones that should now be removed 1673e1358a9fa1ebd3f51c94eb69da55d693895fe7cChris Lattner // from the program. Insert the dead ones in the FunctionsToRemove set. 1683e1358a9fa1ebd3f51c94eb69da55d693895fe7cChris Lattner for (CallGraph::iterator I = CG.begin(), E = CG.end(); I != E; ++I) { 1693e1358a9fa1ebd3f51c94eb69da55d693895fe7cChris Lattner CallGraphNode *CGN = I->second; 17054970c032815edadb1b2988ea33f5a1173e5b29cChris Lattner if (Function *F = CGN ? CGN->getFunction() : 0) { 1710c0aa711b8a0550c21f032125c4663ff45864f81Chris Lattner // If the only remaining users of the function are dead constants, remove 1720c0aa711b8a0550c21f032125c4663ff45864f81Chris Lattner // them. 17354970c032815edadb1b2988ea33f5a1173e5b29cChris Lattner F->removeDeadConstantUsers(); 17454970c032815edadb1b2988ea33f5a1173e5b29cChris Lattner 17554970c032815edadb1b2988ea33f5a1173e5b29cChris Lattner if ((F->hasLinkOnceLinkage() || F->hasInternalLinkage()) && 17654970c032815edadb1b2988ea33f5a1173e5b29cChris Lattner F->use_empty()) { 1770c0aa711b8a0550c21f032125c4663ff45864f81Chris Lattner 17854970c032815edadb1b2988ea33f5a1173e5b29cChris Lattner // Remove any call graph edges from the function to its callees. 17954970c032815edadb1b2988ea33f5a1173e5b29cChris Lattner while (CGN->begin() != CGN->end()) 180d85340f4ec587e22b0239617f3b747a6df113894Chris Lattner CGN->removeCallEdgeTo((CGN->end()-1)->second); 181fd93908ae8b9684fe71c239e3c6cfe13ff6a2663Misha Brukman 1820c0aa711b8a0550c21f032125c4663ff45864f81Chris Lattner // Remove any edges from the external node to the function's call graph 1830c0aa711b8a0550c21f032125c4663ff45864f81Chris Lattner // node. These edges might have been made irrelegant due to 1840c0aa711b8a0550c21f032125c4663ff45864f81Chris Lattner // optimization of the program. 1850c0aa711b8a0550c21f032125c4663ff45864f81Chris Lattner CG.getExternalCallingNode()->removeAnyCallEdgeTo(CGN); 186fd93908ae8b9684fe71c239e3c6cfe13ff6a2663Misha Brukman 18754970c032815edadb1b2988ea33f5a1173e5b29cChris Lattner // Removing the node for callee from the call graph and delete it. 18854970c032815edadb1b2988ea33f5a1173e5b29cChris Lattner FunctionsToRemove.insert(CGN); 18954970c032815edadb1b2988ea33f5a1173e5b29cChris Lattner } 19068d57e7ae80044401efd889270a12c71b3efb9abChris Lattner } 19168d57e7ae80044401efd889270a12c71b3efb9abChris Lattner } 1923e1358a9fa1ebd3f51c94eb69da55d693895fe7cChris Lattner 1933e1358a9fa1ebd3f51c94eb69da55d693895fe7cChris Lattner // Now that we know which functions to delete, do so. We didn't want to do 1943e1358a9fa1ebd3f51c94eb69da55d693895fe7cChris Lattner // this inline, because that would invalidate our CallGraph::iterator 1953e1358a9fa1ebd3f51c94eb69da55d693895fe7cChris Lattner // objects. :( 1963e1358a9fa1ebd3f51c94eb69da55d693895fe7cChris Lattner bool Changed = false; 1973e1358a9fa1ebd3f51c94eb69da55d693895fe7cChris Lattner for (std::set<CallGraphNode*>::iterator I = FunctionsToRemove.begin(), 1983e1358a9fa1ebd3f51c94eb69da55d693895fe7cChris Lattner E = FunctionsToRemove.end(); I != E; ++I) { 1993e1358a9fa1ebd3f51c94eb69da55d693895fe7cChris Lattner delete CG.removeFunctionFromModule(*I); 2003e1358a9fa1ebd3f51c94eb69da55d693895fe7cChris Lattner ++NumDeleted; 2013e1358a9fa1ebd3f51c94eb69da55d693895fe7cChris Lattner Changed = true; 2023e1358a9fa1ebd3f51c94eb69da55d693895fe7cChris Lattner } 2033e1358a9fa1ebd3f51c94eb69da55d693895fe7cChris Lattner 20468d57e7ae80044401efd889270a12c71b3efb9abChris Lattner return Changed; 20568d57e7ae80044401efd889270a12c71b3efb9abChris Lattner} 206