SampleProfile.cpp revision 36b56886974eae4f9c5ebc96befd3e7bfe5de338
1//===- SampleProfile.cpp - Incorporate sample profiles into the IR --------===// 2// 3// The LLVM Compiler Infrastructure 4// 5// This file is distributed under the University of Illinois Open Source 6// License. See LICENSE.TXT for details. 7// 8//===----------------------------------------------------------------------===// 9// 10// This file implements the SampleProfileLoader transformation. This pass 11// reads a profile file generated by a sampling profiler (e.g. Linux Perf - 12// http://perf.wiki.kernel.org/) and generates IR metadata to reflect the 13// profile information in the given profile. 14// 15// This pass generates branch weight annotations on the IR: 16// 17// - prof: Represents branch weights. This annotation is added to branches 18// to indicate the weights of each edge coming out of the branch. 19// The weight of each edge is the weight of the target block for 20// that edge. The weight of a block B is computed as the maximum 21// number of samples found in B. 22// 23//===----------------------------------------------------------------------===// 24 25#define DEBUG_TYPE "sample-profile" 26 27#include "llvm/Transforms/Scalar.h" 28#include "llvm/ADT/DenseMap.h" 29#include "llvm/ADT/SmallPtrSet.h" 30#include "llvm/ADT/SmallSet.h" 31#include "llvm/ADT/StringMap.h" 32#include "llvm/ADT/StringRef.h" 33#include "llvm/Analysis/LoopInfo.h" 34#include "llvm/Analysis/PostDominators.h" 35#include "llvm/IR/Constants.h" 36#include "llvm/IR/DebugInfo.h" 37#include "llvm/IR/DiagnosticInfo.h" 38#include "llvm/IR/Dominators.h" 39#include "llvm/IR/Function.h" 40#include "llvm/IR/InstIterator.h" 41#include "llvm/IR/Instructions.h" 42#include "llvm/IR/LLVMContext.h" 43#include "llvm/IR/MDBuilder.h" 44#include "llvm/IR/Metadata.h" 45#include "llvm/IR/Module.h" 46#include "llvm/Pass.h" 47#include "llvm/Support/CommandLine.h" 48#include "llvm/Support/Debug.h" 49#include "llvm/Support/LineIterator.h" 50#include "llvm/Support/MemoryBuffer.h" 51#include "llvm/Support/Regex.h" 52#include "llvm/Support/raw_ostream.h" 53#include <cctype> 54 55using namespace llvm; 56 57// Command line option to specify the file to read samples from. This is 58// mainly used for debugging. 59static cl::opt<std::string> SampleProfileFile( 60 "sample-profile-file", cl::init(""), cl::value_desc("filename"), 61 cl::desc("Profile file loaded by -sample-profile"), cl::Hidden); 62static cl::opt<unsigned> SampleProfileMaxPropagateIterations( 63 "sample-profile-max-propagate-iterations", cl::init(100), 64 cl::desc("Maximum number of iterations to go through when propagating " 65 "sample block/edge weights through the CFG.")); 66 67namespace { 68/// \brief Represents the relative location of an instruction. 69/// 70/// Instruction locations are specified by the line offset from the 71/// beginning of the function (marked by the line where the function 72/// header is) and the discriminator value within that line. 73/// 74/// The discriminator value is useful to distinguish instructions 75/// that are on the same line but belong to different basic blocks 76/// (e.g., the two post-increment instructions in "if (p) x++; else y++;"). 77struct InstructionLocation { 78 InstructionLocation(int L, unsigned D) : LineOffset(L), Discriminator(D) {} 79 int LineOffset; 80 unsigned Discriminator; 81}; 82} 83 84namespace llvm { 85template <> struct DenseMapInfo<InstructionLocation> { 86 typedef DenseMapInfo<int> OffsetInfo; 87 typedef DenseMapInfo<unsigned> DiscriminatorInfo; 88 static inline InstructionLocation getEmptyKey() { 89 return InstructionLocation(OffsetInfo::getEmptyKey(), 90 DiscriminatorInfo::getEmptyKey()); 91 } 92 static inline InstructionLocation getTombstoneKey() { 93 return InstructionLocation(OffsetInfo::getTombstoneKey(), 94 DiscriminatorInfo::getTombstoneKey()); 95 } 96 static inline unsigned getHashValue(InstructionLocation Val) { 97 return DenseMapInfo<std::pair<int, unsigned>>::getHashValue( 98 std::pair<int, unsigned>(Val.LineOffset, Val.Discriminator)); 99 } 100 static inline bool isEqual(InstructionLocation LHS, InstructionLocation RHS) { 101 return LHS.LineOffset == RHS.LineOffset && 102 LHS.Discriminator == RHS.Discriminator; 103 } 104}; 105} 106 107namespace { 108typedef DenseMap<InstructionLocation, unsigned> BodySampleMap; 109typedef DenseMap<BasicBlock *, unsigned> BlockWeightMap; 110typedef DenseMap<BasicBlock *, BasicBlock *> EquivalenceClassMap; 111typedef std::pair<BasicBlock *, BasicBlock *> Edge; 112typedef DenseMap<Edge, unsigned> EdgeWeightMap; 113typedef DenseMap<BasicBlock *, SmallVector<BasicBlock *, 8>> BlockEdgeMap; 114 115/// \brief Representation of the runtime profile for a function. 116/// 117/// This data structure contains the runtime profile for a given 118/// function. It contains the total number of samples collected 119/// in the function and a map of samples collected in every statement. 120class SampleFunctionProfile { 121public: 122 SampleFunctionProfile() 123 : TotalSamples(0), TotalHeadSamples(0), HeaderLineno(0), DT(0), PDT(0), 124 LI(0), Ctx(0) {} 125 126 unsigned getFunctionLoc(Function &F); 127 bool emitAnnotations(Function &F, DominatorTree *DomTree, 128 PostDominatorTree *PostDomTree, LoopInfo *Loops); 129 unsigned getInstWeight(Instruction &I); 130 unsigned getBlockWeight(BasicBlock *B); 131 void addTotalSamples(unsigned Num) { TotalSamples += Num; } 132 void addHeadSamples(unsigned Num) { TotalHeadSamples += Num; } 133 void addBodySamples(int LineOffset, unsigned Discriminator, unsigned Num) { 134 assert(LineOffset >= 0); 135 BodySamples[InstructionLocation(LineOffset, Discriminator)] += Num; 136 } 137 void print(raw_ostream &OS); 138 void printEdgeWeight(raw_ostream &OS, Edge E); 139 void printBlockWeight(raw_ostream &OS, BasicBlock *BB); 140 void printBlockEquivalence(raw_ostream &OS, BasicBlock *BB); 141 bool computeBlockWeights(Function &F); 142 void findEquivalenceClasses(Function &F); 143 void findEquivalencesFor(BasicBlock *BB1, 144 SmallVector<BasicBlock *, 8> Descendants, 145 DominatorTreeBase<BasicBlock> *DomTree); 146 void propagateWeights(Function &F); 147 unsigned visitEdge(Edge E, unsigned *NumUnknownEdges, Edge *UnknownEdge); 148 void buildEdges(Function &F); 149 bool propagateThroughEdges(Function &F); 150 bool empty() { return BodySamples.empty(); } 151 152protected: 153 /// \brief Total number of samples collected inside this function. 154 /// 155 /// Samples are cumulative, they include all the samples collected 156 /// inside this function and all its inlined callees. 157 unsigned TotalSamples; 158 159 /// \brief Total number of samples collected at the head of the function. 160 /// FIXME: Use head samples to estimate a cold/hot attribute for the function. 161 unsigned TotalHeadSamples; 162 163 /// \brief Line number for the function header. Used to compute relative 164 /// line numbers from the absolute line LOCs found in instruction locations. 165 /// The relative line numbers are needed to address the samples from the 166 /// profile file. 167 unsigned HeaderLineno; 168 169 /// \brief Map line offsets to collected samples. 170 /// 171 /// Each entry in this map contains the number of samples 172 /// collected at the corresponding line offset. All line locations 173 /// are an offset from the start of the function. 174 BodySampleMap BodySamples; 175 176 /// \brief Map basic blocks to their computed weights. 177 /// 178 /// The weight of a basic block is defined to be the maximum 179 /// of all the instruction weights in that block. 180 BlockWeightMap BlockWeights; 181 182 /// \brief Map edges to their computed weights. 183 /// 184 /// Edge weights are computed by propagating basic block weights in 185 /// SampleProfile::propagateWeights. 186 EdgeWeightMap EdgeWeights; 187 188 /// \brief Set of visited blocks during propagation. 189 SmallPtrSet<BasicBlock *, 128> VisitedBlocks; 190 191 /// \brief Set of visited edges during propagation. 192 SmallSet<Edge, 128> VisitedEdges; 193 194 /// \brief Equivalence classes for block weights. 195 /// 196 /// Two blocks BB1 and BB2 are in the same equivalence class if they 197 /// dominate and post-dominate each other, and they are in the same loop 198 /// nest. When this happens, the two blocks are guaranteed to execute 199 /// the same number of times. 200 EquivalenceClassMap EquivalenceClass; 201 202 /// \brief Dominance, post-dominance and loop information. 203 DominatorTree *DT; 204 PostDominatorTree *PDT; 205 LoopInfo *LI; 206 207 /// \brief Predecessors for each basic block in the CFG. 208 BlockEdgeMap Predecessors; 209 210 /// \brief Successors for each basic block in the CFG. 211 BlockEdgeMap Successors; 212 213 /// \brief LLVM context holding the debug data we need. 214 LLVMContext *Ctx; 215}; 216 217/// \brief Sample-based profile reader. 218/// 219/// Each profile contains sample counts for all the functions 220/// executed. Inside each function, statements are annotated with the 221/// collected samples on all the instructions associated with that 222/// statement. 223/// 224/// For this to produce meaningful data, the program needs to be 225/// compiled with some debug information (at minimum, line numbers: 226/// -gline-tables-only). Otherwise, it will be impossible to match IR 227/// instructions to the line numbers collected by the profiler. 228/// 229/// From the profile file, we are interested in collecting the 230/// following information: 231/// 232/// * A list of functions included in the profile (mangled names). 233/// 234/// * For each function F: 235/// 1. The total number of samples collected in F. 236/// 237/// 2. The samples collected at each line in F. To provide some 238/// protection against source code shuffling, line numbers should 239/// be relative to the start of the function. 240class SampleModuleProfile { 241public: 242 SampleModuleProfile(const Module &M, StringRef F) 243 : Profiles(0), Filename(F), M(M) {} 244 245 void dump(); 246 bool loadText(); 247 void loadNative() { llvm_unreachable("not implemented"); } 248 void printFunctionProfile(raw_ostream &OS, StringRef FName); 249 void dumpFunctionProfile(StringRef FName); 250 SampleFunctionProfile &getProfile(const Function &F) { 251 return Profiles[F.getName()]; 252 } 253 254 /// \brief Report a parse error message. 255 void reportParseError(int64_t LineNumber, Twine Msg) const { 256 DiagnosticInfoSampleProfile Diag(Filename.data(), LineNumber, Msg); 257 M.getContext().diagnose(Diag); 258 } 259 260protected: 261 /// \brief Map every function to its associated profile. 262 /// 263 /// The profile of every function executed at runtime is collected 264 /// in the structure SampleFunctionProfile. This maps function objects 265 /// to their corresponding profiles. 266 StringMap<SampleFunctionProfile> Profiles; 267 268 /// \brief Path name to the file holding the profile data. 269 /// 270 /// The format of this file is defined by each profiler 271 /// independently. If possible, the profiler should have a text 272 /// version of the profile format to be used in constructing test 273 /// cases and debugging. 274 StringRef Filename; 275 276 /// \brief Module being compiled. Used mainly to access the current 277 /// LLVM context for diagnostics. 278 const Module &M; 279}; 280 281/// \brief Sample profile pass. 282/// 283/// This pass reads profile data from the file specified by 284/// -sample-profile-file and annotates every affected function with the 285/// profile information found in that file. 286class SampleProfileLoader : public FunctionPass { 287public: 288 // Class identification, replacement for typeinfo 289 static char ID; 290 291 SampleProfileLoader(StringRef Name = SampleProfileFile) 292 : FunctionPass(ID), Profiler(), Filename(Name), ProfileIsValid(false) { 293 initializeSampleProfileLoaderPass(*PassRegistry::getPassRegistry()); 294 } 295 296 bool doInitialization(Module &M) override; 297 298 void dump() { Profiler->dump(); } 299 300 const char *getPassName() const override { return "Sample profile pass"; } 301 302 bool runOnFunction(Function &F) override; 303 304 void getAnalysisUsage(AnalysisUsage &AU) const override { 305 AU.setPreservesCFG(); 306 AU.addRequired<LoopInfo>(); 307 AU.addRequired<DominatorTreeWrapperPass>(); 308 AU.addRequired<PostDominatorTree>(); 309 } 310 311protected: 312 /// \brief Profile reader object. 313 std::unique_ptr<SampleModuleProfile> Profiler; 314 315 /// \brief Name of the profile file to load. 316 StringRef Filename; 317 318 /// \brief Flag indicating whether the profile input loaded succesfully. 319 bool ProfileIsValid; 320}; 321} 322 323/// \brief Print this function profile on stream \p OS. 324/// 325/// \param OS Stream to emit the output to. 326void SampleFunctionProfile::print(raw_ostream &OS) { 327 OS << TotalSamples << ", " << TotalHeadSamples << ", " << BodySamples.size() 328 << " sampled lines\n"; 329 for (BodySampleMap::const_iterator SI = BodySamples.begin(), 330 SE = BodySamples.end(); 331 SI != SE; ++SI) 332 OS << "\tline offset: " << SI->first.LineOffset 333 << ", discriminator: " << SI->first.Discriminator 334 << ", number of samples: " << SI->second << "\n"; 335 OS << "\n"; 336} 337 338/// \brief Print the weight of edge \p E on stream \p OS. 339/// 340/// \param OS Stream to emit the output to. 341/// \param E Edge to print. 342void SampleFunctionProfile::printEdgeWeight(raw_ostream &OS, Edge E) { 343 OS << "weight[" << E.first->getName() << "->" << E.second->getName() 344 << "]: " << EdgeWeights[E] << "\n"; 345} 346 347/// \brief Print the equivalence class of block \p BB on stream \p OS. 348/// 349/// \param OS Stream to emit the output to. 350/// \param BB Block to print. 351void SampleFunctionProfile::printBlockEquivalence(raw_ostream &OS, 352 BasicBlock *BB) { 353 BasicBlock *Equiv = EquivalenceClass[BB]; 354 OS << "equivalence[" << BB->getName() 355 << "]: " << ((Equiv) ? EquivalenceClass[BB]->getName() : "NONE") << "\n"; 356} 357 358/// \brief Print the weight of block \p BB on stream \p OS. 359/// 360/// \param OS Stream to emit the output to. 361/// \param BB Block to print. 362void SampleFunctionProfile::printBlockWeight(raw_ostream &OS, BasicBlock *BB) { 363 OS << "weight[" << BB->getName() << "]: " << BlockWeights[BB] << "\n"; 364} 365 366/// \brief Print the function profile for \p FName on stream \p OS. 367/// 368/// \param OS Stream to emit the output to. 369/// \param FName Name of the function to print. 370void SampleModuleProfile::printFunctionProfile(raw_ostream &OS, 371 StringRef FName) { 372 OS << "Function: " << FName << ":\n"; 373 Profiles[FName].print(OS); 374} 375 376/// \brief Dump the function profile for \p FName. 377/// 378/// \param FName Name of the function to print. 379void SampleModuleProfile::dumpFunctionProfile(StringRef FName) { 380 printFunctionProfile(dbgs(), FName); 381} 382 383/// \brief Dump all the function profiles found. 384void SampleModuleProfile::dump() { 385 for (StringMap<SampleFunctionProfile>::const_iterator I = Profiles.begin(), 386 E = Profiles.end(); 387 I != E; ++I) 388 dumpFunctionProfile(I->getKey()); 389} 390 391/// \brief Load samples from a text file. 392/// 393/// The file contains a list of samples for every function executed at 394/// runtime. Each function profile has the following format: 395/// 396/// function1:total_samples:total_head_samples 397/// offset1[.discriminator]: number_of_samples [fn1:num fn2:num ... ] 398/// offset2[.discriminator]: number_of_samples [fn3:num fn4:num ... ] 399/// ... 400/// offsetN[.discriminator]: number_of_samples [fn5:num fn6:num ... ] 401/// 402/// Function names must be mangled in order for the profile loader to 403/// match them in the current translation unit. The two numbers in the 404/// function header specify how many total samples were accumulated in 405/// the function (first number), and the total number of samples accumulated 406/// at the prologue of the function (second number). This head sample 407/// count provides an indicator of how frequent is the function invoked. 408/// 409/// Each sampled line may contain several items. Some are optional 410/// (marked below): 411/// 412/// a- Source line offset. This number represents the line number 413/// in the function where the sample was collected. The line number 414/// is always relative to the line where symbol of the function 415/// is defined. So, if the function has its header at line 280, 416/// the offset 13 is at line 293 in the file. 417/// 418/// b- [OPTIONAL] Discriminator. This is used if the sampled program 419/// was compiled with DWARF discriminator support 420/// (http://wiki.dwarfstd.org/index.php?title=Path_Discriminators) 421/// 422/// c- Number of samples. This is the number of samples collected by 423/// the profiler at this source location. 424/// 425/// d- [OPTIONAL] Potential call targets and samples. If present, this 426/// line contains a call instruction. This models both direct and 427/// indirect calls. Each called target is listed together with the 428/// number of samples. For example, 429/// 430/// 130: 7 foo:3 bar:2 baz:7 431/// 432/// The above means that at relative line offset 130 there is a 433/// call instruction that calls one of foo(), bar() and baz(). With 434/// baz() being the relatively more frequent call target. 435/// 436/// FIXME: This is currently unhandled, but it has a lot of 437/// potential for aiding the inliner. 438/// 439/// 440/// Since this is a flat profile, a function that shows up more than 441/// once gets all its samples aggregated across all its instances. 442/// 443/// FIXME: flat profiles are too imprecise to provide good optimization 444/// opportunities. Convert them to context-sensitive profile. 445/// 446/// This textual representation is useful to generate unit tests and 447/// for debugging purposes, but it should not be used to generate 448/// profiles for large programs, as the representation is extremely 449/// inefficient. 450/// 451/// \returns true if the file was loaded successfully, false otherwise. 452bool SampleModuleProfile::loadText() { 453 std::unique_ptr<MemoryBuffer> Buffer; 454 error_code EC = MemoryBuffer::getFile(Filename, Buffer); 455 if (EC) { 456 std::string Msg(EC.message()); 457 M.getContext().diagnose(DiagnosticInfoSampleProfile(Filename.data(), Msg)); 458 return false; 459 } 460 line_iterator LineIt(*Buffer, '#'); 461 462 // Read the profile of each function. Since each function may be 463 // mentioned more than once, and we are collecting flat profiles, 464 // accumulate samples as we parse them. 465 Regex HeadRE("^([^0-9].*):([0-9]+):([0-9]+)$"); 466 Regex LineSample("^([0-9]+)\\.?([0-9]+)?: ([0-9]+)(.*)$"); 467 while (!LineIt.is_at_eof()) { 468 // Read the header of each function. 469 // 470 // Note that for function identifiers we are actually expecting 471 // mangled names, but we may not always get them. This happens when 472 // the compiler decides not to emit the function (e.g., it was inlined 473 // and removed). In this case, the binary will not have the linkage 474 // name for the function, so the profiler will emit the function's 475 // unmangled name, which may contain characters like ':' and '>' in its 476 // name (member functions, templates, etc). 477 // 478 // The only requirement we place on the identifier, then, is that it 479 // should not begin with a number. 480 SmallVector<StringRef, 3> Matches; 481 if (!HeadRE.match(*LineIt, &Matches)) { 482 reportParseError(LineIt.line_number(), 483 "Expected 'mangled_name:NUM:NUM', found " + *LineIt); 484 return false; 485 } 486 assert(Matches.size() == 4); 487 StringRef FName = Matches[1]; 488 unsigned NumSamples, NumHeadSamples; 489 Matches[2].getAsInteger(10, NumSamples); 490 Matches[3].getAsInteger(10, NumHeadSamples); 491 Profiles[FName] = SampleFunctionProfile(); 492 SampleFunctionProfile &FProfile = Profiles[FName]; 493 FProfile.addTotalSamples(NumSamples); 494 FProfile.addHeadSamples(NumHeadSamples); 495 ++LineIt; 496 497 // Now read the body. The body of the function ends when we reach 498 // EOF or when we see the start of the next function. 499 while (!LineIt.is_at_eof() && isdigit((*LineIt)[0])) { 500 if (!LineSample.match(*LineIt, &Matches)) { 501 reportParseError( 502 LineIt.line_number(), 503 "Expected 'NUM[.NUM]: NUM[ mangled_name:NUM]*', found " + *LineIt); 504 return false; 505 } 506 assert(Matches.size() == 5); 507 unsigned LineOffset, NumSamples, Discriminator = 0; 508 Matches[1].getAsInteger(10, LineOffset); 509 if (Matches[2] != "") 510 Matches[2].getAsInteger(10, Discriminator); 511 Matches[3].getAsInteger(10, NumSamples); 512 513 // FIXME: Handle called targets (in Matches[4]). 514 515 // When dealing with instruction weights, we use the value 516 // zero to indicate the absence of a sample. If we read an 517 // actual zero from the profile file, return it as 1 to 518 // avoid the confusion later on. 519 if (NumSamples == 0) 520 NumSamples = 1; 521 FProfile.addBodySamples(LineOffset, Discriminator, NumSamples); 522 ++LineIt; 523 } 524 } 525 526 return true; 527} 528 529/// \brief Get the weight for an instruction. 530/// 531/// The "weight" of an instruction \p Inst is the number of samples 532/// collected on that instruction at runtime. To retrieve it, we 533/// need to compute the line number of \p Inst relative to the start of its 534/// function. We use HeaderLineno to compute the offset. We then 535/// look up the samples collected for \p Inst using BodySamples. 536/// 537/// \param Inst Instruction to query. 538/// 539/// \returns The profiled weight of I. 540unsigned SampleFunctionProfile::getInstWeight(Instruction &Inst) { 541 DebugLoc DLoc = Inst.getDebugLoc(); 542 unsigned Lineno = DLoc.getLine(); 543 if (Lineno < HeaderLineno) 544 return 0; 545 546 DILocation DIL(DLoc.getAsMDNode(*Ctx)); 547 int LOffset = Lineno - HeaderLineno; 548 unsigned Discriminator = DIL.getDiscriminator(); 549 unsigned Weight = 550 BodySamples.lookup(InstructionLocation(LOffset, Discriminator)); 551 DEBUG(dbgs() << " " << Lineno << "." << Discriminator << ":" << Inst 552 << " (line offset: " << LOffset << "." << Discriminator 553 << " - weight: " << Weight << ")\n"); 554 return Weight; 555} 556 557/// \brief Compute the weight of a basic block. 558/// 559/// The weight of basic block \p B is the maximum weight of all the 560/// instructions in B. The weight of \p B is computed and cached in 561/// the BlockWeights map. 562/// 563/// \param B The basic block to query. 564/// 565/// \returns The computed weight of B. 566unsigned SampleFunctionProfile::getBlockWeight(BasicBlock *B) { 567 // If we've computed B's weight before, return it. 568 std::pair<BlockWeightMap::iterator, bool> Entry = 569 BlockWeights.insert(std::make_pair(B, 0)); 570 if (!Entry.second) 571 return Entry.first->second; 572 573 // Otherwise, compute and cache B's weight. 574 unsigned Weight = 0; 575 for (BasicBlock::iterator I = B->begin(), E = B->end(); I != E; ++I) { 576 unsigned InstWeight = getInstWeight(*I); 577 if (InstWeight > Weight) 578 Weight = InstWeight; 579 } 580 Entry.first->second = Weight; 581 return Weight; 582} 583 584/// \brief Compute and store the weights of every basic block. 585/// 586/// This populates the BlockWeights map by computing 587/// the weights of every basic block in the CFG. 588/// 589/// \param F The function to query. 590bool SampleFunctionProfile::computeBlockWeights(Function &F) { 591 bool Changed = false; 592 DEBUG(dbgs() << "Block weights\n"); 593 for (Function::iterator B = F.begin(), E = F.end(); B != E; ++B) { 594 unsigned Weight = getBlockWeight(B); 595 Changed |= (Weight > 0); 596 DEBUG(printBlockWeight(dbgs(), B)); 597 } 598 599 return Changed; 600} 601 602/// \brief Find equivalence classes for the given block. 603/// 604/// This finds all the blocks that are guaranteed to execute the same 605/// number of times as \p BB1. To do this, it traverses all the the 606/// descendants of \p BB1 in the dominator or post-dominator tree. 607/// 608/// A block BB2 will be in the same equivalence class as \p BB1 if 609/// the following holds: 610/// 611/// 1- \p BB1 is a descendant of BB2 in the opposite tree. So, if BB2 612/// is a descendant of \p BB1 in the dominator tree, then BB2 should 613/// dominate BB1 in the post-dominator tree. 614/// 615/// 2- Both BB2 and \p BB1 must be in the same loop. 616/// 617/// For every block BB2 that meets those two requirements, we set BB2's 618/// equivalence class to \p BB1. 619/// 620/// \param BB1 Block to check. 621/// \param Descendants Descendants of \p BB1 in either the dom or pdom tree. 622/// \param DomTree Opposite dominator tree. If \p Descendants is filled 623/// with blocks from \p BB1's dominator tree, then 624/// this is the post-dominator tree, and vice versa. 625void SampleFunctionProfile::findEquivalencesFor( 626 BasicBlock *BB1, SmallVector<BasicBlock *, 8> Descendants, 627 DominatorTreeBase<BasicBlock> *DomTree) { 628 for (SmallVectorImpl<BasicBlock *>::iterator I = Descendants.begin(), 629 E = Descendants.end(); 630 I != E; ++I) { 631 BasicBlock *BB2 = *I; 632 bool IsDomParent = DomTree->dominates(BB2, BB1); 633 bool IsInSameLoop = LI->getLoopFor(BB1) == LI->getLoopFor(BB2); 634 if (BB1 != BB2 && VisitedBlocks.insert(BB2) && IsDomParent && 635 IsInSameLoop) { 636 EquivalenceClass[BB2] = BB1; 637 638 // If BB2 is heavier than BB1, make BB2 have the same weight 639 // as BB1. 640 // 641 // Note that we don't worry about the opposite situation here 642 // (when BB2 is lighter than BB1). We will deal with this 643 // during the propagation phase. Right now, we just want to 644 // make sure that BB1 has the largest weight of all the 645 // members of its equivalence set. 646 unsigned &BB1Weight = BlockWeights[BB1]; 647 unsigned &BB2Weight = BlockWeights[BB2]; 648 BB1Weight = std::max(BB1Weight, BB2Weight); 649 } 650 } 651} 652 653/// \brief Find equivalence classes. 654/// 655/// Since samples may be missing from blocks, we can fill in the gaps by setting 656/// the weights of all the blocks in the same equivalence class to the same 657/// weight. To compute the concept of equivalence, we use dominance and loop 658/// information. Two blocks B1 and B2 are in the same equivalence class if B1 659/// dominates B2, B2 post-dominates B1 and both are in the same loop. 660/// 661/// \param F The function to query. 662void SampleFunctionProfile::findEquivalenceClasses(Function &F) { 663 SmallVector<BasicBlock *, 8> DominatedBBs; 664 DEBUG(dbgs() << "\nBlock equivalence classes\n"); 665 // Find equivalence sets based on dominance and post-dominance information. 666 for (Function::iterator B = F.begin(), E = F.end(); B != E; ++B) { 667 BasicBlock *BB1 = B; 668 669 // Compute BB1's equivalence class once. 670 if (EquivalenceClass.count(BB1)) { 671 DEBUG(printBlockEquivalence(dbgs(), BB1)); 672 continue; 673 } 674 675 // By default, blocks are in their own equivalence class. 676 EquivalenceClass[BB1] = BB1; 677 678 // Traverse all the blocks dominated by BB1. We are looking for 679 // every basic block BB2 such that: 680 // 681 // 1- BB1 dominates BB2. 682 // 2- BB2 post-dominates BB1. 683 // 3- BB1 and BB2 are in the same loop nest. 684 // 685 // If all those conditions hold, it means that BB2 is executed 686 // as many times as BB1, so they are placed in the same equivalence 687 // class by making BB2's equivalence class be BB1. 688 DominatedBBs.clear(); 689 DT->getDescendants(BB1, DominatedBBs); 690 findEquivalencesFor(BB1, DominatedBBs, PDT->DT); 691 692 // Repeat the same logic for all the blocks post-dominated by BB1. 693 // We are looking for every basic block BB2 such that: 694 // 695 // 1- BB1 post-dominates BB2. 696 // 2- BB2 dominates BB1. 697 // 3- BB1 and BB2 are in the same loop nest. 698 // 699 // If all those conditions hold, BB2's equivalence class is BB1. 700 DominatedBBs.clear(); 701 PDT->getDescendants(BB1, DominatedBBs); 702 findEquivalencesFor(BB1, DominatedBBs, DT); 703 704 DEBUG(printBlockEquivalence(dbgs(), BB1)); 705 } 706 707 // Assign weights to equivalence classes. 708 // 709 // All the basic blocks in the same equivalence class will execute 710 // the same number of times. Since we know that the head block in 711 // each equivalence class has the largest weight, assign that weight 712 // to all the blocks in that equivalence class. 713 DEBUG(dbgs() << "\nAssign the same weight to all blocks in the same class\n"); 714 for (Function::iterator B = F.begin(), E = F.end(); B != E; ++B) { 715 BasicBlock *BB = B; 716 BasicBlock *EquivBB = EquivalenceClass[BB]; 717 if (BB != EquivBB) 718 BlockWeights[BB] = BlockWeights[EquivBB]; 719 DEBUG(printBlockWeight(dbgs(), BB)); 720 } 721} 722 723/// \brief Visit the given edge to decide if it has a valid weight. 724/// 725/// If \p E has not been visited before, we copy to \p UnknownEdge 726/// and increment the count of unknown edges. 727/// 728/// \param E Edge to visit. 729/// \param NumUnknownEdges Current number of unknown edges. 730/// \param UnknownEdge Set if E has not been visited before. 731/// 732/// \returns E's weight, if known. Otherwise, return 0. 733unsigned SampleFunctionProfile::visitEdge(Edge E, unsigned *NumUnknownEdges, 734 Edge *UnknownEdge) { 735 if (!VisitedEdges.count(E)) { 736 (*NumUnknownEdges)++; 737 *UnknownEdge = E; 738 return 0; 739 } 740 741 return EdgeWeights[E]; 742} 743 744/// \brief Propagate weights through incoming/outgoing edges. 745/// 746/// If the weight of a basic block is known, and there is only one edge 747/// with an unknown weight, we can calculate the weight of that edge. 748/// 749/// Similarly, if all the edges have a known count, we can calculate the 750/// count of the basic block, if needed. 751/// 752/// \param F Function to process. 753/// 754/// \returns True if new weights were assigned to edges or blocks. 755bool SampleFunctionProfile::propagateThroughEdges(Function &F) { 756 bool Changed = false; 757 DEBUG(dbgs() << "\nPropagation through edges\n"); 758 for (Function::iterator BI = F.begin(), EI = F.end(); BI != EI; ++BI) { 759 BasicBlock *BB = BI; 760 761 // Visit all the predecessor and successor edges to determine 762 // which ones have a weight assigned already. Note that it doesn't 763 // matter that we only keep track of a single unknown edge. The 764 // only case we are interested in handling is when only a single 765 // edge is unknown (see setEdgeOrBlockWeight). 766 for (unsigned i = 0; i < 2; i++) { 767 unsigned TotalWeight = 0; 768 unsigned NumUnknownEdges = 0; 769 Edge UnknownEdge, SelfReferentialEdge; 770 771 if (i == 0) { 772 // First, visit all predecessor edges. 773 for (size_t I = 0; I < Predecessors[BB].size(); I++) { 774 Edge E = std::make_pair(Predecessors[BB][I], BB); 775 TotalWeight += visitEdge(E, &NumUnknownEdges, &UnknownEdge); 776 if (E.first == E.second) 777 SelfReferentialEdge = E; 778 } 779 } else { 780 // On the second round, visit all successor edges. 781 for (size_t I = 0; I < Successors[BB].size(); I++) { 782 Edge E = std::make_pair(BB, Successors[BB][I]); 783 TotalWeight += visitEdge(E, &NumUnknownEdges, &UnknownEdge); 784 } 785 } 786 787 // After visiting all the edges, there are three cases that we 788 // can handle immediately: 789 // 790 // - All the edge weights are known (i.e., NumUnknownEdges == 0). 791 // In this case, we simply check that the sum of all the edges 792 // is the same as BB's weight. If not, we change BB's weight 793 // to match. Additionally, if BB had not been visited before, 794 // we mark it visited. 795 // 796 // - Only one edge is unknown and BB has already been visited. 797 // In this case, we can compute the weight of the edge by 798 // subtracting the total block weight from all the known 799 // edge weights. If the edges weight more than BB, then the 800 // edge of the last remaining edge is set to zero. 801 // 802 // - There exists a self-referential edge and the weight of BB is 803 // known. In this case, this edge can be based on BB's weight. 804 // We add up all the other known edges and set the weight on 805 // the self-referential edge as we did in the previous case. 806 // 807 // In any other case, we must continue iterating. Eventually, 808 // all edges will get a weight, or iteration will stop when 809 // it reaches SampleProfileMaxPropagateIterations. 810 if (NumUnknownEdges <= 1) { 811 unsigned &BBWeight = BlockWeights[BB]; 812 if (NumUnknownEdges == 0) { 813 // If we already know the weight of all edges, the weight of the 814 // basic block can be computed. It should be no larger than the sum 815 // of all edge weights. 816 if (TotalWeight > BBWeight) { 817 BBWeight = TotalWeight; 818 Changed = true; 819 DEBUG(dbgs() << "All edge weights for " << BB->getName() 820 << " known. Set weight for block: "; 821 printBlockWeight(dbgs(), BB);); 822 } 823 if (VisitedBlocks.insert(BB)) 824 Changed = true; 825 } else if (NumUnknownEdges == 1 && VisitedBlocks.count(BB)) { 826 // If there is a single unknown edge and the block has been 827 // visited, then we can compute E's weight. 828 if (BBWeight >= TotalWeight) 829 EdgeWeights[UnknownEdge] = BBWeight - TotalWeight; 830 else 831 EdgeWeights[UnknownEdge] = 0; 832 VisitedEdges.insert(UnknownEdge); 833 Changed = true; 834 DEBUG(dbgs() << "Set weight for edge: "; 835 printEdgeWeight(dbgs(), UnknownEdge)); 836 } 837 } else if (SelfReferentialEdge.first && VisitedBlocks.count(BB)) { 838 unsigned &BBWeight = BlockWeights[BB]; 839 // We have a self-referential edge and the weight of BB is known. 840 if (BBWeight >= TotalWeight) 841 EdgeWeights[SelfReferentialEdge] = BBWeight - TotalWeight; 842 else 843 EdgeWeights[SelfReferentialEdge] = 0; 844 VisitedEdges.insert(SelfReferentialEdge); 845 Changed = true; 846 DEBUG(dbgs() << "Set self-referential edge weight to: "; 847 printEdgeWeight(dbgs(), SelfReferentialEdge)); 848 } 849 } 850 } 851 852 return Changed; 853} 854 855/// \brief Build in/out edge lists for each basic block in the CFG. 856/// 857/// We are interested in unique edges. If a block B1 has multiple 858/// edges to another block B2, we only add a single B1->B2 edge. 859void SampleFunctionProfile::buildEdges(Function &F) { 860 for (Function::iterator I = F.begin(), E = F.end(); I != E; ++I) { 861 BasicBlock *B1 = I; 862 863 // Add predecessors for B1. 864 SmallPtrSet<BasicBlock *, 16> Visited; 865 if (!Predecessors[B1].empty()) 866 llvm_unreachable("Found a stale predecessors list in a basic block."); 867 for (pred_iterator PI = pred_begin(B1), PE = pred_end(B1); PI != PE; ++PI) { 868 BasicBlock *B2 = *PI; 869 if (Visited.insert(B2)) 870 Predecessors[B1].push_back(B2); 871 } 872 873 // Add successors for B1. 874 Visited.clear(); 875 if (!Successors[B1].empty()) 876 llvm_unreachable("Found a stale successors list in a basic block."); 877 for (succ_iterator SI = succ_begin(B1), SE = succ_end(B1); SI != SE; ++SI) { 878 BasicBlock *B2 = *SI; 879 if (Visited.insert(B2)) 880 Successors[B1].push_back(B2); 881 } 882 } 883} 884 885/// \brief Propagate weights into edges 886/// 887/// The following rules are applied to every block B in the CFG: 888/// 889/// - If B has a single predecessor/successor, then the weight 890/// of that edge is the weight of the block. 891/// 892/// - If all incoming or outgoing edges are known except one, and the 893/// weight of the block is already known, the weight of the unknown 894/// edge will be the weight of the block minus the sum of all the known 895/// edges. If the sum of all the known edges is larger than B's weight, 896/// we set the unknown edge weight to zero. 897/// 898/// - If there is a self-referential edge, and the weight of the block is 899/// known, the weight for that edge is set to the weight of the block 900/// minus the weight of the other incoming edges to that block (if 901/// known). 902void SampleFunctionProfile::propagateWeights(Function &F) { 903 bool Changed = true; 904 unsigned i = 0; 905 906 // Before propagation starts, build, for each block, a list of 907 // unique predecessors and successors. This is necessary to handle 908 // identical edges in multiway branches. Since we visit all blocks and all 909 // edges of the CFG, it is cleaner to build these lists once at the start 910 // of the pass. 911 buildEdges(F); 912 913 // Propagate until we converge or we go past the iteration limit. 914 while (Changed && i++ < SampleProfileMaxPropagateIterations) { 915 Changed = propagateThroughEdges(F); 916 } 917 918 // Generate MD_prof metadata for every branch instruction using the 919 // edge weights computed during propagation. 920 DEBUG(dbgs() << "\nPropagation complete. Setting branch weights\n"); 921 MDBuilder MDB(F.getContext()); 922 for (Function::iterator I = F.begin(), E = F.end(); I != E; ++I) { 923 BasicBlock *B = I; 924 TerminatorInst *TI = B->getTerminator(); 925 if (TI->getNumSuccessors() == 1) 926 continue; 927 if (!isa<BranchInst>(TI) && !isa<SwitchInst>(TI)) 928 continue; 929 930 DEBUG(dbgs() << "\nGetting weights for branch at line " 931 << TI->getDebugLoc().getLine() << ".\n"); 932 SmallVector<unsigned, 4> Weights; 933 bool AllWeightsZero = true; 934 for (unsigned I = 0; I < TI->getNumSuccessors(); ++I) { 935 BasicBlock *Succ = TI->getSuccessor(I); 936 Edge E = std::make_pair(B, Succ); 937 unsigned Weight = EdgeWeights[E]; 938 DEBUG(dbgs() << "\t"; printEdgeWeight(dbgs(), E)); 939 Weights.push_back(Weight); 940 if (Weight != 0) 941 AllWeightsZero = false; 942 } 943 944 // Only set weights if there is at least one non-zero weight. 945 // In any other case, let the analyzer set weights. 946 if (!AllWeightsZero) { 947 DEBUG(dbgs() << "SUCCESS. Found non-zero weights.\n"); 948 TI->setMetadata(llvm::LLVMContext::MD_prof, 949 MDB.createBranchWeights(Weights)); 950 } else { 951 DEBUG(dbgs() << "SKIPPED. All branch weights are zero.\n"); 952 } 953 } 954} 955 956/// \brief Get the line number for the function header. 957/// 958/// This looks up function \p F in the current compilation unit and 959/// retrieves the line number where the function is defined. This is 960/// line 0 for all the samples read from the profile file. Every line 961/// number is relative to this line. 962/// 963/// \param F Function object to query. 964/// 965/// \returns the line number where \p F is defined. If it returns 0, 966/// it means that there is no debug information available for \p F. 967unsigned SampleFunctionProfile::getFunctionLoc(Function &F) { 968 NamedMDNode *CUNodes = F.getParent()->getNamedMetadata("llvm.dbg.cu"); 969 if (CUNodes) { 970 for (unsigned I = 0, E1 = CUNodes->getNumOperands(); I != E1; ++I) { 971 DICompileUnit CU(CUNodes->getOperand(I)); 972 DIArray Subprograms = CU.getSubprograms(); 973 for (unsigned J = 0, E2 = Subprograms.getNumElements(); J != E2; ++J) { 974 DISubprogram Subprogram(Subprograms.getElement(J)); 975 if (Subprogram.describes(&F)) 976 return Subprogram.getLineNumber(); 977 } 978 } 979 } 980 981 F.getContext().diagnose(DiagnosticInfoSampleProfile( 982 "No debug information found in function " + F.getName())); 983 return 0; 984} 985 986/// \brief Generate branch weight metadata for all branches in \p F. 987/// 988/// Branch weights are computed out of instruction samples using a 989/// propagation heuristic. Propagation proceeds in 3 phases: 990/// 991/// 1- Assignment of block weights. All the basic blocks in the function 992/// are initial assigned the same weight as their most frequently 993/// executed instruction. 994/// 995/// 2- Creation of equivalence classes. Since samples may be missing from 996/// blocks, we can fill in the gaps by setting the weights of all the 997/// blocks in the same equivalence class to the same weight. To compute 998/// the concept of equivalence, we use dominance and loop information. 999/// Two blocks B1 and B2 are in the same equivalence class if B1 1000/// dominates B2, B2 post-dominates B1 and both are in the same loop. 1001/// 1002/// 3- Propagation of block weights into edges. This uses a simple 1003/// propagation heuristic. The following rules are applied to every 1004/// block B in the CFG: 1005/// 1006/// - If B has a single predecessor/successor, then the weight 1007/// of that edge is the weight of the block. 1008/// 1009/// - If all the edges are known except one, and the weight of the 1010/// block is already known, the weight of the unknown edge will 1011/// be the weight of the block minus the sum of all the known 1012/// edges. If the sum of all the known edges is larger than B's weight, 1013/// we set the unknown edge weight to zero. 1014/// 1015/// - If there is a self-referential edge, and the weight of the block is 1016/// known, the weight for that edge is set to the weight of the block 1017/// minus the weight of the other incoming edges to that block (if 1018/// known). 1019/// 1020/// Since this propagation is not guaranteed to finalize for every CFG, we 1021/// only allow it to proceed for a limited number of iterations (controlled 1022/// by -sample-profile-max-propagate-iterations). 1023/// 1024/// FIXME: Try to replace this propagation heuristic with a scheme 1025/// that is guaranteed to finalize. A work-list approach similar to 1026/// the standard value propagation algorithm used by SSA-CCP might 1027/// work here. 1028/// 1029/// Once all the branch weights are computed, we emit the MD_prof 1030/// metadata on B using the computed values for each of its branches. 1031/// 1032/// \param F The function to query. 1033/// 1034/// \returns true if \p F was modified. Returns false, otherwise. 1035bool SampleFunctionProfile::emitAnnotations(Function &F, DominatorTree *DomTree, 1036 PostDominatorTree *PostDomTree, 1037 LoopInfo *Loops) { 1038 bool Changed = false; 1039 1040 // Initialize invariants used during computation and propagation. 1041 HeaderLineno = getFunctionLoc(F); 1042 if (HeaderLineno == 0) 1043 return false; 1044 1045 DEBUG(dbgs() << "Line number for the first instruction in " << F.getName() 1046 << ": " << HeaderLineno << "\n"); 1047 DT = DomTree; 1048 PDT = PostDomTree; 1049 LI = Loops; 1050 Ctx = &F.getParent()->getContext(); 1051 1052 // Compute basic block weights. 1053 Changed |= computeBlockWeights(F); 1054 1055 if (Changed) { 1056 // Find equivalence classes. 1057 findEquivalenceClasses(F); 1058 1059 // Propagate weights to all edges. 1060 propagateWeights(F); 1061 } 1062 1063 return Changed; 1064} 1065 1066char SampleProfileLoader::ID = 0; 1067INITIALIZE_PASS_BEGIN(SampleProfileLoader, "sample-profile", 1068 "Sample Profile loader", false, false) 1069INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass) 1070INITIALIZE_PASS_DEPENDENCY(PostDominatorTree) 1071INITIALIZE_PASS_DEPENDENCY(LoopInfo) 1072INITIALIZE_PASS_DEPENDENCY(AddDiscriminators) 1073INITIALIZE_PASS_END(SampleProfileLoader, "sample-profile", 1074 "Sample Profile loader", false, false) 1075 1076bool SampleProfileLoader::doInitialization(Module &M) { 1077 Profiler.reset(new SampleModuleProfile(M, Filename)); 1078 ProfileIsValid = Profiler->loadText(); 1079 return true; 1080} 1081 1082FunctionPass *llvm::createSampleProfileLoaderPass() { 1083 return new SampleProfileLoader(SampleProfileFile); 1084} 1085 1086FunctionPass *llvm::createSampleProfileLoaderPass(StringRef Name) { 1087 return new SampleProfileLoader(Name); 1088} 1089 1090bool SampleProfileLoader::runOnFunction(Function &F) { 1091 if (!ProfileIsValid) 1092 return false; 1093 DominatorTree *DT = &getAnalysis<DominatorTreeWrapperPass>().getDomTree(); 1094 PostDominatorTree *PDT = &getAnalysis<PostDominatorTree>(); 1095 LoopInfo *LI = &getAnalysis<LoopInfo>(); 1096 SampleFunctionProfile &FunctionProfile = Profiler->getProfile(F); 1097 if (!FunctionProfile.empty()) 1098 return FunctionProfile.emitAnnotations(F, DT, PDT, LI); 1099 return false; 1100} 1101