1//===--- CodeGenPGO.h - PGO Instrumentation for LLVM CodeGen ----*- C++ -*-===//
2//
3//                     The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10// Instrumentation-based profile-guided optimization
11//
12//===----------------------------------------------------------------------===//
13
14#ifndef CLANG_CODEGEN_CODEGENPGO_H
15#define CLANG_CODEGEN_CODEGENPGO_H
16
17#include "CGBuilder.h"
18#include "CodeGenModule.h"
19#include "CodeGenTypes.h"
20#include "clang/Frontend/CodeGenOptions.h"
21#include "llvm/ADT/StringMap.h"
22#include "llvm/Support/MemoryBuffer.h"
23#include <memory>
24
25namespace clang {
26namespace CodeGen {
27class RegionCounter;
28
29/// Per-function PGO state. This class should generally not be used directly,
30/// but instead through the CodeGenFunction and RegionCounter types.
31class CodeGenPGO {
32private:
33  CodeGenModule &CGM;
34  std::unique_ptr<std::string> PrefixedFuncName;
35  StringRef RawFuncName;
36  llvm::GlobalValue::LinkageTypes VarLinkage;
37
38  unsigned NumRegionCounters;
39  uint64_t FunctionHash;
40  llvm::GlobalVariable *RegionCounters;
41  std::unique_ptr<llvm::DenseMap<const Stmt *, unsigned>> RegionCounterMap;
42  std::unique_ptr<llvm::DenseMap<const Stmt *, uint64_t>> StmtCountMap;
43  std::unique_ptr<std::vector<uint64_t>> RegionCounts;
44  uint64_t CurrentRegionCount;
45
46public:
47  CodeGenPGO(CodeGenModule &CGM)
48      : CGM(CGM), NumRegionCounters(0), FunctionHash(0),
49        RegionCounters(nullptr), CurrentRegionCount(0) {}
50
51  /// Whether or not we have PGO region data for the current function. This is
52  /// false both when we have no data at all and when our data has been
53  /// discarded.
54  bool haveRegionCounts() const { return RegionCounts != nullptr; }
55
56  /// Get the string used to identify this function in the profile data.
57  /// For functions with local linkage, this includes the main file name.
58  StringRef getFuncName() const { return StringRef(*PrefixedFuncName); }
59  std::string getFuncVarName(StringRef VarName) const {
60    return ("__llvm_profile_" + VarName + "_" + RawFuncName).str();
61  }
62
63  /// Return the counter value of the current region.
64  uint64_t getCurrentRegionCount() const { return CurrentRegionCount; }
65
66  /// Set the counter value for the current region. This is used to keep track
67  /// of changes to the most recent counter from control flow and non-local
68  /// exits.
69  void setCurrentRegionCount(uint64_t Count) { CurrentRegionCount = Count; }
70
71  /// Indicate that the current region is never reached, and thus should have a
72  /// counter value of zero. This is important so that subsequent regions can
73  /// correctly track their parent counts.
74  void setCurrentRegionUnreachable() { setCurrentRegionCount(0); }
75
76  /// Check if an execution count is known for a given statement. If so, return
77  /// true and put the value in Count; else return false.
78  bool getStmtCount(const Stmt *S, uint64_t &Count) {
79    if (!StmtCountMap)
80      return false;
81    llvm::DenseMap<const Stmt*, uint64_t>::const_iterator
82      I = StmtCountMap->find(S);
83    if (I == StmtCountMap->end())
84      return false;
85    Count = I->second;
86    return true;
87  }
88
89  /// If the execution count for the current statement is known, record that
90  /// as the current count.
91  void setCurrentStmt(const Stmt *S) {
92    uint64_t Count;
93    if (getStmtCount(S, Count))
94      setCurrentRegionCount(Count);
95  }
96
97  /// Calculate branch weights appropriate for PGO data
98  llvm::MDNode *createBranchWeights(uint64_t TrueCount, uint64_t FalseCount);
99  llvm::MDNode *createBranchWeights(ArrayRef<uint64_t> Weights);
100  llvm::MDNode *createLoopWeights(const Stmt *Cond, RegionCounter &Cnt);
101
102  /// Assign counters to regions and configure them for PGO of a given
103  /// function. Does nothing if instrumentation is not enabled and either
104  /// generates global variables or associates PGO data with each of the
105  /// counters depending on whether we are generating or using instrumentation.
106  void assignRegionCounters(const Decl *D, llvm::Function *Fn);
107  /// Emit static data structures for instrumentation data.
108  void emitInstrumentationData();
109  /// Clean up region counter state. Must be called if assignRegionCounters is
110  /// used.
111  void destroyRegionCounters();
112  /// Emit static initialization code, if any.
113  static llvm::Function *emitInitialization(CodeGenModule &CGM);
114
115private:
116  void setFuncName(llvm::Function *Fn);
117  void mapRegionCounters(const Decl *D);
118  void computeRegionCounts(const Decl *D);
119  void applyFunctionAttributes(llvm::IndexedInstrProfReader *PGOReader,
120                               llvm::Function *Fn);
121  void loadRegionCounts(llvm::IndexedInstrProfReader *PGOReader,
122                        bool IsInMainFile);
123  void emitCounterVariables();
124  llvm::GlobalVariable *buildDataVar();
125
126  /// Emit code to increment the counter at the given index
127  void emitCounterIncrement(CGBuilderTy &Builder, unsigned Counter);
128
129  /// Return the region counter for the given statement. This should only be
130  /// called on statements that have a dedicated counter.
131  unsigned getRegionCounter(const Stmt *S) {
132    if (!RegionCounterMap)
133      return 0;
134    return (*RegionCounterMap)[S];
135  }
136
137  /// Return the region count for the counter at the given index.
138  uint64_t getRegionCount(unsigned Counter) {
139    if (!haveRegionCounts())
140      return 0;
141    return (*RegionCounts)[Counter];
142  }
143
144  friend class RegionCounter;
145};
146
147/// A counter for a particular region. This is the primary interface through
148/// which clients manage PGO counters and their values.
149class RegionCounter {
150  CodeGenPGO *PGO;
151  unsigned Counter;
152  uint64_t Count;
153  uint64_t ParentCount;
154  uint64_t RegionCount;
155  int64_t Adjust;
156
157  RegionCounter(CodeGenPGO &PGO, unsigned CounterIndex)
158    : PGO(&PGO), Counter(CounterIndex), Count(PGO.getRegionCount(Counter)),
159      ParentCount(PGO.getCurrentRegionCount()), Adjust(0) {}
160
161public:
162  RegionCounter(CodeGenPGO &PGO, const Stmt *S)
163    : PGO(&PGO), Counter(PGO.getRegionCounter(S)),
164      Count(PGO.getRegionCount(Counter)),
165      ParentCount(PGO.getCurrentRegionCount()), Adjust(0) {}
166
167  /// Get the value of the counter. In most cases this is the number of times
168  /// the region of the counter was entered, but for switch labels it's the
169  /// number of direct jumps to that label.
170  uint64_t getCount() const { return Count; }
171
172  /// Get the value of the counter with adjustments applied. Adjustments occur
173  /// when control enters or leaves the region abnormally; i.e., if there is a
174  /// jump to a label within the region, or if the function can return from
175  /// within the region. The adjusted count, then, is the value of the counter
176  /// at the end of the region.
177  uint64_t getAdjustedCount() const {
178    return Count + Adjust;
179  }
180
181  /// Get the value of the counter in this region's parent, i.e., the region
182  /// that was active when this region began. This is useful for deriving
183  /// counts in implicitly counted regions, like the false case of a condition
184  /// or the normal exits of a loop.
185  uint64_t getParentCount() const { return ParentCount; }
186
187  /// Activate the counter by emitting an increment and starting to track
188  /// adjustments. If AddIncomingFallThrough is true, the current region count
189  /// will be added to the counter for the purposes of tracking the region.
190  void beginRegion(CGBuilderTy &Builder, bool AddIncomingFallThrough=false) {
191    beginRegion(AddIncomingFallThrough);
192    PGO->emitCounterIncrement(Builder, Counter);
193  }
194  void beginRegion(bool AddIncomingFallThrough=false) {
195    RegionCount = Count;
196    if (AddIncomingFallThrough)
197      RegionCount += PGO->getCurrentRegionCount();
198    PGO->setCurrentRegionCount(RegionCount);
199  }
200
201  /// For counters on boolean branches, begins tracking adjustments for the
202  /// uncounted path.
203  void beginElseRegion() {
204    RegionCount = ParentCount - Count;
205    PGO->setCurrentRegionCount(RegionCount);
206  }
207
208  /// Reset the current region count.
209  void setCurrentRegionCount(uint64_t CurrentCount) {
210    RegionCount = CurrentCount;
211    PGO->setCurrentRegionCount(RegionCount);
212  }
213
214  /// Adjust for non-local control flow after emitting a subexpression or
215  /// substatement. This must be called to account for constructs such as gotos,
216  /// labels, and returns, so that we can ensure that our region's count is
217  /// correct in the code that follows.
218  void adjustForControlFlow() {
219    Adjust += PGO->getCurrentRegionCount() - RegionCount;
220    // Reset the region count in case this is called again later.
221    RegionCount = PGO->getCurrentRegionCount();
222  }
223
224  /// Commit all adjustments to the current region. If the region is a loop,
225  /// the LoopAdjust value should be the count of all the breaks and continues
226  /// from the loop, to compensate for those counts being deducted from the
227  /// adjustments for the body of the loop.
228  void applyAdjustmentsToRegion(uint64_t LoopAdjust) {
229    PGO->setCurrentRegionCount(ParentCount + Adjust + LoopAdjust);
230  }
231};
232
233}  // end namespace CodeGen
234}  // end namespace clang
235
236#endif
237