1 //===--- CodeGenPGO.h - PGO Instrumentation for LLVM CodeGen ----*- C++ -*-===//
2 //
3 //                     The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // Instrumentation-based profile-guided optimization
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #ifndef LLVM_CLANG_LIB_CODEGEN_CODEGENPGO_H
15 #define LLVM_CLANG_LIB_CODEGEN_CODEGENPGO_H
16 
17 #include "CGBuilder.h"
18 #include "CodeGenModule.h"
19 #include "CodeGenTypes.h"
20 #include "clang/Frontend/CodeGenOptions.h"
21 #include "llvm/ADT/StringMap.h"
22 #include "llvm/Support/MemoryBuffer.h"
23 #include <memory>
24 
25 namespace clang {
26 namespace CodeGen {
27 class RegionCounter;
28 
29 /// Per-function PGO state. This class should generally not be used directly,
30 /// but instead through the CodeGenFunction and RegionCounter types.
31 class CodeGenPGO {
32 private:
33   CodeGenModule &CGM;
34   std::string FuncName;
35   llvm::GlobalVariable *FuncNameVar;
36 
37   unsigned NumRegionCounters;
38   uint64_t FunctionHash;
39   std::unique_ptr<llvm::DenseMap<const Stmt *, unsigned>> RegionCounterMap;
40   std::unique_ptr<llvm::DenseMap<const Stmt *, uint64_t>> StmtCountMap;
41   std::vector<uint64_t> RegionCounts;
42   uint64_t CurrentRegionCount;
43   /// \brief A flag that is set to true when this function doesn't need
44   /// to have coverage mapping data.
45   bool SkipCoverageMapping;
46 
47 public:
CodeGenPGO(CodeGenModule & CGM)48   CodeGenPGO(CodeGenModule &CGM)
49       : CGM(CGM), NumRegionCounters(0), FunctionHash(0), CurrentRegionCount(0),
50         SkipCoverageMapping(false) {}
51 
52   /// Whether or not we have PGO region data for the current function. This is
53   /// false both when we have no data at all and when our data has been
54   /// discarded.
haveRegionCounts()55   bool haveRegionCounts() const { return !RegionCounts.empty(); }
56 
57   /// Return the counter value of the current region.
getCurrentRegionCount()58   uint64_t getCurrentRegionCount() const { return CurrentRegionCount; }
59 
60   /// Set the counter value for the current region. This is used to keep track
61   /// of changes to the most recent counter from control flow and non-local
62   /// exits.
setCurrentRegionCount(uint64_t Count)63   void setCurrentRegionCount(uint64_t Count) { CurrentRegionCount = Count; }
64 
65   /// Indicate that the current region is never reached, and thus should have a
66   /// counter value of zero. This is important so that subsequent regions can
67   /// correctly track their parent counts.
setCurrentRegionUnreachable()68   void setCurrentRegionUnreachable() { setCurrentRegionCount(0); }
69 
70   /// Check if an execution count is known for a given statement. If so, return
71   /// true and put the value in Count; else return false.
getStmtCount(const Stmt * S)72   Optional<uint64_t> getStmtCount(const Stmt *S) {
73     if (!StmtCountMap)
74       return None;
75     auto I = StmtCountMap->find(S);
76     if (I == StmtCountMap->end())
77       return None;
78     return I->second;
79   }
80 
81   /// If the execution count for the current statement is known, record that
82   /// as the current count.
setCurrentStmt(const Stmt * S)83   void setCurrentStmt(const Stmt *S) {
84     if (auto Count = getStmtCount(S))
85       setCurrentRegionCount(*Count);
86   }
87 
88   /// Calculate branch weights appropriate for PGO data
89   llvm::MDNode *createBranchWeights(uint64_t TrueCount, uint64_t FalseCount);
90   llvm::MDNode *createBranchWeights(ArrayRef<uint64_t> Weights);
91   llvm::MDNode *createLoopWeights(const Stmt *Cond, RegionCounter &Cnt);
92 
93   /// Check if we need to emit coverage mapping for a given declaration
94   void checkGlobalDecl(GlobalDecl GD);
95   /// Assign counters to regions and configure them for PGO of a given
96   /// function. Does nothing if instrumentation is not enabled and either
97   /// generates global variables or associates PGO data with each of the
98   /// counters depending on whether we are generating or using instrumentation.
99   void assignRegionCounters(const Decl *D, llvm::Function *Fn);
100   /// Emit a coverage mapping range with a counter zero
101   /// for an unused declaration.
102   void emitEmptyCounterMapping(const Decl *D, StringRef FuncName,
103                                llvm::GlobalValue::LinkageTypes Linkage);
104 private:
105   void setFuncName(llvm::Function *Fn);
106   void setFuncName(StringRef Name, llvm::GlobalValue::LinkageTypes Linkage);
107   void createFuncNameVar(llvm::GlobalValue::LinkageTypes Linkage);
108   void mapRegionCounters(const Decl *D);
109   void computeRegionCounts(const Decl *D);
110   void applyFunctionAttributes(llvm::IndexedInstrProfReader *PGOReader,
111                                llvm::Function *Fn);
112   void loadRegionCounts(llvm::IndexedInstrProfReader *PGOReader,
113                         bool IsInMainFile);
114   void emitCounterVariables();
115   void emitCounterRegionMapping(const Decl *D);
116 
117   /// Emit code to increment the counter at the given index
118   void emitCounterIncrement(CGBuilderTy &Builder, unsigned Counter);
119 
120   /// Return the region counter for the given statement. This should only be
121   /// called on statements that have a dedicated counter.
getRegionCounter(const Stmt * S)122   unsigned getRegionCounter(const Stmt *S) {
123     if (!RegionCounterMap)
124       return 0;
125     return (*RegionCounterMap)[S];
126   }
127 
128   /// Return the region count for the counter at the given index.
getRegionCount(unsigned Counter)129   uint64_t getRegionCount(unsigned Counter) {
130     if (!haveRegionCounts())
131       return 0;
132     return RegionCounts[Counter];
133   }
134 
135   friend class RegionCounter;
136 };
137 
138 /// A counter for a particular region. This is the primary interface through
139 /// which clients manage PGO counters and their values.
140 class RegionCounter {
141   CodeGenPGO *PGO;
142   unsigned Counter;
143   uint64_t Count;
144   uint64_t ParentCount;
145   uint64_t RegionCount;
146   int64_t Adjust;
147 
RegionCounter(CodeGenPGO & PGO,unsigned CounterIndex)148   RegionCounter(CodeGenPGO &PGO, unsigned CounterIndex)
149     : PGO(&PGO), Counter(CounterIndex), Count(PGO.getRegionCount(Counter)),
150       ParentCount(PGO.getCurrentRegionCount()), Adjust(0) {}
151 
152 public:
RegionCounter(CodeGenPGO & PGO,const Stmt * S)153   RegionCounter(CodeGenPGO &PGO, const Stmt *S)
154     : PGO(&PGO), Counter(PGO.getRegionCounter(S)),
155       Count(PGO.getRegionCount(Counter)),
156       ParentCount(PGO.getCurrentRegionCount()), Adjust(0) {}
157 
158   /// Get the value of the counter. In most cases this is the number of times
159   /// the region of the counter was entered, but for switch labels it's the
160   /// number of direct jumps to that label.
getCount()161   uint64_t getCount() const { return Count; }
162 
163   /// Get the value of the counter with adjustments applied. Adjustments occur
164   /// when control enters or leaves the region abnormally; i.e., if there is a
165   /// jump to a label within the region, or if the function can return from
166   /// within the region. The adjusted count, then, is the value of the counter
167   /// at the end of the region.
getAdjustedCount()168   uint64_t getAdjustedCount() const {
169     return Count + Adjust;
170   }
171 
172   /// Get the value of the counter in this region's parent, i.e., the region
173   /// that was active when this region began. This is useful for deriving
174   /// counts in implicitly counted regions, like the false case of a condition
175   /// or the normal exits of a loop.
getParentCount()176   uint64_t getParentCount() const { return ParentCount; }
177 
178   /// Activate the counter by emitting an increment and starting to track
179   /// adjustments. If AddIncomingFallThrough is true, the current region count
180   /// will be added to the counter for the purposes of tracking the region.
181   void beginRegion(CGBuilderTy &Builder, bool AddIncomingFallThrough=false) {
182     beginRegion(AddIncomingFallThrough);
183     PGO->emitCounterIncrement(Builder, Counter);
184   }
185   void beginRegion(bool AddIncomingFallThrough=false) {
186     RegionCount = Count;
187     if (AddIncomingFallThrough)
188       RegionCount += PGO->getCurrentRegionCount();
189     PGO->setCurrentRegionCount(RegionCount);
190   }
191 
192   /// For counters on boolean branches, begins tracking adjustments for the
193   /// uncounted path.
beginElseRegion()194   void beginElseRegion() {
195     RegionCount = ParentCount - Count;
196     PGO->setCurrentRegionCount(RegionCount);
197   }
198 
199   /// Reset the current region count.
setCurrentRegionCount(uint64_t CurrentCount)200   void setCurrentRegionCount(uint64_t CurrentCount) {
201     RegionCount = CurrentCount;
202     PGO->setCurrentRegionCount(RegionCount);
203   }
204 
205   /// Adjust for non-local control flow after emitting a subexpression or
206   /// substatement. This must be called to account for constructs such as gotos,
207   /// labels, and returns, so that we can ensure that our region's count is
208   /// correct in the code that follows.
adjustForControlFlow()209   void adjustForControlFlow() {
210     Adjust += PGO->getCurrentRegionCount() - RegionCount;
211     // Reset the region count in case this is called again later.
212     RegionCount = PGO->getCurrentRegionCount();
213   }
214 
215   /// Commit all adjustments to the current region. If the region is a loop,
216   /// the LoopAdjust value should be the count of all the breaks and continues
217   /// from the loop, to compensate for those counts being deducted from the
218   /// adjustments for the body of the loop.
applyAdjustmentsToRegion(uint64_t LoopAdjust)219   void applyAdjustmentsToRegion(uint64_t LoopAdjust) {
220     PGO->setCurrentRegionCount(ParentCount + Adjust + LoopAdjust);
221   }
222 };
223 
224 }  // end namespace CodeGen
225 }  // end namespace clang
226 
227 #endif
228