1 /*===- InstrProfilingValue.c - Support library for PGO instrumentation ----===*\
2 |*
3 |*                     The LLVM Compiler Infrastructure
4 |*
5 |* This file is distributed under the University of Illinois Open Source
6 |* License. See LICENSE.TXT for details.
7 |*
8 \*===----------------------------------------------------------------------===*/
9 
10 #include "InstrProfiling.h"
11 #include "InstrProfilingInternal.h"
12 #include <limits.h>
13 #include <stdio.h>
14 #include <stdlib.h>
15 #include <string.h>
16 #define INSTR_PROF_VALUE_PROF_DATA
17 #define INSTR_PROF_COMMON_API_IMPL
18 #include "InstrProfData.inc"
19 
20 #define PROF_OOM(Msg) PROF_ERR(Msg ":%s\n", "Out of memory");
21 #define PROF_OOM_RETURN(Msg)                                                   \
22   {                                                                            \
23     PROF_OOM(Msg)                                                              \
24     return 0;                                                                  \
25   }
26 
27 #if COMPILER_RT_HAS_ATOMICS != 1
28 COMPILER_RT_VISIBILITY
BoolCmpXchg(void ** Ptr,void * OldV,void * NewV)29 uint32_t BoolCmpXchg(void **Ptr, void *OldV, void *NewV) {
30   void *R = *Ptr;
31   if (R == OldV) {
32     *Ptr = NewV;
33     return 1;
34   }
35   return 0;
36 }
37 #endif
38 
39 /* This method is only used in value profiler mock testing.  */
40 COMPILER_RT_VISIBILITY void
__llvm_profile_set_num_value_sites(__llvm_profile_data * Data,uint32_t ValueKind,uint16_t NumValueSites)41 __llvm_profile_set_num_value_sites(__llvm_profile_data *Data,
42                                    uint32_t ValueKind, uint16_t NumValueSites) {
43   *((uint16_t *)&Data->NumValueSites[ValueKind]) = NumValueSites;
44 }
45 
46 /* This method is only used in value profiler mock testing.  */
47 COMPILER_RT_VISIBILITY const __llvm_profile_data *
__llvm_profile_iterate_data(const __llvm_profile_data * Data)48 __llvm_profile_iterate_data(const __llvm_profile_data *Data) {
49   return Data + 1;
50 }
51 
52 /* This method is only used in value profiler mock testing.  */
53 COMPILER_RT_VISIBILITY void *
__llvm_get_function_addr(const __llvm_profile_data * Data)54 __llvm_get_function_addr(const __llvm_profile_data *Data) {
55   return Data->FunctionPointer;
56 }
57 
58 /* Allocate an array that holds the pointers to the linked lists of
59  * value profile counter nodes. The number of element of the array
60  * is the total number of value profile sites instrumented. Returns
61  * 0 if allocation fails.
62  */
63 
allocateValueProfileCounters(__llvm_profile_data * Data)64 static int allocateValueProfileCounters(__llvm_profile_data *Data) {
65   uint64_t NumVSites = 0;
66   uint32_t VKI;
67   for (VKI = IPVK_First; VKI <= IPVK_Last; ++VKI)
68     NumVSites += Data->NumValueSites[VKI];
69 
70   ValueProfNode **Mem =
71       (ValueProfNode **)calloc(NumVSites, sizeof(ValueProfNode *));
72   if (!Mem)
73     return 0;
74   if (!COMPILER_RT_BOOL_CMPXCHG(&Data->Values, 0, Mem)) {
75     free(Mem);
76     return 0;
77   }
78   return 1;
79 }
80 
deallocateValueProfileCounters(__llvm_profile_data * Data)81 static void deallocateValueProfileCounters(__llvm_profile_data *Data) {
82   uint64_t NumVSites = 0, I;
83   uint32_t VKI;
84   if (!Data->Values)
85     return;
86   for (VKI = IPVK_First; VKI <= IPVK_Last; ++VKI)
87     NumVSites += Data->NumValueSites[VKI];
88   for (I = 0; I < NumVSites; I++) {
89     ValueProfNode *Node = ((ValueProfNode **)Data->Values)[I];
90     while (Node) {
91       ValueProfNode *Next = Node->Next;
92       free(Node);
93       Node = Next;
94     }
95   }
96   free(Data->Values);
97 }
98 
99 COMPILER_RT_VISIBILITY void
__llvm_profile_instrument_target(uint64_t TargetValue,void * Data,uint32_t CounterIndex)100 __llvm_profile_instrument_target(uint64_t TargetValue, void *Data,
101                                  uint32_t CounterIndex) {
102 
103   __llvm_profile_data *PData = (__llvm_profile_data *)Data;
104   if (!PData)
105     return;
106 
107   if (!PData->Values) {
108     if (!allocateValueProfileCounters(PData))
109       return;
110   }
111 
112   ValueProfNode **ValueCounters = (ValueProfNode **)PData->Values;
113   ValueProfNode *PrevVNode = NULL;
114   ValueProfNode *CurrentVNode = ValueCounters[CounterIndex];
115 
116   uint8_t VDataCount = 0;
117   while (CurrentVNode) {
118     if (TargetValue == CurrentVNode->VData.Value) {
119       CurrentVNode->VData.Count++;
120       return;
121     }
122     PrevVNode = CurrentVNode;
123     CurrentVNode = CurrentVNode->Next;
124     ++VDataCount;
125   }
126 
127   if (VDataCount >= UCHAR_MAX)
128     return;
129 
130   CurrentVNode = (ValueProfNode *)calloc(1, sizeof(ValueProfNode));
131   if (!CurrentVNode)
132     return;
133 
134   CurrentVNode->VData.Value = TargetValue;
135   CurrentVNode->VData.Count++;
136 
137   uint32_t Success = 0;
138   if (!ValueCounters[CounterIndex])
139     Success =
140         COMPILER_RT_BOOL_CMPXCHG(&ValueCounters[CounterIndex], 0, CurrentVNode);
141   else if (PrevVNode && !PrevVNode->Next)
142     Success = COMPILER_RT_BOOL_CMPXCHG(&(PrevVNode->Next), 0, CurrentVNode);
143 
144   if (!Success) {
145     free(CurrentVNode);
146     return;
147   }
148 }
149 
150 /* For multi-threaded programs, while the profile is being dumped, other
151    threads may still be updating the value profile data and creating new
152    value entries. To accommadate this, we need to add extra bytes to the
153    data buffer. The size of the extra space is controlled by an environment
154    variable. */
getVprofExtraBytes()155 static unsigned getVprofExtraBytes() {
156   const char *ExtraStr =
157       GetEnvHook ? GetEnvHook("LLVM_VALUE_PROF_BUFFER_EXTRA") : 0;
158   if (!ExtraStr || !ExtraStr[0])
159     return 1024;
160   return (unsigned)atoi(ExtraStr);
161 }
162 
163 /* Extract the value profile data info from the runtime. */
164 #define DEF_VALUE_RECORD(R, NS, V)                                             \
165   ValueProfRuntimeRecord R;                                                    \
166   if (initializeValueProfRuntimeRecord(&R, NS, V))                             \
167     PROF_OOM_RETURN("Failed to write value profile data ");
168 
169 #define DTOR_VALUE_RECORD(R) finalizeValueProfRuntimeRecord(&R);
170 
171 COMPILER_RT_VISIBILITY uint64_t
__llvm_profile_gather_value_data(uint8_t ** VDataArray)172 __llvm_profile_gather_value_data(uint8_t **VDataArray) {
173   size_t S = 0, RealSize = 0, BufferCapacity = 0, Extra = 0;
174   __llvm_profile_data *I;
175   if (!VDataArray)
176     PROF_OOM_RETURN("Failed to write value profile data ");
177 
178   const __llvm_profile_data *DataEnd = __llvm_profile_end_data();
179   const __llvm_profile_data *DataBegin = __llvm_profile_begin_data();
180 
181   /*
182    * Compute the total Size of the buffer to hold ValueProfData
183    * structures for functions with value profile data.
184    */
185   for (I = (__llvm_profile_data *)DataBegin; I != DataEnd; ++I) {
186 
187     DEF_VALUE_RECORD(R, I->NumValueSites, I->Values);
188 
189     /* Compute the size of ValueProfData from this runtime record.  */
190     if (getNumValueKindsRT(&R) != 0)
191       S += getValueProfDataSizeRT(&R);
192 
193     DTOR_VALUE_RECORD(R);
194   }
195   /* No value sites or no value profile data is collected. */
196   if (!S)
197     return 0;
198 
199   Extra = getVprofExtraBytes();
200   BufferCapacity = S + Extra;
201   *VDataArray = calloc(BufferCapacity, sizeof(uint8_t));
202   if (!*VDataArray)
203     PROF_OOM_RETURN("Failed to write value profile data ");
204 
205   ValueProfData *VD = (ValueProfData *)(*VDataArray);
206   /*
207    * Extract value profile data and write into ValueProfData structure
208    * one by one. Note that new value profile data added to any value
209    * site (from another thread) after the ValueProfRuntimeRecord is
210    * initialized (when the profile data snapshot is taken) won't be
211    * collected. This is not a problem as those dropped value will have
212    * very low taken count.
213    */
214   for (I = (__llvm_profile_data *)DataBegin; I != DataEnd; ++I) {
215     DEF_VALUE_RECORD(R, I->NumValueSites, I->Values);
216     if (getNumValueKindsRT(&R) == 0)
217       continue;
218 
219     /* Record R has taken a snapshot of the VP data at this point. Newly
220        added VP data for this function will be dropped.  */
221     /* Check if there is enough space.  */
222     if (BufferCapacity - RealSize < getValueProfDataSizeRT(&R)) {
223       PROF_ERR("Value profile data is dropped :%s \n",
224                "Out of buffer space. Use environment "
225                " LLVM_VALUE_PROF_BUFFER_EXTRA to allocate more");
226       I->Values = 0;
227     }
228 
229     serializeValueProfDataFromRT(&R, VD);
230     deallocateValueProfileCounters(I);
231     I->Values = VD;
232     RealSize += VD->TotalSize;
233     VD = (ValueProfData *)((char *)VD + VD->TotalSize);
234     DTOR_VALUE_RECORD(R);
235   }
236 
237   return RealSize;
238 }
239