1/*===- InstrProfilingValue.c - Support library for PGO instrumentation ----===*\
2|*
3|*                     The LLVM Compiler Infrastructure
4|*
5|* This file is distributed under the University of Illinois Open Source
6|* License. See LICENSE.TXT for details.
7|*
8\*===----------------------------------------------------------------------===*/
9
10#include "InstrProfiling.h"
11#include "InstrProfilingInternal.h"
12#include <limits.h>
13#include <stdio.h>
14#include <stdlib.h>
15#include <string.h>
16#define INSTR_PROF_VALUE_PROF_DATA
17#define INSTR_PROF_COMMON_API_IMPL
18#include "InstrProfData.inc"
19
20#define PROF_OOM(Msg) PROF_ERR(Msg ":%s\n", "Out of memory");
21#define PROF_OOM_RETURN(Msg)                                                   \
22  {                                                                            \
23    PROF_OOM(Msg)                                                              \
24    return 0;                                                                  \
25  }
26
27#if COMPILER_RT_HAS_ATOMICS != 1
28COMPILER_RT_VISIBILITY
29uint32_t BoolCmpXchg(void **Ptr, void *OldV, void *NewV) {
30  void *R = *Ptr;
31  if (R == OldV) {
32    *Ptr = NewV;
33    return 1;
34  }
35  return 0;
36}
37#endif
38
39/* This method is only used in value profiler mock testing.  */
40COMPILER_RT_VISIBILITY void
41__llvm_profile_set_num_value_sites(__llvm_profile_data *Data,
42                                   uint32_t ValueKind, uint16_t NumValueSites) {
43  *((uint16_t *)&Data->NumValueSites[ValueKind]) = NumValueSites;
44}
45
46/* This method is only used in value profiler mock testing.  */
47COMPILER_RT_VISIBILITY const __llvm_profile_data *
48__llvm_profile_iterate_data(const __llvm_profile_data *Data) {
49  return Data + 1;
50}
51
52/* This method is only used in value profiler mock testing.  */
53COMPILER_RT_VISIBILITY void *
54__llvm_get_function_addr(const __llvm_profile_data *Data) {
55  return Data->FunctionPointer;
56}
57
58/* Allocate an array that holds the pointers to the linked lists of
59 * value profile counter nodes. The number of element of the array
60 * is the total number of value profile sites instrumented. Returns
61 * 0 if allocation fails.
62 */
63
64static int allocateValueProfileCounters(__llvm_profile_data *Data) {
65  uint64_t NumVSites = 0;
66  uint32_t VKI;
67  for (VKI = IPVK_First; VKI <= IPVK_Last; ++VKI)
68    NumVSites += Data->NumValueSites[VKI];
69
70  ValueProfNode **Mem =
71      (ValueProfNode **)calloc(NumVSites, sizeof(ValueProfNode *));
72  if (!Mem)
73    return 0;
74  if (!COMPILER_RT_BOOL_CMPXCHG(&Data->Values, 0, Mem)) {
75    free(Mem);
76    return 0;
77  }
78  return 1;
79}
80
81static void deallocateValueProfileCounters(__llvm_profile_data *Data) {
82  uint64_t NumVSites = 0, I;
83  uint32_t VKI;
84  if (!Data->Values)
85    return;
86  for (VKI = IPVK_First; VKI <= IPVK_Last; ++VKI)
87    NumVSites += Data->NumValueSites[VKI];
88  for (I = 0; I < NumVSites; I++) {
89    ValueProfNode *Node = ((ValueProfNode **)Data->Values)[I];
90    while (Node) {
91      ValueProfNode *Next = Node->Next;
92      free(Node);
93      Node = Next;
94    }
95  }
96  free(Data->Values);
97}
98
99COMPILER_RT_VISIBILITY void
100__llvm_profile_instrument_target(uint64_t TargetValue, void *Data,
101                                 uint32_t CounterIndex) {
102
103  __llvm_profile_data *PData = (__llvm_profile_data *)Data;
104  if (!PData)
105    return;
106
107  if (!PData->Values) {
108    if (!allocateValueProfileCounters(PData))
109      return;
110  }
111
112  ValueProfNode **ValueCounters = (ValueProfNode **)PData->Values;
113  ValueProfNode *PrevVNode = NULL;
114  ValueProfNode *CurrentVNode = ValueCounters[CounterIndex];
115
116  uint8_t VDataCount = 0;
117  while (CurrentVNode) {
118    if (TargetValue == CurrentVNode->VData.Value) {
119      CurrentVNode->VData.Count++;
120      return;
121    }
122    PrevVNode = CurrentVNode;
123    CurrentVNode = CurrentVNode->Next;
124    ++VDataCount;
125  }
126
127  if (VDataCount >= UCHAR_MAX)
128    return;
129
130  CurrentVNode = (ValueProfNode *)calloc(1, sizeof(ValueProfNode));
131  if (!CurrentVNode)
132    return;
133
134  CurrentVNode->VData.Value = TargetValue;
135  CurrentVNode->VData.Count++;
136
137  uint32_t Success = 0;
138  if (!ValueCounters[CounterIndex])
139    Success =
140        COMPILER_RT_BOOL_CMPXCHG(&ValueCounters[CounterIndex], 0, CurrentVNode);
141  else if (PrevVNode && !PrevVNode->Next)
142    Success = COMPILER_RT_BOOL_CMPXCHG(&(PrevVNode->Next), 0, CurrentVNode);
143
144  if (!Success) {
145    free(CurrentVNode);
146    return;
147  }
148}
149
150/* For multi-threaded programs, while the profile is being dumped, other
151   threads may still be updating the value profile data and creating new
152   value entries. To accommadate this, we need to add extra bytes to the
153   data buffer. The size of the extra space is controlled by an environment
154   variable. */
155static unsigned getVprofExtraBytes() {
156  const char *ExtraStr =
157      GetEnvHook ? GetEnvHook("LLVM_VALUE_PROF_BUFFER_EXTRA") : 0;
158  if (!ExtraStr || !ExtraStr[0])
159    return 1024;
160  return (unsigned)atoi(ExtraStr);
161}
162
163/* Extract the value profile data info from the runtime. */
164#define DEF_VALUE_RECORD(R, NS, V)                                             \
165  ValueProfRuntimeRecord R;                                                    \
166  if (initializeValueProfRuntimeRecord(&R, NS, V))                             \
167    PROF_OOM_RETURN("Failed to write value profile data ");
168
169#define DTOR_VALUE_RECORD(R) finalizeValueProfRuntimeRecord(&R);
170
171COMPILER_RT_VISIBILITY uint64_t
172__llvm_profile_gather_value_data(uint8_t **VDataArray) {
173  size_t S = 0, RealSize = 0, BufferCapacity = 0, Extra = 0;
174  __llvm_profile_data *I;
175  if (!VDataArray)
176    PROF_OOM_RETURN("Failed to write value profile data ");
177
178  const __llvm_profile_data *DataEnd = __llvm_profile_end_data();
179  const __llvm_profile_data *DataBegin = __llvm_profile_begin_data();
180
181  /*
182   * Compute the total Size of the buffer to hold ValueProfData
183   * structures for functions with value profile data.
184   */
185  for (I = (__llvm_profile_data *)DataBegin; I != DataEnd; ++I) {
186
187    DEF_VALUE_RECORD(R, I->NumValueSites, I->Values);
188
189    /* Compute the size of ValueProfData from this runtime record.  */
190    if (getNumValueKindsRT(&R) != 0)
191      S += getValueProfDataSizeRT(&R);
192
193    DTOR_VALUE_RECORD(R);
194  }
195  /* No value sites or no value profile data is collected. */
196  if (!S)
197    return 0;
198
199  Extra = getVprofExtraBytes();
200  BufferCapacity = S + Extra;
201  *VDataArray = calloc(BufferCapacity, sizeof(uint8_t));
202  if (!*VDataArray)
203    PROF_OOM_RETURN("Failed to write value profile data ");
204
205  ValueProfData *VD = (ValueProfData *)(*VDataArray);
206  /*
207   * Extract value profile data and write into ValueProfData structure
208   * one by one. Note that new value profile data added to any value
209   * site (from another thread) after the ValueProfRuntimeRecord is
210   * initialized (when the profile data snapshot is taken) won't be
211   * collected. This is not a problem as those dropped value will have
212   * very low taken count.
213   */
214  for (I = (__llvm_profile_data *)DataBegin; I != DataEnd; ++I) {
215    DEF_VALUE_RECORD(R, I->NumValueSites, I->Values);
216    if (getNumValueKindsRT(&R) == 0)
217      continue;
218
219    /* Record R has taken a snapshot of the VP data at this point. Newly
220       added VP data for this function will be dropped.  */
221    /* Check if there is enough space.  */
222    if (BufferCapacity - RealSize < getValueProfDataSizeRT(&R)) {
223      PROF_ERR("Value profile data is dropped :%s \n",
224               "Out of buffer space. Use environment "
225               " LLVM_VALUE_PROF_BUFFER_EXTRA to allocate more");
226      I->Values = 0;
227    }
228
229    serializeValueProfDataFromRT(&R, VD);
230    deallocateValueProfileCounters(I);
231    I->Values = VD;
232    RealSize += VD->TotalSize;
233    VD = (ValueProfData *)((char *)VD + VD->TotalSize);
234    DTOR_VALUE_RECORD(R);
235  }
236
237  return RealSize;
238}
239