1/*===- InstrProfilingWriter.c - Write instrumentation to a file or buffer -===*\
2|*
3|*                     The LLVM Compiler Infrastructure
4|*
5|* This file is distributed under the University of Illinois Open Source
6|* License. See LICENSE.TXT for details.
7|*
8\*===----------------------------------------------------------------------===*/
9
10#include "InstrProfiling.h"
11#include "InstrProfilingInternal.h"
12#ifdef _MSC_VER
13/* For _alloca */
14#include <malloc.h>
15#endif
16#include <string.h>
17
18#define INSTR_PROF_VALUE_PROF_DATA
19#include "InstrProfData.inc"
20
21COMPILER_RT_VISIBILITY void (*FreeHook)(void *) = NULL;
22static ProfBufferIO TheBufferIO;
23#define VP_BUFFER_SIZE 8 * 1024
24static uint8_t BufferIOBuffer[VP_BUFFER_SIZE];
25static InstrProfValueData VPDataArray[16];
26static uint32_t VPDataArraySize = sizeof(VPDataArray) / sizeof(*VPDataArray);
27
28COMPILER_RT_VISIBILITY uint8_t *DynamicBufferIOBuffer = 0;
29COMPILER_RT_VISIBILITY uint32_t VPBufferSize = 0;
30
31/* The buffer writer is reponsponsible in keeping writer state
32 * across the call.
33 */
34COMPILER_RT_VISIBILITY uint32_t lprofBufferWriter(ProfDataIOVec *IOVecs,
35                                                  uint32_t NumIOVecs,
36                                                  void **WriterCtx) {
37  uint32_t I;
38  char **Buffer = (char **)WriterCtx;
39  for (I = 0; I < NumIOVecs; I++) {
40    size_t Length = IOVecs[I].ElmSize * IOVecs[I].NumElm;
41    memcpy(*Buffer, IOVecs[I].Data, Length);
42    *Buffer += Length;
43  }
44  return 0;
45}
46
47static void llvmInitBufferIO(ProfBufferIO *BufferIO, WriterCallback FileWriter,
48                             void *File, uint8_t *Buffer, uint32_t BufferSz) {
49  BufferIO->File = File;
50  BufferIO->FileWriter = FileWriter;
51  BufferIO->BufferStart = Buffer;
52  BufferIO->BufferSz = BufferSz;
53  BufferIO->CurOffset = 0;
54}
55
56COMPILER_RT_VISIBILITY ProfBufferIO *
57lprofCreateBufferIO(WriterCallback FileWriter, void *File) {
58  uint8_t *Buffer = DynamicBufferIOBuffer;
59  uint32_t BufferSize = VPBufferSize;
60  if (!Buffer) {
61    Buffer = &BufferIOBuffer[0];
62    BufferSize = sizeof(BufferIOBuffer);
63  }
64  llvmInitBufferIO(&TheBufferIO, FileWriter, File, Buffer, BufferSize);
65  return &TheBufferIO;
66}
67
68COMPILER_RT_VISIBILITY void lprofDeleteBufferIO(ProfBufferIO *BufferIO) {
69  if (DynamicBufferIOBuffer) {
70    FreeHook(DynamicBufferIOBuffer);
71    DynamicBufferIOBuffer = 0;
72    VPBufferSize = 0;
73  }
74}
75
76COMPILER_RT_VISIBILITY int
77lprofBufferIOWrite(ProfBufferIO *BufferIO, const uint8_t *Data, uint32_t Size) {
78  /* Buffer is not large enough, it is time to flush.  */
79  if (Size + BufferIO->CurOffset > BufferIO->BufferSz) {
80    if (lprofBufferIOFlush(BufferIO) != 0)
81      return -1;
82  }
83  /* Special case, bypass the buffer completely. */
84  ProfDataIOVec IO[] = {{Data, sizeof(uint8_t), Size}};
85  if (Size > BufferIO->BufferSz) {
86    if (BufferIO->FileWriter(IO, 1, &BufferIO->File))
87      return -1;
88  } else {
89    /* Write the data to buffer */
90    uint8_t *Buffer = BufferIO->BufferStart + BufferIO->CurOffset;
91    lprofBufferWriter(IO, 1, (void **)&Buffer);
92    BufferIO->CurOffset = Buffer - BufferIO->BufferStart;
93  }
94  return 0;
95}
96
97COMPILER_RT_VISIBILITY int lprofBufferIOFlush(ProfBufferIO *BufferIO) {
98  if (BufferIO->CurOffset) {
99    ProfDataIOVec IO[] = {
100        {BufferIO->BufferStart, sizeof(uint8_t), BufferIO->CurOffset}};
101    if (BufferIO->FileWriter(IO, 1, &BufferIO->File))
102      return -1;
103    BufferIO->CurOffset = 0;
104  }
105  return 0;
106}
107
108/* Write out value profile data for function specified with \c Data.
109 * The implementation does not use the method \c serializeValueProfData
110 * which depends on dynamic memory allocation. In this implementation,
111 * value profile data is written out to \c BufferIO piecemeal.
112 */
113static int writeOneValueProfData(ProfBufferIO *BufferIO,
114                                 VPDataReaderType *VPDataReader,
115                                 const __llvm_profile_data *Data) {
116  unsigned I, NumValueKinds = 0;
117  ValueProfData VPHeader;
118  uint8_t *SiteCountArray[IPVK_Last + 1];
119
120  for (I = 0; I <= IPVK_Last; I++) {
121    if (!Data->NumValueSites[I])
122      SiteCountArray[I] = 0;
123    else {
124      uint32_t Sz =
125          VPDataReader->GetValueProfRecordHeaderSize(Data->NumValueSites[I]) -
126          offsetof(ValueProfRecord, SiteCountArray);
127      /* Only use alloca for this small byte array to avoid excessive
128       * stack growth.  */
129      SiteCountArray[I] = (uint8_t *)COMPILER_RT_ALLOCA(Sz);
130      memset(SiteCountArray[I], 0, Sz);
131    }
132  }
133
134  /* If NumValueKinds returned is 0, there is nothing to write, report
135     success and return. This should match the raw profile reader's behavior. */
136  if (!(NumValueKinds = VPDataReader->InitRTRecord(Data, SiteCountArray)))
137    return 0;
138
139  /* First write the header structure. */
140  VPHeader.TotalSize = VPDataReader->GetValueProfDataSize();
141  VPHeader.NumValueKinds = NumValueKinds;
142  if (lprofBufferIOWrite(BufferIO, (const uint8_t *)&VPHeader,
143                         sizeof(ValueProfData)))
144    return -1;
145
146  /* Make sure nothing else needs to be written before value profile
147   * records. */
148  if ((void *)VPDataReader->GetFirstValueProfRecord(&VPHeader) !=
149      (void *)(&VPHeader + 1))
150    return -1;
151
152  /* Write out the value profile record for each value kind
153   * one by one. */
154  for (I = 0; I <= IPVK_Last; I++) {
155    uint32_t J;
156    ValueProfRecord RecordHeader;
157    /* The size of the value prof record header without counting the
158     * site count array .*/
159    uint32_t RecordHeaderSize = offsetof(ValueProfRecord, SiteCountArray);
160    uint32_t SiteCountArraySize;
161
162    if (!Data->NumValueSites[I])
163      continue;
164
165    /* Write out the record header.  */
166    RecordHeader.Kind = I;
167    RecordHeader.NumValueSites = Data->NumValueSites[I];
168    if (lprofBufferIOWrite(BufferIO, (const uint8_t *)&RecordHeader,
169                           RecordHeaderSize))
170      return -1;
171
172    /* Write out the site value count array including padding space. */
173    SiteCountArraySize =
174        VPDataReader->GetValueProfRecordHeaderSize(Data->NumValueSites[I]) -
175        RecordHeaderSize;
176    if (lprofBufferIOWrite(BufferIO, SiteCountArray[I], SiteCountArraySize))
177      return -1;
178
179    /* Write out the value profile data for each value site.  */
180    for (J = 0; J < Data->NumValueSites[I]; J++) {
181      uint32_t NRead, NRemain;
182      ValueProfNode *NextStartNode = 0;
183      NRemain = VPDataReader->GetNumValueDataForSite(I, J);
184      if (!NRemain)
185        continue;
186      /* Read and write out value data in small chunks till it is done. */
187      do {
188        NRead = (NRemain > VPDataArraySize ? VPDataArraySize : NRemain);
189        NextStartNode =
190            VPDataReader->GetValueData(I, /* ValueKind */
191                                       J, /* Site */
192                                       &VPDataArray[0], NextStartNode, NRead);
193        if (lprofBufferIOWrite(BufferIO, (const uint8_t *)&VPDataArray[0],
194                               NRead * sizeof(InstrProfValueData)))
195          return -1;
196        NRemain -= NRead;
197      } while (NRemain != 0);
198    }
199  }
200  /* All done report success.  */
201  return 0;
202}
203
204static int writeValueProfData(WriterCallback Writer, void *WriterCtx,
205                              VPDataReaderType *VPDataReader,
206                              const __llvm_profile_data *DataBegin,
207                              const __llvm_profile_data *DataEnd) {
208  ProfBufferIO *BufferIO;
209  const __llvm_profile_data *DI = 0;
210
211  if (!VPDataReader)
212    return 0;
213
214  BufferIO = lprofCreateBufferIO(Writer, WriterCtx);
215
216  for (DI = DataBegin; DI < DataEnd; DI++) {
217    if (writeOneValueProfData(BufferIO, VPDataReader, DI))
218      return -1;
219  }
220
221  if (lprofBufferIOFlush(BufferIO) != 0)
222    return -1;
223  lprofDeleteBufferIO(BufferIO);
224
225  return 0;
226}
227
228COMPILER_RT_VISIBILITY int lprofWriteData(WriterCallback Writer,
229                                          void *WriterCtx,
230                                          VPDataReaderType *VPDataReader) {
231  /* Match logic in __llvm_profile_write_buffer(). */
232  const __llvm_profile_data *DataBegin = __llvm_profile_begin_data();
233  const __llvm_profile_data *DataEnd = __llvm_profile_end_data();
234  const uint64_t *CountersBegin = __llvm_profile_begin_counters();
235  const uint64_t *CountersEnd = __llvm_profile_end_counters();
236  const char *NamesBegin = __llvm_profile_begin_names();
237  const char *NamesEnd = __llvm_profile_end_names();
238  return lprofWriteDataImpl(Writer, WriterCtx, DataBegin, DataEnd,
239                            CountersBegin, CountersEnd, VPDataReader,
240                            NamesBegin, NamesEnd);
241}
242
243COMPILER_RT_VISIBILITY int
244lprofWriteDataImpl(WriterCallback Writer, void *WriterCtx,
245                   const __llvm_profile_data *DataBegin,
246                   const __llvm_profile_data *DataEnd,
247                   const uint64_t *CountersBegin, const uint64_t *CountersEnd,
248                   VPDataReaderType *VPDataReader, const char *NamesBegin,
249                   const char *NamesEnd) {
250
251  /* Calculate size of sections. */
252  const uint64_t DataSize = __llvm_profile_get_data_size(DataBegin, DataEnd);
253  const uint64_t CountersSize = CountersEnd - CountersBegin;
254  const uint64_t NamesSize = NamesEnd - NamesBegin;
255  const uint64_t Padding = __llvm_profile_get_num_padding_bytes(NamesSize);
256
257  /* Enough zeroes for padding. */
258  const char Zeroes[sizeof(uint64_t)] = {0};
259
260  /* Create the header. */
261  __llvm_profile_header Header;
262
263  if (!DataSize)
264    return 0;
265
266/* Initialize header structure.  */
267#define INSTR_PROF_RAW_HEADER(Type, Name, Init) Header.Name = Init;
268#include "InstrProfData.inc"
269
270  /* Write the data. */
271  ProfDataIOVec IOVec[] = {{&Header, sizeof(__llvm_profile_header), 1},
272                           {DataBegin, sizeof(__llvm_profile_data), DataSize},
273                           {CountersBegin, sizeof(uint64_t), CountersSize},
274                           {NamesBegin, sizeof(uint8_t), NamesSize},
275                           {Zeroes, sizeof(uint8_t), Padding}};
276  if (Writer(IOVec, sizeof(IOVec) / sizeof(*IOVec), &WriterCtx))
277    return -1;
278
279  return writeValueProfData(Writer, WriterCtx, VPDataReader, DataBegin,
280                            DataEnd);
281}
282