InstrProf.h revision f3014761c955345d6e05491608e73228d014afb7
1//===- InstrProf.h - Instrumented profiling format support ------*- C++ -*-===// 2// 3// The LLVM Compiler Infrastructure 4// 5// This file is distributed under the University of Illinois Open Source 6// License. See LICENSE.TXT for details. 7// 8//===----------------------------------------------------------------------===// 9// 10// Instrumentation-based profiling data is generated by instrumented 11// binaries through library functions in compiler-rt, and read by the clang 12// frontend to feed PGO. 13// 14//===----------------------------------------------------------------------===// 15 16#ifndef LLVM_PROFILEDATA_INSTRPROF_H 17#define LLVM_PROFILEDATA_INSTRPROF_H 18 19#include "llvm/ADT/ArrayRef.h" 20#include "llvm/ADT/STLExtras.h" 21#include "llvm/ADT/StringRef.h" 22#include "llvm/ADT/StringSet.h" 23#include "llvm/ADT/Triple.h" 24#include "llvm/IR/GlobalValue.h" 25#include "llvm/IR/ProfileSummary.h" 26#include "llvm/ProfileData/InstrProfData.inc" 27#include "llvm/Support/Compiler.h" 28#include "llvm/Support/Endian.h" 29#include "llvm/Support/Error.h" 30#include "llvm/Support/ErrorHandling.h" 31#include "llvm/Support/Host.h" 32#include "llvm/Support/MD5.h" 33#include "llvm/Support/MathExtras.h" 34#include "llvm/Support/raw_ostream.h" 35#include <algorithm> 36#include <cassert> 37#include <cstddef> 38#include <cstdint> 39#include <cstring> 40#include <list> 41#include <memory> 42#include <string> 43#include <system_error> 44#include <utility> 45#include <vector> 46 47namespace llvm { 48 49class Function; 50class GlobalVariable; 51struct InstrProfRecord; 52class InstrProfSymtab; 53class Instruction; 54class MDNode; 55class Module; 56 57enum InstrProfSectKind { 58#define INSTR_PROF_SECT_ENTRY(Kind, SectNameCommon, SectNameCoff, Prefix) Kind, 59#include "llvm/ProfileData/InstrProfData.inc" 60}; 61 62/// Return the name of the profile section corresponding to \p IPSK. 63/// 64/// The name of the section depends on the object format type \p OF. If 65/// \p AddSegmentInfo is true, a segment prefix and additional linker hints may 66/// be added to the section name (this is the default). 67std::string getInstrProfSectionName(InstrProfSectKind IPSK, 68 Triple::ObjectFormatType OF, 69 bool AddSegmentInfo = true); 70 71/// Return the name profile runtime entry point to do value profiling 72/// for a given site. 73inline StringRef getInstrProfValueProfFuncName() { 74 return INSTR_PROF_VALUE_PROF_FUNC_STR; 75} 76 77/// Return the name profile runtime entry point to do value range profiling. 78inline StringRef getInstrProfValueRangeProfFuncName() { 79 return INSTR_PROF_VALUE_RANGE_PROF_FUNC_STR; 80} 81 82/// Return the name prefix of variables containing instrumented function names. 83inline StringRef getInstrProfNameVarPrefix() { return "__profn_"; } 84 85/// Return the name prefix of variables containing per-function control data. 86inline StringRef getInstrProfDataVarPrefix() { return "__profd_"; } 87 88/// Return the name prefix of profile counter variables. 89inline StringRef getInstrProfCountersVarPrefix() { return "__profc_"; } 90 91/// Return the name prefix of value profile variables. 92inline StringRef getInstrProfValuesVarPrefix() { return "__profvp_"; } 93 94/// Return the name of value profile node array variables: 95inline StringRef getInstrProfVNodesVarName() { return "__llvm_prf_vnodes"; } 96 97/// Return the name prefix of the COMDAT group for instrumentation variables 98/// associated with a COMDAT function. 99inline StringRef getInstrProfComdatPrefix() { return "__profv_"; } 100 101/// Return the name of the variable holding the strings (possibly compressed) 102/// of all function's PGO names. 103inline StringRef getInstrProfNamesVarName() { 104 return "__llvm_prf_nm"; 105} 106 107/// Return the name of a covarage mapping variable (internal linkage) 108/// for each instrumented source module. Such variables are allocated 109/// in the __llvm_covmap section. 110inline StringRef getCoverageMappingVarName() { 111 return "__llvm_coverage_mapping"; 112} 113 114/// Return the name of the internal variable recording the array 115/// of PGO name vars referenced by the coverage mapping. The owning 116/// functions of those names are not emitted by FE (e.g, unused inline 117/// functions.) 118inline StringRef getCoverageUnusedNamesVarName() { 119 return "__llvm_coverage_names"; 120} 121 122/// Return the name of function that registers all the per-function control 123/// data at program startup time by calling __llvm_register_function. This 124/// function has internal linkage and is called by __llvm_profile_init 125/// runtime method. This function is not generated for these platforms: 126/// Darwin, Linux, and FreeBSD. 127inline StringRef getInstrProfRegFuncsName() { 128 return "__llvm_profile_register_functions"; 129} 130 131/// Return the name of the runtime interface that registers per-function control 132/// data for one instrumented function. 133inline StringRef getInstrProfRegFuncName() { 134 return "__llvm_profile_register_function"; 135} 136 137/// Return the name of the runtime interface that registers the PGO name strings. 138inline StringRef getInstrProfNamesRegFuncName() { 139 return "__llvm_profile_register_names_function"; 140} 141 142/// Return the name of the runtime initialization method that is generated by 143/// the compiler. The function calls __llvm_profile_register_functions and 144/// __llvm_profile_override_default_filename functions if needed. This function 145/// has internal linkage and invoked at startup time via init_array. 146inline StringRef getInstrProfInitFuncName() { return "__llvm_profile_init"; } 147 148/// Return the name of the hook variable defined in profile runtime library. 149/// A reference to the variable causes the linker to link in the runtime 150/// initialization module (which defines the hook variable). 151inline StringRef getInstrProfRuntimeHookVarName() { 152 return INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_RUNTIME_VAR); 153} 154 155/// Return the name of the compiler generated function that references the 156/// runtime hook variable. The function is a weak global. 157inline StringRef getInstrProfRuntimeHookVarUseFuncName() { 158 return "__llvm_profile_runtime_user"; 159} 160 161/// Return the marker used to separate PGO names during serialization. 162inline StringRef getInstrProfNameSeparator() { return "\01"; } 163 164/// Return the modified name for function \c F suitable to be 165/// used the key for profile lookup. Variable \c InLTO indicates if this 166/// is called in LTO optimization passes. 167std::string getPGOFuncName(const Function &F, bool InLTO = false, 168 uint64_t Version = INSTR_PROF_INDEX_VERSION); 169 170/// Return the modified name for a function suitable to be 171/// used the key for profile lookup. The function's original 172/// name is \c RawFuncName and has linkage of type \c Linkage. 173/// The function is defined in module \c FileName. 174std::string getPGOFuncName(StringRef RawFuncName, 175 GlobalValue::LinkageTypes Linkage, 176 StringRef FileName, 177 uint64_t Version = INSTR_PROF_INDEX_VERSION); 178 179/// Return the name of the global variable used to store a function 180/// name in PGO instrumentation. \c FuncName is the name of the function 181/// returned by the \c getPGOFuncName call. 182std::string getPGOFuncNameVarName(StringRef FuncName, 183 GlobalValue::LinkageTypes Linkage); 184 185/// Create and return the global variable for function name used in PGO 186/// instrumentation. \c FuncName is the name of the function returned 187/// by \c getPGOFuncName call. 188GlobalVariable *createPGOFuncNameVar(Function &F, StringRef PGOFuncName); 189 190/// Create and return the global variable for function name used in PGO 191/// instrumentation. /// \c FuncName is the name of the function 192/// returned by \c getPGOFuncName call, \c M is the owning module, 193/// and \c Linkage is the linkage of the instrumented function. 194GlobalVariable *createPGOFuncNameVar(Module &M, 195 GlobalValue::LinkageTypes Linkage, 196 StringRef PGOFuncName); 197 198/// Return the initializer in string of the PGO name var \c NameVar. 199StringRef getPGOFuncNameVarInitializer(GlobalVariable *NameVar); 200 201/// Given a PGO function name, remove the filename prefix and return 202/// the original (static) function name. 203StringRef getFuncNameWithoutPrefix(StringRef PGOFuncName, 204 StringRef FileName = "<unknown>"); 205 206/// Given a vector of strings (function PGO names) \c NameStrs, the 207/// method generates a combined string \c Result thatis ready to be 208/// serialized. The \c Result string is comprised of three fields: 209/// The first field is the legnth of the uncompressed strings, and the 210/// the second field is the length of the zlib-compressed string. 211/// Both fields are encoded in ULEB128. If \c doCompress is false, the 212/// third field is the uncompressed strings; otherwise it is the 213/// compressed string. When the string compression is off, the 214/// second field will have value zero. 215Error collectPGOFuncNameStrings(ArrayRef<std::string> NameStrs, 216 bool doCompression, std::string &Result); 217 218/// Produce \c Result string with the same format described above. The input 219/// is vector of PGO function name variables that are referenced. 220Error collectPGOFuncNameStrings(ArrayRef<GlobalVariable *> NameVars, 221 std::string &Result, bool doCompression = true); 222 223/// \c NameStrings is a string composed of one of more sub-strings encoded in 224/// the format described above. The substrings are separated by 0 or more zero 225/// bytes. This method decodes the string and populates the \c Symtab. 226Error readPGOFuncNameStrings(StringRef NameStrings, InstrProfSymtab &Symtab); 227 228/// Check if INSTR_PROF_RAW_VERSION_VAR is defined. This global is only being 229/// set in IR PGO compilation. 230bool isIRPGOFlagSet(const Module *M); 231 232/// Check if we can safely rename this Comdat function. Instances of the same 233/// comdat function may have different control flows thus can not share the 234/// same counter variable. 235bool canRenameComdatFunc(const Function &F, bool CheckAddressTaken = false); 236 237enum InstrProfValueKind : uint32_t { 238#define VALUE_PROF_KIND(Enumerator, Value) Enumerator = Value, 239#include "llvm/ProfileData/InstrProfData.inc" 240}; 241 242/// Get the value profile data for value site \p SiteIdx from \p InstrProfR 243/// and annotate the instruction \p Inst with the value profile meta data. 244/// Annotate up to \p MaxMDCount (default 3) number of records per value site. 245void annotateValueSite(Module &M, Instruction &Inst, 246 const InstrProfRecord &InstrProfR, 247 InstrProfValueKind ValueKind, uint32_t SiteIndx, 248 uint32_t MaxMDCount = 3); 249 250/// Same as the above interface but using an ArrayRef, as well as \p Sum. 251void annotateValueSite(Module &M, Instruction &Inst, 252 ArrayRef<InstrProfValueData> VDs, 253 uint64_t Sum, InstrProfValueKind ValueKind, 254 uint32_t MaxMDCount); 255 256/// Extract the value profile data from \p Inst which is annotated with 257/// value profile meta data. Return false if there is no value data annotated, 258/// otherwise return true. 259bool getValueProfDataFromInst(const Instruction &Inst, 260 InstrProfValueKind ValueKind, 261 uint32_t MaxNumValueData, 262 InstrProfValueData ValueData[], 263 uint32_t &ActualNumValueData, uint64_t &TotalC); 264 265inline StringRef getPGOFuncNameMetadataName() { return "PGOFuncName"; } 266 267/// Return the PGOFuncName meta data associated with a function. 268MDNode *getPGOFuncNameMetadata(const Function &F); 269 270/// Create the PGOFuncName meta data if PGOFuncName is different from 271/// function's raw name. This should only apply to internal linkage functions 272/// declared by users only. 273void createPGOFuncNameMetadata(Function &F, StringRef PGOFuncName); 274 275/// Check if we can use Comdat for profile variables. This will eliminate 276/// the duplicated profile variables for Comdat functions. 277bool needsComdatForCounter(const Function &F, const Module &M); 278 279const std::error_category &instrprof_category(); 280 281enum class instrprof_error { 282 success = 0, 283 eof, 284 unrecognized_format, 285 bad_magic, 286 bad_header, 287 unsupported_version, 288 unsupported_hash_type, 289 too_large, 290 truncated, 291 malformed, 292 unknown_function, 293 hash_mismatch, 294 count_mismatch, 295 counter_overflow, 296 value_site_count_mismatch, 297 compress_failed, 298 uncompress_failed, 299 empty_raw_profile 300}; 301 302inline std::error_code make_error_code(instrprof_error E) { 303 return std::error_code(static_cast<int>(E), instrprof_category()); 304} 305 306class InstrProfError : public ErrorInfo<InstrProfError> { 307public: 308 InstrProfError(instrprof_error Err) : Err(Err) { 309 assert(Err != instrprof_error::success && "Not an error"); 310 } 311 312 std::string message() const override; 313 314 void log(raw_ostream &OS) const override { OS << message(); } 315 316 std::error_code convertToErrorCode() const override { 317 return make_error_code(Err); 318 } 319 320 instrprof_error get() const { return Err; } 321 322 /// Consume an Error and return the raw enum value contained within it. The 323 /// Error must either be a success value, or contain a single InstrProfError. 324 static instrprof_error take(Error E) { 325 auto Err = instrprof_error::success; 326 handleAllErrors(std::move(E), [&Err](const InstrProfError &IPE) { 327 assert(Err == instrprof_error::success && "Multiple errors encountered"); 328 Err = IPE.get(); 329 }); 330 return Err; 331 } 332 333 static char ID; 334 335private: 336 instrprof_error Err; 337}; 338 339class SoftInstrProfErrors { 340 /// Count the number of soft instrprof_errors encountered and keep track of 341 /// the first such error for reporting purposes. 342 343 /// The first soft error encountered. 344 instrprof_error FirstError = instrprof_error::success; 345 346 /// The number of hash mismatches. 347 unsigned NumHashMismatches = 0; 348 349 /// The number of count mismatches. 350 unsigned NumCountMismatches = 0; 351 352 /// The number of counter overflows. 353 unsigned NumCounterOverflows = 0; 354 355 /// The number of value site count mismatches. 356 unsigned NumValueSiteCountMismatches = 0; 357 358public: 359 SoftInstrProfErrors() = default; 360 361 ~SoftInstrProfErrors() { 362 assert(FirstError == instrprof_error::success && 363 "Unchecked soft error encountered"); 364 } 365 366 /// Track a soft error (\p IE) and increment its associated counter. 367 void addError(instrprof_error IE); 368 369 /// Get the number of hash mismatches. 370 unsigned getNumHashMismatches() const { return NumHashMismatches; } 371 372 /// Get the number of count mismatches. 373 unsigned getNumCountMismatches() const { return NumCountMismatches; } 374 375 /// Get the number of counter overflows. 376 unsigned getNumCounterOverflows() const { return NumCounterOverflows; } 377 378 /// Get the number of value site count mismatches. 379 unsigned getNumValueSiteCountMismatches() const { 380 return NumValueSiteCountMismatches; 381 } 382 383 /// Return the first encountered error and reset FirstError to a success 384 /// value. 385 Error takeError() { 386 if (FirstError == instrprof_error::success) 387 return Error::success(); 388 auto E = make_error<InstrProfError>(FirstError); 389 FirstError = instrprof_error::success; 390 return E; 391 } 392}; 393 394namespace object { 395 396class SectionRef; 397 398} // end namespace object 399 400namespace IndexedInstrProf { 401 402uint64_t ComputeHash(StringRef K); 403 404} // end namespace IndexedInstrProf 405 406/// A symbol table used for function PGO name look-up with keys 407/// (such as pointers, md5hash values) to the function. A function's 408/// PGO name or name's md5hash are used in retrieving the profile 409/// data of the function. See \c getPGOFuncName() method for details 410/// on how PGO name is formed. 411class InstrProfSymtab { 412public: 413 typedef std::vector<std::pair<uint64_t, uint64_t>> AddrHashMap; 414 415private: 416 StringRef Data; 417 uint64_t Address = 0; 418 // Unique name strings. 419 StringSet<> NameTab; 420 // A map from MD5 keys to function name strings. 421 std::vector<std::pair<uint64_t, StringRef>> MD5NameMap; 422 // A map from MD5 keys to function define. We only populate this map 423 // when build the Symtab from a Module. 424 std::vector<std::pair<uint64_t, Function *>> MD5FuncMap; 425 // A map from function runtime address to function name MD5 hash. 426 // This map is only populated and used by raw instr profile reader. 427 AddrHashMap AddrToMD5Map; 428 429public: 430 InstrProfSymtab() = default; 431 432 /// Create InstrProfSymtab from an object file section which 433 /// contains function PGO names. When section may contain raw 434 /// string data or string data in compressed form. This method 435 /// only initialize the symtab with reference to the data and 436 /// the section base address. The decompression will be delayed 437 /// until before it is used. See also \c create(StringRef) method. 438 Error create(object::SectionRef &Section); 439 440 /// This interface is used by reader of CoverageMapping test 441 /// format. 442 inline Error create(StringRef D, uint64_t BaseAddr); 443 444 /// \c NameStrings is a string composed of one of more sub-strings 445 /// encoded in the format described in \c collectPGOFuncNameStrings. 446 /// This method is a wrapper to \c readPGOFuncNameStrings method. 447 inline Error create(StringRef NameStrings); 448 449 /// A wrapper interface to populate the PGO symtab with functions 450 /// decls from module \c M. This interface is used by transformation 451 /// passes such as indirect function call promotion. Variable \c InLTO 452 /// indicates if this is called from LTO optimization passes. 453 void create(Module &M, bool InLTO = false); 454 455 /// Create InstrProfSymtab from a set of names iteratable from 456 /// \p IterRange. This interface is used by IndexedProfReader. 457 template <typename NameIterRange> void create(const NameIterRange &IterRange); 458 459 // If the symtab is created by a series of calls to \c addFuncName, \c 460 // finalizeSymtab needs to be called before looking up function names. 461 // This is required because the underlying map is a vector (for space 462 // efficiency) which needs to be sorted. 463 inline void finalizeSymtab(); 464 465 /// Update the symtab by adding \p FuncName to the table. This interface 466 /// is used by the raw and text profile readers. 467 void addFuncName(StringRef FuncName) { 468 auto Ins = NameTab.insert(FuncName); 469 if (Ins.second) 470 MD5NameMap.push_back(std::make_pair( 471 IndexedInstrProf::ComputeHash(FuncName), Ins.first->getKey())); 472 } 473 474 /// Map a function address to its name's MD5 hash. This interface 475 /// is only used by the raw profiler reader. 476 void mapAddress(uint64_t Addr, uint64_t MD5Val) { 477 AddrToMD5Map.push_back(std::make_pair(Addr, MD5Val)); 478 } 479 480 AddrHashMap &getAddrHashMap() { return AddrToMD5Map; } 481 482 /// Return function's PGO name from the function name's symbol 483 /// address in the object file. If an error occurs, return 484 /// an empty string. 485 StringRef getFuncName(uint64_t FuncNameAddress, size_t NameSize); 486 487 /// Return function's PGO name from the name's md5 hash value. 488 /// If not found, return an empty string. 489 inline StringRef getFuncName(uint64_t FuncMD5Hash); 490 491 /// Return function from the name's md5 hash. Return nullptr if not found. 492 inline Function *getFunction(uint64_t FuncMD5Hash); 493 494 /// Return the function's original assembly name by stripping off 495 /// the prefix attached (to symbols with priviate linkage). For 496 /// global functions, it returns the same string as getFuncName. 497 inline StringRef getOrigFuncName(uint64_t FuncMD5Hash); 498 499 /// Return the name section data. 500 inline StringRef getNameData() const { return Data; } 501}; 502 503Error InstrProfSymtab::create(StringRef D, uint64_t BaseAddr) { 504 Data = D; 505 Address = BaseAddr; 506 return Error::success(); 507} 508 509Error InstrProfSymtab::create(StringRef NameStrings) { 510 return readPGOFuncNameStrings(NameStrings, *this); 511} 512 513template <typename NameIterRange> 514void InstrProfSymtab::create(const NameIterRange &IterRange) { 515 for (auto Name : IterRange) 516 addFuncName(Name); 517 518 finalizeSymtab(); 519} 520 521void InstrProfSymtab::finalizeSymtab() { 522 std::sort(MD5NameMap.begin(), MD5NameMap.end(), less_first()); 523 std::sort(MD5FuncMap.begin(), MD5FuncMap.end(), less_first()); 524 std::sort(AddrToMD5Map.begin(), AddrToMD5Map.end(), less_first()); 525 AddrToMD5Map.erase(std::unique(AddrToMD5Map.begin(), AddrToMD5Map.end()), 526 AddrToMD5Map.end()); 527} 528 529StringRef InstrProfSymtab::getFuncName(uint64_t FuncMD5Hash) { 530 auto Result = 531 std::lower_bound(MD5NameMap.begin(), MD5NameMap.end(), FuncMD5Hash, 532 [](const std::pair<uint64_t, std::string> &LHS, 533 uint64_t RHS) { return LHS.first < RHS; }); 534 if (Result != MD5NameMap.end() && Result->first == FuncMD5Hash) 535 return Result->second; 536 return StringRef(); 537} 538 539Function* InstrProfSymtab::getFunction(uint64_t FuncMD5Hash) { 540 auto Result = 541 std::lower_bound(MD5FuncMap.begin(), MD5FuncMap.end(), FuncMD5Hash, 542 [](const std::pair<uint64_t, Function*> &LHS, 543 uint64_t RHS) { return LHS.first < RHS; }); 544 if (Result != MD5FuncMap.end() && Result->first == FuncMD5Hash) 545 return Result->second; 546 return nullptr; 547} 548 549// See also getPGOFuncName implementation. These two need to be 550// matched. 551StringRef InstrProfSymtab::getOrigFuncName(uint64_t FuncMD5Hash) { 552 StringRef PGOName = getFuncName(FuncMD5Hash); 553 size_t S = PGOName.find_first_of(':'); 554 if (S == StringRef::npos) 555 return PGOName; 556 return PGOName.drop_front(S + 1); 557} 558 559struct InstrProfValueSiteRecord { 560 /// Value profiling data pairs at a given value site. 561 std::list<InstrProfValueData> ValueData; 562 563 InstrProfValueSiteRecord() { ValueData.clear(); } 564 template <class InputIterator> 565 InstrProfValueSiteRecord(InputIterator F, InputIterator L) 566 : ValueData(F, L) {} 567 568 /// Sort ValueData ascending by Value 569 void sortByTargetValues() { 570 ValueData.sort( 571 [](const InstrProfValueData &left, const InstrProfValueData &right) { 572 return left.Value < right.Value; 573 }); 574 } 575 /// Sort ValueData Descending by Count 576 inline void sortByCount(); 577 578 /// Merge data from another InstrProfValueSiteRecord 579 /// Optionally scale merged counts by \p Weight. 580 void merge(SoftInstrProfErrors &SIPE, InstrProfValueSiteRecord &Input, 581 uint64_t Weight = 1); 582 /// Scale up value profile data counts. 583 void scale(SoftInstrProfErrors &SIPE, uint64_t Weight); 584}; 585 586/// Profiling information for a single function. 587struct InstrProfRecord { 588 StringRef Name; 589 uint64_t Hash; 590 std::vector<uint64_t> Counts; 591 SoftInstrProfErrors SIPE; 592 593 InstrProfRecord() = default; 594 InstrProfRecord(StringRef Name, uint64_t Hash, std::vector<uint64_t> Counts) 595 : Name(Name), Hash(Hash), Counts(std::move(Counts)) {} 596 597 typedef std::vector<std::pair<uint64_t, uint64_t>> ValueMapType; 598 599 /// Return the number of value profile kinds with non-zero number 600 /// of profile sites. 601 inline uint32_t getNumValueKinds() const; 602 603 /// Return the number of instrumented sites for ValueKind. 604 inline uint32_t getNumValueSites(uint32_t ValueKind) const; 605 606 /// Return the total number of ValueData for ValueKind. 607 inline uint32_t getNumValueData(uint32_t ValueKind) const; 608 609 /// Return the number of value data collected for ValueKind at profiling 610 /// site: Site. 611 inline uint32_t getNumValueDataForSite(uint32_t ValueKind, 612 uint32_t Site) const; 613 614 /// Return the array of profiled values at \p Site. If \p TotalC 615 /// is not null, the total count of all target values at this site 616 /// will be stored in \c *TotalC. 617 inline std::unique_ptr<InstrProfValueData[]> 618 getValueForSite(uint32_t ValueKind, uint32_t Site, 619 uint64_t *TotalC = nullptr) const; 620 621 /// Get the target value/counts of kind \p ValueKind collected at site 622 /// \p Site and store the result in array \p Dest. Return the total 623 /// counts of all target values at this site. 624 inline uint64_t getValueForSite(InstrProfValueData Dest[], uint32_t ValueKind, 625 uint32_t Site) const; 626 627 /// Reserve space for NumValueSites sites. 628 inline void reserveSites(uint32_t ValueKind, uint32_t NumValueSites); 629 630 /// Add ValueData for ValueKind at value Site. 631 void addValueData(uint32_t ValueKind, uint32_t Site, 632 InstrProfValueData *VData, uint32_t N, 633 ValueMapType *ValueMap); 634 635 /// Merge the counts in \p Other into this one. 636 /// Optionally scale merged counts by \p Weight. 637 void merge(InstrProfRecord &Other, uint64_t Weight = 1); 638 639 /// Scale up profile counts (including value profile data) by 640 /// \p Weight. 641 void scale(uint64_t Weight); 642 643 /// Sort value profile data (per site) by count. 644 void sortValueData() { 645 for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind) { 646 std::vector<InstrProfValueSiteRecord> &SiteRecords = 647 getValueSitesForKind(Kind); 648 for (auto &SR : SiteRecords) 649 SR.sortByCount(); 650 } 651 } 652 653 /// Clear value data entries and edge counters. 654 void Clear() { 655 Counts.clear(); 656 clearValueData(); 657 } 658 659 /// Clear value data entries 660 void clearValueData() { 661 for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind) 662 getValueSitesForKind(Kind).clear(); 663 } 664 665 /// Get the error contained within the record's soft error counter. 666 Error takeError() { return SIPE.takeError(); } 667 668private: 669 std::vector<InstrProfValueSiteRecord> IndirectCallSites; 670 std::vector<InstrProfValueSiteRecord> MemOPSizes; 671 const std::vector<InstrProfValueSiteRecord> & 672 673 getValueSitesForKind(uint32_t ValueKind) const { 674 switch (ValueKind) { 675 case IPVK_IndirectCallTarget: 676 return IndirectCallSites; 677 case IPVK_MemOPSize: 678 return MemOPSizes; 679 default: 680 llvm_unreachable("Unknown value kind!"); 681 } 682 return IndirectCallSites; 683 } 684 685 std::vector<InstrProfValueSiteRecord> & 686 getValueSitesForKind(uint32_t ValueKind) { 687 return const_cast<std::vector<InstrProfValueSiteRecord> &>( 688 const_cast<const InstrProfRecord *>(this) 689 ->getValueSitesForKind(ValueKind)); 690 } 691 692 // Map indirect call target name hash to name string. 693 uint64_t remapValue(uint64_t Value, uint32_t ValueKind, 694 ValueMapType *HashKeys); 695 696 // Merge Value Profile data from Src record to this record for ValueKind. 697 // Scale merged value counts by \p Weight. 698 void mergeValueProfData(uint32_t ValueKind, InstrProfRecord &Src, 699 uint64_t Weight); 700 701 // Scale up value profile data count. 702 void scaleValueProfData(uint32_t ValueKind, uint64_t Weight); 703}; 704 705uint32_t InstrProfRecord::getNumValueKinds() const { 706 uint32_t NumValueKinds = 0; 707 for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind) 708 NumValueKinds += !(getValueSitesForKind(Kind).empty()); 709 return NumValueKinds; 710} 711 712uint32_t InstrProfRecord::getNumValueData(uint32_t ValueKind) const { 713 uint32_t N = 0; 714 const std::vector<InstrProfValueSiteRecord> &SiteRecords = 715 getValueSitesForKind(ValueKind); 716 for (auto &SR : SiteRecords) { 717 N += SR.ValueData.size(); 718 } 719 return N; 720} 721 722uint32_t InstrProfRecord::getNumValueSites(uint32_t ValueKind) const { 723 return getValueSitesForKind(ValueKind).size(); 724} 725 726uint32_t InstrProfRecord::getNumValueDataForSite(uint32_t ValueKind, 727 uint32_t Site) const { 728 return getValueSitesForKind(ValueKind)[Site].ValueData.size(); 729} 730 731std::unique_ptr<InstrProfValueData[]> 732InstrProfRecord::getValueForSite(uint32_t ValueKind, uint32_t Site, 733 uint64_t *TotalC) const { 734 uint64_t Dummy; 735 uint64_t &TotalCount = (TotalC == nullptr ? Dummy : *TotalC); 736 uint32_t N = getNumValueDataForSite(ValueKind, Site); 737 if (N == 0) { 738 TotalCount = 0; 739 return std::unique_ptr<InstrProfValueData[]>(nullptr); 740 } 741 742 auto VD = llvm::make_unique<InstrProfValueData[]>(N); 743 TotalCount = getValueForSite(VD.get(), ValueKind, Site); 744 745 return VD; 746} 747 748uint64_t InstrProfRecord::getValueForSite(InstrProfValueData Dest[], 749 uint32_t ValueKind, 750 uint32_t Site) const { 751 uint32_t I = 0; 752 uint64_t TotalCount = 0; 753 for (auto V : getValueSitesForKind(ValueKind)[Site].ValueData) { 754 Dest[I].Value = V.Value; 755 Dest[I].Count = V.Count; 756 TotalCount = SaturatingAdd(TotalCount, V.Count); 757 I++; 758 } 759 return TotalCount; 760} 761 762void InstrProfRecord::reserveSites(uint32_t ValueKind, uint32_t NumValueSites) { 763 std::vector<InstrProfValueSiteRecord> &ValueSites = 764 getValueSitesForKind(ValueKind); 765 ValueSites.reserve(NumValueSites); 766} 767 768inline support::endianness getHostEndianness() { 769 return sys::IsLittleEndianHost ? support::little : support::big; 770} 771 772// Include definitions for value profile data 773#define INSTR_PROF_VALUE_PROF_DATA 774#include "llvm/ProfileData/InstrProfData.inc" 775 776void InstrProfValueSiteRecord::sortByCount() { 777 ValueData.sort( 778 [](const InstrProfValueData &left, const InstrProfValueData &right) { 779 return left.Count > right.Count; 780 }); 781 // Now truncate 782 size_t max_s = INSTR_PROF_MAX_NUM_VAL_PER_SITE; 783 if (ValueData.size() > max_s) 784 ValueData.resize(max_s); 785} 786 787namespace IndexedInstrProf { 788 789enum class HashT : uint32_t { 790 MD5, 791 Last = MD5 792}; 793 794inline uint64_t ComputeHash(HashT Type, StringRef K) { 795 switch (Type) { 796 case HashT::MD5: 797 return MD5Hash(K); 798 } 799 llvm_unreachable("Unhandled hash type"); 800} 801 802const uint64_t Magic = 0x8169666f72706cff; // "\xfflprofi\x81" 803 804enum ProfVersion { 805 // Version 1 is the first version. In this version, the value of 806 // a key/value pair can only include profile data of a single function. 807 // Due to this restriction, the number of block counters for a given 808 // function is not recorded but derived from the length of the value. 809 Version1 = 1, 810 // The version 2 format supports recording profile data of multiple 811 // functions which share the same key in one value field. To support this, 812 // the number block counters is recorded as an uint64_t field right after the 813 // function structural hash. 814 Version2 = 2, 815 // Version 3 supports value profile data. The value profile data is expected 816 // to follow the block counter profile data. 817 Version3 = 3, 818 // In this version, profile summary data \c IndexedInstrProf::Summary is 819 // stored after the profile header. 820 Version4 = 4, 821 // The current version is 4. 822 CurrentVersion = INSTR_PROF_INDEX_VERSION 823}; 824const uint64_t Version = ProfVersion::CurrentVersion; 825 826const HashT HashType = HashT::MD5; 827 828inline uint64_t ComputeHash(StringRef K) { return ComputeHash(HashType, K); } 829 830// This structure defines the file header of the LLVM profile 831// data file in indexed-format. 832struct Header { 833 uint64_t Magic; 834 uint64_t Version; 835 uint64_t Unused; // Becomes unused since version 4 836 uint64_t HashType; 837 uint64_t HashOffset; 838}; 839 840// Profile summary data recorded in the profile data file in indexed 841// format. It is introduced in version 4. The summary data follows 842// right after the profile file header. 843struct Summary { 844 struct Entry { 845 uint64_t Cutoff; ///< The required percentile of total execution count. 846 uint64_t 847 MinBlockCount; ///< The minimum execution count for this percentile. 848 uint64_t NumBlocks; ///< Number of blocks >= the minumum execution count. 849 }; 850 // The field kind enumerator to assigned value mapping should remain 851 // unchanged when a new kind is added or an old kind gets deleted in 852 // the future. 853 enum SummaryFieldKind { 854 /// The total number of functions instrumented. 855 TotalNumFunctions = 0, 856 /// Total number of instrumented blocks/edges. 857 TotalNumBlocks = 1, 858 /// The maximal execution count among all functions. 859 /// This field does not exist for profile data from IR based 860 /// instrumentation. 861 MaxFunctionCount = 2, 862 /// Max block count of the program. 863 MaxBlockCount = 3, 864 /// Max internal block count of the program (excluding entry blocks). 865 MaxInternalBlockCount = 4, 866 /// The sum of all instrumented block counts. 867 TotalBlockCount = 5, 868 NumKinds = TotalBlockCount + 1 869 }; 870 871 // The number of summmary fields following the summary header. 872 uint64_t NumSummaryFields; 873 // The number of Cutoff Entries (Summary::Entry) following summary fields. 874 uint64_t NumCutoffEntries; 875 876 static uint32_t getSize(uint32_t NumSumFields, uint32_t NumCutoffEntries) { 877 return sizeof(Summary) + NumCutoffEntries * sizeof(Entry) + 878 NumSumFields * sizeof(uint64_t); 879 } 880 881 const uint64_t *getSummaryDataBase() const { 882 return reinterpret_cast<const uint64_t *>(this + 1); 883 } 884 885 uint64_t *getSummaryDataBase() { 886 return reinterpret_cast<uint64_t *>(this + 1); 887 } 888 889 const Entry *getCutoffEntryBase() const { 890 return reinterpret_cast<const Entry *>( 891 &getSummaryDataBase()[NumSummaryFields]); 892 } 893 894 Entry *getCutoffEntryBase() { 895 return reinterpret_cast<Entry *>(&getSummaryDataBase()[NumSummaryFields]); 896 } 897 898 uint64_t get(SummaryFieldKind K) const { 899 return getSummaryDataBase()[K]; 900 } 901 902 void set(SummaryFieldKind K, uint64_t V) { 903 getSummaryDataBase()[K] = V; 904 } 905 906 const Entry &getEntry(uint32_t I) const { return getCutoffEntryBase()[I]; } 907 908 void setEntry(uint32_t I, const ProfileSummaryEntry &E) { 909 Entry &ER = getCutoffEntryBase()[I]; 910 ER.Cutoff = E.Cutoff; 911 ER.MinBlockCount = E.MinCount; 912 ER.NumBlocks = E.NumCounts; 913 } 914 915 Summary(uint32_t Size) { memset(this, 0, Size); } 916 void operator delete(void *ptr) { ::operator delete(ptr); } 917 918 Summary() = delete; 919}; 920 921inline std::unique_ptr<Summary> allocSummary(uint32_t TotalSize) { 922 return std::unique_ptr<Summary>(new (::operator new(TotalSize)) 923 Summary(TotalSize)); 924} 925 926} // end namespace IndexedInstrProf 927 928namespace RawInstrProf { 929 930// Version 1: First version 931// Version 2: Added value profile data section. Per-function control data 932// struct has more fields to describe value profile information. 933// Version 3: Compressed name section support. Function PGO name reference 934// from control data struct is changed from raw pointer to Name's MD5 value. 935// Version 4: ValueDataBegin and ValueDataSizes fields are removed from the 936// raw header. 937const uint64_t Version = INSTR_PROF_RAW_VERSION; 938 939template <class IntPtrT> inline uint64_t getMagic(); 940template <> inline uint64_t getMagic<uint64_t>() { 941 return INSTR_PROF_RAW_MAGIC_64; 942} 943 944template <> inline uint64_t getMagic<uint32_t>() { 945 return INSTR_PROF_RAW_MAGIC_32; 946} 947 948// Per-function profile data header/control structure. 949// The definition should match the structure defined in 950// compiler-rt/lib/profile/InstrProfiling.h. 951// It should also match the synthesized type in 952// Transforms/Instrumentation/InstrProfiling.cpp:getOrCreateRegionCounters. 953template <class IntPtrT> struct LLVM_ALIGNAS(8) ProfileData { 954 #define INSTR_PROF_DATA(Type, LLVMType, Name, Init) Type Name; 955 #include "llvm/ProfileData/InstrProfData.inc" 956}; 957 958// File header structure of the LLVM profile data in raw format. 959// The definition should match the header referenced in 960// compiler-rt/lib/profile/InstrProfilingFile.c and 961// InstrProfilingBuffer.c. 962struct Header { 963#define INSTR_PROF_RAW_HEADER(Type, Name, Init) const Type Name; 964#include "llvm/ProfileData/InstrProfData.inc" 965}; 966 967} // end namespace RawInstrProf 968 969// Parse MemOP Size range option. 970void getMemOPSizeRangeFromOption(StringRef Str, int64_t &RangeStart, 971 int64_t &RangeLast); 972 973} // end namespace llvm 974 975#endif // LLVM_PROFILEDATA_INSTRPROF_H 976