GlobalModuleIndex.cpp revision 8e31d064e7543dde6e4205d5b27a95c96a4c3a4e
1//===--- GlobalModuleIndex.cpp - Global Module Index ------------*- C++ -*-===// 2// 3// The LLVM Compiler Infrastructure 4// 5// This file is distributed under the University of Illinois Open Source 6// License. See LICENSE.TXT for details. 7// 8//===----------------------------------------------------------------------===// 9// 10// This file implements the GlobalModuleIndex class. 11// 12//===----------------------------------------------------------------------===// 13 14#include "ASTReaderInternals.h" 15#include "clang/Basic/FileManager.h" 16#include "clang/Basic/OnDiskHashTable.h" 17#include "clang/Serialization/ASTBitCodes.h" 18#include "clang/Serialization/GlobalModuleIndex.h" 19#include "llvm/ADT/DenseMap.h" 20#include "llvm/ADT/MapVector.h" 21#include "llvm/ADT/SmallString.h" 22#include "llvm/ADT/StringExtras.h" 23#include "llvm/Bitcode/BitstreamReader.h" 24#include "llvm/Bitcode/BitstreamWriter.h" 25#include "llvm/Support/FileSystem.h" 26#include "llvm/Support/LockFileManager.h" 27#include "llvm/Support/MemoryBuffer.h" 28#include "llvm/Support/PathV2.h" 29#include <cstdio> 30using namespace clang; 31using namespace serialization; 32 33//----------------------------------------------------------------------------// 34// Shared constants 35//----------------------------------------------------------------------------// 36namespace { 37 enum { 38 /// \brief The block containing the index. 39 GLOBAL_INDEX_BLOCK_ID = llvm::bitc::FIRST_APPLICATION_BLOCKID 40 }; 41 42 /// \brief Describes the record types in the index. 43 enum IndexRecordTypes { 44 /// \brief Contains version information and potentially other metadata, 45 /// used to determine if we can read this global index file. 46 INDEX_METADATA, 47 /// \brief Describes a module, including its file name and dependencies. 48 MODULE, 49 /// \brief The index for identifiers. 50 IDENTIFIER_INDEX 51 }; 52} 53 54/// \brief The name of the global index file. 55static const char * const IndexFileName = "modules.idx"; 56 57/// \brief The global index file version. 58static const unsigned CurrentVersion = 1; 59 60//----------------------------------------------------------------------------// 61// Global module index reader. 62//----------------------------------------------------------------------------// 63 64namespace { 65 66/// \brief Trait used to read the identifier index from the on-disk hash 67/// table. 68class IdentifierIndexReaderTrait { 69public: 70 typedef StringRef external_key_type; 71 typedef StringRef internal_key_type; 72 typedef SmallVector<unsigned, 2> data_type; 73 74 static bool EqualKey(const internal_key_type& a, const internal_key_type& b) { 75 return a == b; 76 } 77 78 static unsigned ComputeHash(const internal_key_type& a) { 79 return llvm::HashString(a); 80 } 81 82 static std::pair<unsigned, unsigned> 83 ReadKeyDataLength(const unsigned char*& d) { 84 using namespace clang::io; 85 unsigned KeyLen = ReadUnalignedLE16(d); 86 unsigned DataLen = ReadUnalignedLE16(d); 87 return std::make_pair(KeyLen, DataLen); 88 } 89 90 static const internal_key_type& 91 GetInternalKey(const external_key_type& x) { return x; } 92 93 static const external_key_type& 94 GetExternalKey(const internal_key_type& x) { return x; } 95 96 static internal_key_type ReadKey(const unsigned char* d, unsigned n) { 97 return StringRef((const char *)d, n); 98 } 99 100 static data_type ReadData(const internal_key_type& k, 101 const unsigned char* d, 102 unsigned DataLen) { 103 using namespace clang::io; 104 105 data_type Result; 106 while (DataLen > 0) { 107 unsigned ID = ReadUnalignedLE32(d); 108 Result.push_back(ID); 109 DataLen -= 4; 110 } 111 112 return Result; 113 } 114}; 115 116typedef OnDiskChainedHashTable<IdentifierIndexReaderTrait> IdentifierIndexTable; 117 118/// \brief Module information as it was loaded from the index file. 119struct LoadedModuleInfo { 120 const FileEntry *File; 121 SmallVector<unsigned, 2> Dependencies; 122 SmallVector<unsigned, 2> ImportedBy; 123}; 124 125} 126 127GlobalModuleIndex::GlobalModuleIndex(FileManager &FileMgr, 128 llvm::MemoryBuffer *Buffer, 129 llvm::BitstreamCursor Cursor) 130 : Buffer(Buffer), IdentifierIndex(), 131 NumIdentifierLookups(), NumIdentifierLookupHits() 132{ 133 typedef llvm::DenseMap<unsigned, LoadedModuleInfo> LoadedModulesMap; 134 LoadedModulesMap LoadedModules; 135 136 // Read the global index. 137 unsigned LargestID = 0; 138 bool InGlobalIndexBlock = false; 139 bool Done = false; 140 bool AnyOutOfDate = false; 141 while (!Done) { 142 llvm::BitstreamEntry Entry = Cursor.advance(); 143 144 switch (Entry.Kind) { 145 case llvm::BitstreamEntry::Error: 146 return; 147 148 case llvm::BitstreamEntry::EndBlock: 149 if (InGlobalIndexBlock) { 150 InGlobalIndexBlock = false; 151 Done = true; 152 continue; 153 } 154 return; 155 156 157 case llvm::BitstreamEntry::Record: 158 // Entries in the global index block are handled below. 159 if (InGlobalIndexBlock) 160 break; 161 162 return; 163 164 case llvm::BitstreamEntry::SubBlock: 165 if (!InGlobalIndexBlock && Entry.ID == GLOBAL_INDEX_BLOCK_ID) { 166 if (Cursor.EnterSubBlock(GLOBAL_INDEX_BLOCK_ID)) 167 return; 168 169 InGlobalIndexBlock = true; 170 } else if (Cursor.SkipBlock()) { 171 return; 172 } 173 continue; 174 } 175 176 SmallVector<uint64_t, 64> Record; 177 StringRef Blob; 178 switch ((IndexRecordTypes)Cursor.readRecord(Entry.ID, Record, &Blob)) { 179 case INDEX_METADATA: 180 // Make sure that the version matches. 181 if (Record.size() < 1 || Record[0] != CurrentVersion) 182 return; 183 break; 184 185 case MODULE: { 186 unsigned Idx = 0; 187 unsigned ID = Record[Idx++]; 188 if (ID > LargestID) 189 LargestID = ID; 190 191 off_t Size = Record[Idx++]; 192 time_t ModTime = Record[Idx++]; 193 194 // File name. 195 unsigned NameLen = Record[Idx++]; 196 llvm::SmallString<64> FileName(Record.begin() + Idx, 197 Record.begin() + Idx + NameLen); 198 Idx += NameLen; 199 200 // Dependencies 201 unsigned NumDeps = Record[Idx++]; 202 llvm::SmallVector<unsigned, 2> 203 Dependencies(Record.begin() + Idx, Record.begin() + Idx + NumDeps); 204 205 // Find the file. If we can't find it, ignore it. 206 const FileEntry *File = FileMgr.getFile(FileName); 207 if (!File) { 208 AnyOutOfDate = true; 209 break; 210 } 211 212 // If the module file is newer than the index, ignore it. 213 if (File->getSize() != Size || File->getModificationTime() != ModTime) { 214 AnyOutOfDate = true; 215 break; 216 } 217 218 // Record this module. The dependencies will be resolved later. 219 LoadedModuleInfo &Info = LoadedModules[ID]; 220 Info.File = File; 221 Info.Dependencies.swap(Dependencies); 222 break; 223 } 224 225 case IDENTIFIER_INDEX: 226 // Wire up the identifier index. 227 if (Record[0]) { 228 IdentifierIndex = IdentifierIndexTable::Create( 229 (const unsigned char *)Blob.data() + Record[0], 230 (const unsigned char *)Blob.data(), 231 IdentifierIndexReaderTrait()); 232 } 233 break; 234 } 235 } 236 237 // If there are any modules that have gone out-of-date, prune out any modules 238 // that depend on them. 239 if (AnyOutOfDate) { 240 // First, build back links in the module dependency graph. 241 SmallVector<unsigned, 4> Stack; 242 for (LoadedModulesMap::iterator LM = LoadedModules.begin(), 243 LMEnd = LoadedModules.end(); 244 LM != LMEnd; ++LM) { 245 unsigned ID = LM->first; 246 247 // If this module is out-of-date, push it onto the stack. 248 if (LM->second.File == 0) 249 Stack.push_back(ID); 250 251 for (unsigned I = 0, N = LM->second.Dependencies.size(); I != N; ++I) { 252 unsigned DepID = LM->second.Dependencies[I]; 253 LoadedModulesMap::iterator Known = LoadedModules.find(DepID); 254 if (Known == LoadedModules.end() || !Known->second.File) { 255 // The dependency was out-of-date, so mark us as out of date. 256 // This is just an optimization. 257 if (LM->second.File) 258 Stack.push_back(ID); 259 260 LM->second.File = 0; 261 continue; 262 } 263 264 // Record this reverse dependency. 265 Known->second.ImportedBy.push_back(ID); 266 } 267 } 268 269 // Second, walk the back links from out-of-date modules to those modules 270 // that depend on them, making those modules out-of-date as well. 271 while (!Stack.empty()) { 272 unsigned ID = Stack.back(); 273 Stack.pop_back(); 274 275 LoadedModuleInfo &Info = LoadedModules[ID]; 276 for (unsigned I = 0, N = Info.ImportedBy.size(); I != N; ++I) { 277 unsigned FromID = Info.ImportedBy[I]; 278 if (LoadedModules[FromID].File) { 279 LoadedModules[FromID].File = 0; 280 Stack.push_back(FromID); 281 } 282 } 283 } 284 } 285 286 // Allocate the vector containing information about all of the modules. 287 Modules.resize(LargestID + 1); 288 for (LoadedModulesMap::iterator LM = LoadedModules.begin(), 289 LMEnd = LoadedModules.end(); 290 LM != LMEnd; ++LM) { 291 if (!LM->second.File) 292 continue; 293 294 Modules[LM->first].File = LM->second.File; 295 296 // Resolve dependencies. Drop any we can't resolve due to out-of-date 297 // module files. 298 for (unsigned I = 0, N = LM->second.Dependencies.size(); I != N; ++I) { 299 unsigned DepID = LM->second.Dependencies[I]; 300 LoadedModulesMap::iterator Known = LoadedModules.find(DepID); 301 if (Known == LoadedModules.end() || !Known->second.File) 302 continue; 303 304 Modules[LM->first].Dependencies.push_back(Known->second.File); 305 } 306 } 307} 308 309GlobalModuleIndex::~GlobalModuleIndex() { } 310 311std::pair<GlobalModuleIndex *, GlobalModuleIndex::ErrorCode> 312GlobalModuleIndex::readIndex(FileManager &FileMgr, StringRef Path) { 313 // Load the index file, if it's there. 314 llvm::SmallString<128> IndexPath; 315 IndexPath += Path; 316 llvm::sys::path::append(IndexPath, IndexFileName); 317 318 llvm::OwningPtr<llvm::MemoryBuffer> Buffer( 319 FileMgr.getBufferForFile(IndexPath)); 320 if (!Buffer) 321 return std::make_pair((GlobalModuleIndex *)0, EC_NotFound); 322 323 /// \brief The bitstream reader from which we'll read the AST file. 324 llvm::BitstreamReader Reader((const unsigned char *)Buffer->getBufferStart(), 325 (const unsigned char *)Buffer->getBufferEnd()); 326 327 /// \brief The main bitstream cursor for the main block. 328 llvm::BitstreamCursor Cursor(Reader); 329 330 // Sniff for the signature. 331 if (Cursor.Read(8) != 'B' || 332 Cursor.Read(8) != 'C' || 333 Cursor.Read(8) != 'G' || 334 Cursor.Read(8) != 'I') { 335 return std::make_pair((GlobalModuleIndex *)0, EC_IOError); 336 } 337 338 return std::make_pair(new GlobalModuleIndex(FileMgr, Buffer.take(), Cursor), 339 EC_None); 340} 341 342void GlobalModuleIndex::getKnownModules( 343 SmallVectorImpl<const FileEntry *> &ModuleFiles) { 344 ModuleFiles.clear(); 345 for (unsigned I = 0, N = Modules.size(); I != N; ++I) { 346 if (Modules[I].File) 347 ModuleFiles.push_back(Modules[I].File); 348 } 349} 350 351void GlobalModuleIndex::getModuleDependencies( 352 const clang::FileEntry *ModuleFile, 353 SmallVectorImpl<const clang::FileEntry *> &Dependencies) { 354 // If the file -> index mapping is empty, populate it now. 355 if (ModulesByFile.empty()) { 356 for (unsigned I = 0, N = Modules.size(); I != N; ++I) { 357 if (Modules[I].File) 358 ModulesByFile[Modules[I].File] = I; 359 } 360 } 361 362 // Look for information about this module file. 363 llvm::DenseMap<const FileEntry *, unsigned>::iterator Known 364 = ModulesByFile.find(ModuleFile); 365 if (Known == ModulesByFile.end()) 366 return; 367 368 // Record dependencies. 369 Dependencies = Modules[Known->second].Dependencies; 370} 371 372bool GlobalModuleIndex::lookupIdentifier(StringRef Name, HitSet &Hits) { 373 Hits.clear(); 374 375 // If there's no identifier index, there is nothing we can do. 376 if (!IdentifierIndex) 377 return false; 378 379 // Look into the identifier index. 380 ++NumIdentifierLookups; 381 IdentifierIndexTable &Table 382 = *static_cast<IdentifierIndexTable *>(IdentifierIndex); 383 IdentifierIndexTable::iterator Known = Table.find(Name); 384 if (Known == Table.end()) { 385 return true; 386 } 387 388 SmallVector<unsigned, 2> ModuleIDs = *Known; 389 for (unsigned I = 0, N = ModuleIDs.size(); I != N; ++I) { 390 unsigned ID = ModuleIDs[I]; 391 if (ID >= Modules.size() || !Modules[ID].File) 392 continue; 393 394 Hits.insert(Modules[ID].File); 395 } 396 397 ++NumIdentifierLookupHits; 398 return true; 399} 400 401void GlobalModuleIndex::printStats() { 402 std::fprintf(stderr, "*** Global Module Index Statistics:\n"); 403 if (NumIdentifierLookups) { 404 fprintf(stderr, " %u / %u identifier lookups succeeded (%f%%)\n", 405 NumIdentifierLookupHits, NumIdentifierLookups, 406 (double)NumIdentifierLookupHits*100.0/NumIdentifierLookups); 407 } 408 std::fprintf(stderr, "\n"); 409} 410 411//----------------------------------------------------------------------------// 412// Global module index writer. 413//----------------------------------------------------------------------------// 414 415namespace { 416 /// \brief Provides information about a specific module file. 417 struct ModuleFileInfo { 418 /// \brief The numberic ID for this module file. 419 unsigned ID; 420 421 /// \brief The set of modules on which this module depends. Each entry is 422 /// a module ID. 423 SmallVector<unsigned, 4> Dependencies; 424 }; 425 426 /// \brief Builder that generates the global module index file. 427 class GlobalModuleIndexBuilder { 428 FileManager &FileMgr; 429 430 /// \brief Mapping from files to module file information. 431 typedef llvm::MapVector<const FileEntry *, ModuleFileInfo> ModuleFilesMap; 432 433 /// \brief Information about each of the known module files. 434 ModuleFilesMap ModuleFiles; 435 436 /// \brief Mapping from identifiers to the list of module file IDs that 437 /// consider this identifier to be interesting. 438 typedef llvm::StringMap<SmallVector<unsigned, 2> > InterestingIdentifierMap; 439 440 /// \brief A mapping from all interesting identifiers to the set of module 441 /// files in which those identifiers are considered interesting. 442 InterestingIdentifierMap InterestingIdentifiers; 443 444 /// \brief Write the block-info block for the global module index file. 445 void emitBlockInfoBlock(llvm::BitstreamWriter &Stream); 446 447 /// \brief Retrieve the module file information for the given file. 448 ModuleFileInfo &getModuleFileInfo(const FileEntry *File) { 449 llvm::MapVector<const FileEntry *, ModuleFileInfo>::iterator Known 450 = ModuleFiles.find(File); 451 if (Known != ModuleFiles.end()) 452 return Known->second; 453 454 unsigned NewID = ModuleFiles.size(); 455 ModuleFileInfo &Info = ModuleFiles[File]; 456 Info.ID = NewID; 457 return Info; 458 } 459 460 public: 461 explicit GlobalModuleIndexBuilder(FileManager &FileMgr) : FileMgr(FileMgr){} 462 463 /// \brief Load the contents of the given module file into the builder. 464 /// 465 /// \returns true if an error occurred, false otherwise. 466 bool loadModuleFile(const FileEntry *File); 467 468 /// \brief Write the index to the given bitstream. 469 void writeIndex(llvm::BitstreamWriter &Stream); 470 }; 471} 472 473static void emitBlockID(unsigned ID, const char *Name, 474 llvm::BitstreamWriter &Stream, 475 SmallVectorImpl<uint64_t> &Record) { 476 Record.clear(); 477 Record.push_back(ID); 478 Stream.EmitRecord(llvm::bitc::BLOCKINFO_CODE_SETBID, Record); 479 480 // Emit the block name if present. 481 if (Name == 0 || Name[0] == 0) return; 482 Record.clear(); 483 while (*Name) 484 Record.push_back(*Name++); 485 Stream.EmitRecord(llvm::bitc::BLOCKINFO_CODE_BLOCKNAME, Record); 486} 487 488static void emitRecordID(unsigned ID, const char *Name, 489 llvm::BitstreamWriter &Stream, 490 SmallVectorImpl<uint64_t> &Record) { 491 Record.clear(); 492 Record.push_back(ID); 493 while (*Name) 494 Record.push_back(*Name++); 495 Stream.EmitRecord(llvm::bitc::BLOCKINFO_CODE_SETRECORDNAME, Record); 496} 497 498void 499GlobalModuleIndexBuilder::emitBlockInfoBlock(llvm::BitstreamWriter &Stream) { 500 SmallVector<uint64_t, 64> Record; 501 Stream.EnterSubblock(llvm::bitc::BLOCKINFO_BLOCK_ID, 3); 502 503#define BLOCK(X) emitBlockID(X ## _ID, #X, Stream, Record) 504#define RECORD(X) emitRecordID(X, #X, Stream, Record) 505 BLOCK(GLOBAL_INDEX_BLOCK); 506 RECORD(INDEX_METADATA); 507 RECORD(MODULE); 508 RECORD(IDENTIFIER_INDEX); 509#undef RECORD 510#undef BLOCK 511 512 Stream.ExitBlock(); 513} 514 515namespace { 516 class InterestingASTIdentifierLookupTrait 517 : public serialization::reader::ASTIdentifierLookupTraitBase { 518 519 public: 520 /// \brief The identifier and whether it is "interesting". 521 typedef std::pair<StringRef, bool> data_type; 522 523 data_type ReadData(const internal_key_type& k, 524 const unsigned char* d, 525 unsigned DataLen) { 526 // The first bit indicates whether this identifier is interesting. 527 // That's all we care about. 528 using namespace clang::io; 529 unsigned RawID = ReadUnalignedLE32(d); 530 bool IsInteresting = RawID & 0x01; 531 return std::make_pair(k, IsInteresting); 532 } 533 }; 534} 535 536bool GlobalModuleIndexBuilder::loadModuleFile(const FileEntry *File) { 537 // Open the module file. 538 OwningPtr<llvm::MemoryBuffer> Buffer; 539 std::string ErrorStr; 540 Buffer.reset(FileMgr.getBufferForFile(File, &ErrorStr, /*isVolatile=*/true)); 541 if (!Buffer) { 542 return true; 543 } 544 545 // Initialize the input stream 546 llvm::BitstreamReader InStreamFile; 547 llvm::BitstreamCursor InStream; 548 InStreamFile.init((const unsigned char *)Buffer->getBufferStart(), 549 (const unsigned char *)Buffer->getBufferEnd()); 550 InStream.init(InStreamFile); 551 552 // Sniff for the signature. 553 if (InStream.Read(8) != 'C' || 554 InStream.Read(8) != 'P' || 555 InStream.Read(8) != 'C' || 556 InStream.Read(8) != 'H') { 557 return true; 558 } 559 560 // Record this module file and assign it a unique ID (if it doesn't have 561 // one already). 562 unsigned ID = getModuleFileInfo(File).ID; 563 564 // Search for the blocks and records we care about. 565 enum { Other, ControlBlock, ASTBlock } State = Other; 566 bool Done = false; 567 while (!Done) { 568 llvm::BitstreamEntry Entry = InStream.advance(); 569 switch (Entry.Kind) { 570 case llvm::BitstreamEntry::Error: 571 Done = true; 572 continue; 573 574 case llvm::BitstreamEntry::Record: 575 // In the 'other' state, just skip the record. We don't care. 576 if (State == Other) { 577 InStream.skipRecord(Entry.ID); 578 continue; 579 } 580 581 // Handle potentially-interesting records below. 582 break; 583 584 case llvm::BitstreamEntry::SubBlock: 585 if (Entry.ID == CONTROL_BLOCK_ID) { 586 if (InStream.EnterSubBlock(CONTROL_BLOCK_ID)) 587 return true; 588 589 // Found the control block. 590 State = ControlBlock; 591 continue; 592 } 593 594 if (Entry.ID == AST_BLOCK_ID) { 595 if (InStream.EnterSubBlock(AST_BLOCK_ID)) 596 return true; 597 598 // Found the AST block. 599 State = ASTBlock; 600 continue; 601 } 602 603 if (InStream.SkipBlock()) 604 return true; 605 606 continue; 607 608 case llvm::BitstreamEntry::EndBlock: 609 State = Other; 610 continue; 611 } 612 613 // Read the given record. 614 SmallVector<uint64_t, 64> Record; 615 StringRef Blob; 616 unsigned Code = InStream.readRecord(Entry.ID, Record, &Blob); 617 618 // Handle module dependencies. 619 if (State == ControlBlock && Code == IMPORTS) { 620 // Load each of the imported PCH files. 621 unsigned Idx = 0, N = Record.size(); 622 while (Idx < N) { 623 // Read information about the AST file. 624 625 // Skip the imported kind 626 ++Idx; 627 628 // Skip the import location 629 ++Idx; 630 631 // Retrieve the imported file name. 632 unsigned Length = Record[Idx++]; 633 SmallString<128> ImportedFile(Record.begin() + Idx, 634 Record.begin() + Idx + Length); 635 Idx += Length; 636 637 // Find the imported module file. 638 const FileEntry *DependsOnFile = FileMgr.getFile(ImportedFile); 639 if (!DependsOnFile) 640 return true; 641 642 // Record the dependency. 643 unsigned DependsOnID = getModuleFileInfo(DependsOnFile).ID; 644 getModuleFileInfo(File).Dependencies.push_back(DependsOnID); 645 } 646 647 continue; 648 } 649 650 // Handle the identifier table 651 if (State == ASTBlock && Code == IDENTIFIER_TABLE && Record[0] > 0) { 652 typedef OnDiskChainedHashTable<InterestingASTIdentifierLookupTrait> 653 InterestingIdentifierTable; 654 llvm::OwningPtr<InterestingIdentifierTable> 655 Table(InterestingIdentifierTable::Create( 656 (const unsigned char *)Blob.data() + Record[0], 657 (const unsigned char *)Blob.data())); 658 for (InterestingIdentifierTable::data_iterator D = Table->data_begin(), 659 DEnd = Table->data_end(); 660 D != DEnd; ++D) { 661 std::pair<StringRef, bool> Ident = *D; 662 if (Ident.second) 663 InterestingIdentifiers[Ident.first].push_back(ID); 664 else 665 (void)InterestingIdentifiers[Ident.first]; 666 } 667 } 668 669 // We don't care about this record. 670 } 671 672 return false; 673} 674 675namespace { 676 677/// \brief Trait used to generate the identifier index as an on-disk hash 678/// table. 679class IdentifierIndexWriterTrait { 680public: 681 typedef StringRef key_type; 682 typedef StringRef key_type_ref; 683 typedef SmallVector<unsigned, 2> data_type; 684 typedef const SmallVector<unsigned, 2> &data_type_ref; 685 686 static unsigned ComputeHash(key_type_ref Key) { 687 return llvm::HashString(Key); 688 } 689 690 std::pair<unsigned,unsigned> 691 EmitKeyDataLength(raw_ostream& Out, key_type_ref Key, data_type_ref Data) { 692 unsigned KeyLen = Key.size(); 693 unsigned DataLen = Data.size() * 4; 694 clang::io::Emit16(Out, KeyLen); 695 clang::io::Emit16(Out, DataLen); 696 return std::make_pair(KeyLen, DataLen); 697 } 698 699 void EmitKey(raw_ostream& Out, key_type_ref Key, unsigned KeyLen) { 700 Out.write(Key.data(), KeyLen); 701 } 702 703 void EmitData(raw_ostream& Out, key_type_ref Key, data_type_ref Data, 704 unsigned DataLen) { 705 for (unsigned I = 0, N = Data.size(); I != N; ++I) 706 clang::io::Emit32(Out, Data[I]); 707 } 708}; 709 710} 711 712void GlobalModuleIndexBuilder::writeIndex(llvm::BitstreamWriter &Stream) { 713 using namespace llvm; 714 715 // Emit the file header. 716 Stream.Emit((unsigned)'B', 8); 717 Stream.Emit((unsigned)'C', 8); 718 Stream.Emit((unsigned)'G', 8); 719 Stream.Emit((unsigned)'I', 8); 720 721 // Write the block-info block, which describes the records in this bitcode 722 // file. 723 emitBlockInfoBlock(Stream); 724 725 Stream.EnterSubblock(GLOBAL_INDEX_BLOCK_ID, 3); 726 727 // Write the metadata. 728 SmallVector<uint64_t, 2> Record; 729 Record.push_back(CurrentVersion); 730 Stream.EmitRecord(INDEX_METADATA, Record); 731 732 // Write the set of known module files. 733 for (ModuleFilesMap::iterator M = ModuleFiles.begin(), 734 MEnd = ModuleFiles.end(); 735 M != MEnd; ++M) { 736 Record.clear(); 737 Record.push_back(M->second.ID); 738 Record.push_back(M->first->getSize()); 739 Record.push_back(M->first->getModificationTime()); 740 741 // File name 742 StringRef Name(M->first->getName()); 743 Record.push_back(Name.size()); 744 Record.append(Name.begin(), Name.end()); 745 746 // Dependencies 747 Record.push_back(M->second.Dependencies.size()); 748 Record.append(M->second.Dependencies.begin(), M->second.Dependencies.end()); 749 Stream.EmitRecord(MODULE, Record); 750 } 751 752 // Write the identifier -> module file mapping. 753 { 754 OnDiskChainedHashTableGenerator<IdentifierIndexWriterTrait> Generator; 755 IdentifierIndexWriterTrait Trait; 756 757 // Populate the hash table. 758 for (InterestingIdentifierMap::iterator I = InterestingIdentifiers.begin(), 759 IEnd = InterestingIdentifiers.end(); 760 I != IEnd; ++I) { 761 Generator.insert(I->first(), I->second, Trait); 762 } 763 764 // Create the on-disk hash table in a buffer. 765 SmallString<4096> IdentifierTable; 766 uint32_t BucketOffset; 767 { 768 llvm::raw_svector_ostream Out(IdentifierTable); 769 // Make sure that no bucket is at offset 0 770 clang::io::Emit32(Out, 0); 771 BucketOffset = Generator.Emit(Out, Trait); 772 } 773 774 // Create a blob abbreviation 775 BitCodeAbbrev *Abbrev = new BitCodeAbbrev(); 776 Abbrev->Add(BitCodeAbbrevOp(IDENTIFIER_INDEX)); 777 Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32)); 778 Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Blob)); 779 unsigned IDTableAbbrev = Stream.EmitAbbrev(Abbrev); 780 781 // Write the identifier table 782 Record.clear(); 783 Record.push_back(IDENTIFIER_INDEX); 784 Record.push_back(BucketOffset); 785 Stream.EmitRecordWithBlob(IDTableAbbrev, Record, IdentifierTable.str()); 786 } 787 788 Stream.ExitBlock(); 789} 790 791GlobalModuleIndex::ErrorCode 792GlobalModuleIndex::writeIndex(FileManager &FileMgr, StringRef Path) { 793 llvm::SmallString<128> IndexPath; 794 IndexPath += Path; 795 llvm::sys::path::append(IndexPath, IndexFileName); 796 797 // Coordinate building the global index file with other processes that might 798 // try to do the same. 799 llvm::LockFileManager Locked(IndexPath); 800 switch (Locked) { 801 case llvm::LockFileManager::LFS_Error: 802 return EC_IOError; 803 804 case llvm::LockFileManager::LFS_Owned: 805 // We're responsible for building the index ourselves. Do so below. 806 break; 807 808 case llvm::LockFileManager::LFS_Shared: 809 // Someone else is responsible for building the index. We don't care 810 // when they finish, so we're done. 811 return EC_Building; 812 } 813 814 // The module index builder. 815 GlobalModuleIndexBuilder Builder(FileMgr); 816 817 // Load each of the module files. 818 llvm::error_code EC; 819 for (llvm::sys::fs::directory_iterator D(Path, EC), DEnd; 820 D != DEnd && !EC; 821 D.increment(EC)) { 822 // If this isn't a module file, we don't care. 823 if (llvm::sys::path::extension(D->path()) != ".pcm") { 824 // ... unless it's a .pcm.lock file, which indicates that someone is 825 // in the process of rebuilding a module. They'll rebuild the index 826 // at the end of that translation unit, so we don't have to. 827 if (llvm::sys::path::extension(D->path()) == ".pcm.lock") 828 return EC_Building; 829 830 continue; 831 } 832 833 // If we can't find the module file, skip it. 834 const FileEntry *ModuleFile = FileMgr.getFile(D->path()); 835 if (!ModuleFile) 836 continue; 837 838 // Load this module file. 839 if (Builder.loadModuleFile(ModuleFile)) 840 return EC_IOError; 841 } 842 843 // The output buffer, into which the global index will be written. 844 SmallVector<char, 16> OutputBuffer; 845 { 846 llvm::BitstreamWriter OutputStream(OutputBuffer); 847 Builder.writeIndex(OutputStream); 848 } 849 850 // Write the global index file to a temporary file. 851 llvm::SmallString<128> IndexTmpPath; 852 int TmpFD; 853 if (llvm::sys::fs::unique_file(IndexPath + "-%%%%%%%%", TmpFD, IndexTmpPath)) 854 return EC_IOError; 855 856 // Open the temporary global index file for output. 857 llvm::raw_fd_ostream Out(TmpFD, true); 858 if (Out.has_error()) 859 return EC_IOError; 860 861 // Write the index. 862 Out.write(OutputBuffer.data(), OutputBuffer.size()); 863 Out.close(); 864 if (Out.has_error()) 865 return EC_IOError; 866 867 // Remove the old index file. It isn't relevant any more. 868 bool OldIndexExisted; 869 llvm::sys::fs::remove(IndexPath.str(), OldIndexExisted); 870 871 // Rename the newly-written index file to the proper name. 872 if (llvm::sys::fs::rename(IndexTmpPath.str(), IndexPath.str())) { 873 // Rename failed; just remove the 874 llvm::sys::fs::remove(IndexTmpPath.str(), OldIndexExisted); 875 return EC_IOError; 876 } 877 878 // We're done. 879 return EC_None; 880} 881