GlobalModuleIndex.cpp revision 6bcf27bb9a4b5c3f79cb44c0e4654a6d7619ad89 
1//===--- GlobalModuleIndex.cpp - Global Module Index ------------*- C++ -*-===//
2//
3//                     The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10// This file implements the GlobalModuleIndex class.
11//
12//===----------------------------------------------------------------------===//
13
14#include "ASTReaderInternals.h"
15#include "clang/Basic/FileManager.h"
16#include "clang/Lex/HeaderSearch.h"
17#include "clang/Serialization/ASTBitCodes.h"
18#include "clang/Serialization/GlobalModuleIndex.h"
19#include "clang/Serialization/Module.h"
20#include "llvm/ADT/DenseMap.h"
21#include "llvm/ADT/MapVector.h"
22#include "llvm/ADT/SmallString.h"
23#include "llvm/ADT/StringExtras.h"
24#include "llvm/Bitcode/BitstreamReader.h"
25#include "llvm/Bitcode/BitstreamWriter.h"
26#include "llvm/Support/FileSystem.h"
27#include "llvm/Support/LockFileManager.h"
28#include "llvm/Support/MemoryBuffer.h"
29#include "llvm/Support/OnDiskHashTable.h"
30#include "llvm/Support/Path.h"
31#include <cstdio>
32using namespace clang;
33using namespace serialization;
34
35//----------------------------------------------------------------------------//
36// Shared constants
37//----------------------------------------------------------------------------//
38namespace {
39  enum {
40    /// \brief The block containing the index.
41    GLOBAL_INDEX_BLOCK_ID = llvm::bitc::FIRST_APPLICATION_BLOCKID
42  };
43
44  /// \brief Describes the record types in the index.
45  enum IndexRecordTypes {
46    /// \brief Contains version information and potentially other metadata,
47    /// used to determine if we can read this global index file.
48    INDEX_METADATA,
49    /// \brief Describes a module, including its file name and dependencies.
50    MODULE,
51    /// \brief The index for identifiers.
52    IDENTIFIER_INDEX
53  };
54}
55
56/// \brief The name of the global index file.
57static const char * const IndexFileName = "modules.idx";
58
59/// \brief The global index file version.
60static const unsigned CurrentVersion = 1;
61
62//----------------------------------------------------------------------------//
63// Global module index reader.
64//----------------------------------------------------------------------------//
65
66namespace {
67
68/// \brief Trait used to read the identifier index from the on-disk hash
69/// table.
70class IdentifierIndexReaderTrait {
71public:
72  typedef StringRef external_key_type;
73  typedef StringRef internal_key_type;
74  typedef SmallVector<unsigned, 2> data_type;
75  typedef unsigned hash_value_type;
76  typedef unsigned offset_type;
77
78  static bool EqualKey(const internal_key_type& a, const internal_key_type& b) {
79    return a == b;
80  }
81
82  static hash_value_type ComputeHash(const internal_key_type& a) {
83    return llvm::HashString(a);
84  }
85
86  static std::pair<unsigned, unsigned>
87  ReadKeyDataLength(const unsigned char*& d) {
88    using namespace llvm::support;
89    unsigned KeyLen = endian::readNext<uint16_t, little, unaligned>(d);
90    unsigned DataLen = endian::readNext<uint16_t, little, unaligned>(d);
91    return std::make_pair(KeyLen, DataLen);
92  }
93
94  static const internal_key_type&
95  GetInternalKey(const external_key_type& x) { return x; }
96
97  static const external_key_type&
98  GetExternalKey(const internal_key_type& x) { return x; }
99
100  static internal_key_type ReadKey(const unsigned char* d, unsigned n) {
101    return StringRef((const char *)d, n);
102  }
103
104  static data_type ReadData(const internal_key_type& k,
105                            const unsigned char* d,
106                            unsigned DataLen) {
107    using namespace llvm::support;
108
109    data_type Result;
110    while (DataLen > 0) {
111      unsigned ID = endian::readNext<uint32_t, little, unaligned>(d);
112      Result.push_back(ID);
113      DataLen -= 4;
114    }
115
116    return Result;
117  }
118};
119
120typedef llvm::OnDiskIterableChainedHashTable<IdentifierIndexReaderTrait>
121    IdentifierIndexTable;
122
123}
124
125GlobalModuleIndex::GlobalModuleIndex(llvm::MemoryBuffer *Buffer,
126                                     llvm::BitstreamCursor Cursor)
127  : Buffer(Buffer), IdentifierIndex(),
128    NumIdentifierLookups(), NumIdentifierLookupHits()
129{
130  // Read the global index.
131  bool InGlobalIndexBlock = false;
132  bool Done = false;
133  while (!Done) {
134    llvm::BitstreamEntry Entry = Cursor.advance();
135
136    switch (Entry.Kind) {
137    case llvm::BitstreamEntry::Error:
138      return;
139
140    case llvm::BitstreamEntry::EndBlock:
141      if (InGlobalIndexBlock) {
142        InGlobalIndexBlock = false;
143        Done = true;
144        continue;
145      }
146      return;
147
148
149    case llvm::BitstreamEntry::Record:
150      // Entries in the global index block are handled below.
151      if (InGlobalIndexBlock)
152        break;
153
154      return;
155
156    case llvm::BitstreamEntry::SubBlock:
157      if (!InGlobalIndexBlock && Entry.ID == GLOBAL_INDEX_BLOCK_ID) {
158        if (Cursor.EnterSubBlock(GLOBAL_INDEX_BLOCK_ID))
159          return;
160
161        InGlobalIndexBlock = true;
162      } else if (Cursor.SkipBlock()) {
163        return;
164      }
165      continue;
166    }
167
168    SmallVector<uint64_t, 64> Record;
169    StringRef Blob;
170    switch ((IndexRecordTypes)Cursor.readRecord(Entry.ID, Record, &Blob)) {
171    case INDEX_METADATA:
172      // Make sure that the version matches.
173      if (Record.size() < 1 || Record[0] != CurrentVersion)
174        return;
175      break;
176
177    case MODULE: {
178      unsigned Idx = 0;
179      unsigned ID = Record[Idx++];
180
181      // Make room for this module's information.
182      if (ID == Modules.size())
183        Modules.push_back(ModuleInfo());
184      else
185        Modules.resize(ID + 1);
186
187      // Size/modification time for this module file at the time the
188      // global index was built.
189      Modules[ID].Size = Record[Idx++];
190      Modules[ID].ModTime = Record[Idx++];
191
192      // File name.
193      unsigned NameLen = Record[Idx++];
194      Modules[ID].FileName.assign(Record.begin() + Idx,
195                                  Record.begin() + Idx + NameLen);
196      Idx += NameLen;
197
198      // Dependencies
199      unsigned NumDeps = Record[Idx++];
200      Modules[ID].Dependencies.insert(Modules[ID].Dependencies.end(),
201                                      Record.begin() + Idx,
202                                      Record.begin() + Idx + NumDeps);
203      Idx += NumDeps;
204
205      // Make sure we're at the end of the record.
206      assert(Idx == Record.size() && "More module info?");
207
208      // Record this module as an unresolved module.
209      // FIXME: this doesn't work correctly for module names containing path
210      // separators.
211      StringRef ModuleName = llvm::sys::path::stem(Modules[ID].FileName);
212      // Remove the -<hash of ModuleMapPath>
213      ModuleName = ModuleName.rsplit('-').first;
214      UnresolvedModules[ModuleName] = ID;
215      break;
216    }
217
218    case IDENTIFIER_INDEX:
219      // Wire up the identifier index.
220      if (Record[0]) {
221        IdentifierIndex = IdentifierIndexTable::Create(
222            (const unsigned char *)Blob.data() + Record[0],
223            (const unsigned char *)Blob.data() + sizeof(uint32_t),
224            (const unsigned char *)Blob.data(), IdentifierIndexReaderTrait());
225      }
226      break;
227    }
228  }
229}
230
231GlobalModuleIndex::~GlobalModuleIndex() {
232  delete static_cast<IdentifierIndexTable *>(IdentifierIndex);
233}
234
235std::pair<GlobalModuleIndex *, GlobalModuleIndex::ErrorCode>
236GlobalModuleIndex::readIndex(StringRef Path) {
237  // Load the index file, if it's there.
238  llvm::SmallString<128> IndexPath;
239  IndexPath += Path;
240  llvm::sys::path::append(IndexPath, IndexFileName);
241
242  std::unique_ptr<llvm::MemoryBuffer> Buffer;
243  if (llvm::MemoryBuffer::getFile(IndexPath.c_str(), Buffer) !=
244      llvm::errc::success)
245    return std::make_pair(nullptr, EC_NotFound);
246
247  /// \brief The bitstream reader from which we'll read the AST file.
248  llvm::BitstreamReader Reader((const unsigned char *)Buffer->getBufferStart(),
249                               (const unsigned char *)Buffer->getBufferEnd());
250
251  /// \brief The main bitstream cursor for the main block.
252  llvm::BitstreamCursor Cursor(Reader);
253
254  // Sniff for the signature.
255  if (Cursor.Read(8) != 'B' ||
256      Cursor.Read(8) != 'C' ||
257      Cursor.Read(8) != 'G' ||
258      Cursor.Read(8) != 'I') {
259    return std::make_pair(nullptr, EC_IOError);
260  }
261
262  return std::make_pair(new GlobalModuleIndex(Buffer.release(), Cursor),
263                        EC_None);
264}
265
266void
267GlobalModuleIndex::getKnownModules(SmallVectorImpl<ModuleFile *> &ModuleFiles) {
268  ModuleFiles.clear();
269  for (unsigned I = 0, N = Modules.size(); I != N; ++I) {
270    if (ModuleFile *MF = Modules[I].File)
271      ModuleFiles.push_back(MF);
272  }
273}
274
275void GlobalModuleIndex::getModuleDependencies(
276       ModuleFile *File,
277       SmallVectorImpl<ModuleFile *> &Dependencies) {
278  // Look for information about this module file.
279  llvm::DenseMap<ModuleFile *, unsigned>::iterator Known
280    = ModulesByFile.find(File);
281  if (Known == ModulesByFile.end())
282    return;
283
284  // Record dependencies.
285  Dependencies.clear();
286  ArrayRef<unsigned> StoredDependencies = Modules[Known->second].Dependencies;
287  for (unsigned I = 0, N = StoredDependencies.size(); I != N; ++I) {
288    if (ModuleFile *MF = Modules[I].File)
289      Dependencies.push_back(MF);
290  }
291}
292
293bool GlobalModuleIndex::lookupIdentifier(StringRef Name, HitSet &Hits) {
294  Hits.clear();
295
296  // If there's no identifier index, there is nothing we can do.
297  if (!IdentifierIndex)
298    return false;
299
300  // Look into the identifier index.
301  ++NumIdentifierLookups;
302  IdentifierIndexTable &Table
303    = *static_cast<IdentifierIndexTable *>(IdentifierIndex);
304  IdentifierIndexTable::iterator Known = Table.find(Name);
305  if (Known == Table.end()) {
306    return true;
307  }
308
309  SmallVector<unsigned, 2> ModuleIDs = *Known;
310  for (unsigned I = 0, N = ModuleIDs.size(); I != N; ++I) {
311    if (ModuleFile *MF = Modules[ModuleIDs[I]].File)
312      Hits.insert(MF);
313  }
314
315  ++NumIdentifierLookupHits;
316  return true;
317}
318
319bool GlobalModuleIndex::loadedModuleFile(ModuleFile *File) {
320  // Look for the module in the global module index based on the module name.
321  StringRef Name = File->ModuleName;
322  llvm::StringMap<unsigned>::iterator Known = UnresolvedModules.find(Name);
323  if (Known == UnresolvedModules.end()) {
324    return true;
325  }
326
327  // Rectify this module with the global module index.
328  ModuleInfo &Info = Modules[Known->second];
329
330  //  If the size and modification time match what we expected, record this
331  // module file.
332  bool Failed = true;
333  if (File->File->getSize() == Info.Size &&
334      File->File->getModificationTime() == Info.ModTime) {
335    Info.File = File;
336    ModulesByFile[File] = Known->second;
337
338    Failed = false;
339  }
340
341  // One way or another, we have resolved this module file.
342  UnresolvedModules.erase(Known);
343  return Failed;
344}
345
346void GlobalModuleIndex::printStats() {
347  std::fprintf(stderr, "*** Global Module Index Statistics:\n");
348  if (NumIdentifierLookups) {
349    fprintf(stderr, "  %u / %u identifier lookups succeeded (%f%%)\n",
350            NumIdentifierLookupHits, NumIdentifierLookups,
351            (double)NumIdentifierLookupHits*100.0/NumIdentifierLookups);
352  }
353  std::fprintf(stderr, "\n");
354}
355
356void GlobalModuleIndex::dump() {
357  llvm::errs() << "*** Global Module Index Dump:\n";
358  llvm::errs() << "Module files:\n";
359  for (auto &MI : Modules) {
360    llvm::errs() << "** " << MI.FileName << "\n";
361    if (MI.File)
362      MI.File->dump();
363    else
364      llvm::errs() << "\n";
365  }
366  llvm::errs() << "\n";
367}
368
369//----------------------------------------------------------------------------//
370// Global module index writer.
371//----------------------------------------------------------------------------//
372
373namespace {
374  /// \brief Provides information about a specific module file.
375  struct ModuleFileInfo {
376    /// \brief The numberic ID for this module file.
377    unsigned ID;
378
379    /// \brief The set of modules on which this module depends. Each entry is
380    /// a module ID.
381    SmallVector<unsigned, 4> Dependencies;
382  };
383
384  /// \brief Builder that generates the global module index file.
385  class GlobalModuleIndexBuilder {
386    FileManager &FileMgr;
387
388    /// \brief Mapping from files to module file information.
389    typedef llvm::MapVector<const FileEntry *, ModuleFileInfo> ModuleFilesMap;
390
391    /// \brief Information about each of the known module files.
392    ModuleFilesMap ModuleFiles;
393
394    /// \brief Mapping from identifiers to the list of module file IDs that
395    /// consider this identifier to be interesting.
396    typedef llvm::StringMap<SmallVector<unsigned, 2> > InterestingIdentifierMap;
397
398    /// \brief A mapping from all interesting identifiers to the set of module
399    /// files in which those identifiers are considered interesting.
400    InterestingIdentifierMap InterestingIdentifiers;
401
402    /// \brief Write the block-info block for the global module index file.
403    void emitBlockInfoBlock(llvm::BitstreamWriter &Stream);
404
405    /// \brief Retrieve the module file information for the given file.
406    ModuleFileInfo &getModuleFileInfo(const FileEntry *File) {
407      llvm::MapVector<const FileEntry *, ModuleFileInfo>::iterator Known
408        = ModuleFiles.find(File);
409      if (Known != ModuleFiles.end())
410        return Known->second;
411
412      unsigned NewID = ModuleFiles.size();
413      ModuleFileInfo &Info = ModuleFiles[File];
414      Info.ID = NewID;
415      return Info;
416    }
417
418  public:
419    explicit GlobalModuleIndexBuilder(FileManager &FileMgr) : FileMgr(FileMgr){}
420
421    /// \brief Load the contents of the given module file into the builder.
422    ///
423    /// \returns true if an error occurred, false otherwise.
424    bool loadModuleFile(const FileEntry *File);
425
426    /// \brief Write the index to the given bitstream.
427    void writeIndex(llvm::BitstreamWriter &Stream);
428  };
429}
430
431static void emitBlockID(unsigned ID, const char *Name,
432                        llvm::BitstreamWriter &Stream,
433                        SmallVectorImpl<uint64_t> &Record) {
434  Record.clear();
435  Record.push_back(ID);
436  Stream.EmitRecord(llvm::bitc::BLOCKINFO_CODE_SETBID, Record);
437
438  // Emit the block name if present.
439  if (!Name || Name[0] == 0) return;
440  Record.clear();
441  while (*Name)
442    Record.push_back(*Name++);
443  Stream.EmitRecord(llvm::bitc::BLOCKINFO_CODE_BLOCKNAME, Record);
444}
445
446static void emitRecordID(unsigned ID, const char *Name,
447                         llvm::BitstreamWriter &Stream,
448                         SmallVectorImpl<uint64_t> &Record) {
449  Record.clear();
450  Record.push_back(ID);
451  while (*Name)
452    Record.push_back(*Name++);
453  Stream.EmitRecord(llvm::bitc::BLOCKINFO_CODE_SETRECORDNAME, Record);
454}
455
456void
457GlobalModuleIndexBuilder::emitBlockInfoBlock(llvm::BitstreamWriter &Stream) {
458  SmallVector<uint64_t, 64> Record;
459  Stream.EnterSubblock(llvm::bitc::BLOCKINFO_BLOCK_ID, 3);
460
461#define BLOCK(X) emitBlockID(X ## _ID, #X, Stream, Record)
462#define RECORD(X) emitRecordID(X, #X, Stream, Record)
463  BLOCK(GLOBAL_INDEX_BLOCK);
464  RECORD(INDEX_METADATA);
465  RECORD(MODULE);
466  RECORD(IDENTIFIER_INDEX);
467#undef RECORD
468#undef BLOCK
469
470  Stream.ExitBlock();
471}
472
473namespace {
474  class InterestingASTIdentifierLookupTrait
475    : public serialization::reader::ASTIdentifierLookupTraitBase {
476
477  public:
478    /// \brief The identifier and whether it is "interesting".
479    typedef std::pair<StringRef, bool> data_type;
480
481    data_type ReadData(const internal_key_type& k,
482                       const unsigned char* d,
483                       unsigned DataLen) {
484      // The first bit indicates whether this identifier is interesting.
485      // That's all we care about.
486      using namespace llvm::support;
487      unsigned RawID = endian::readNext<uint32_t, little, unaligned>(d);
488      bool IsInteresting = RawID & 0x01;
489      return std::make_pair(k, IsInteresting);
490    }
491  };
492}
493
494bool GlobalModuleIndexBuilder::loadModuleFile(const FileEntry *File) {
495  // Open the module file.
496  std::unique_ptr<llvm::MemoryBuffer> Buffer;
497  std::string ErrorStr;
498  Buffer.reset(FileMgr.getBufferForFile(File, &ErrorStr, /*isVolatile=*/true));
499  if (!Buffer) {
500    return true;
501  }
502
503  // Initialize the input stream
504  llvm::BitstreamReader InStreamFile;
505  llvm::BitstreamCursor InStream;
506  InStreamFile.init((const unsigned char *)Buffer->getBufferStart(),
507                  (const unsigned char *)Buffer->getBufferEnd());
508  InStream.init(InStreamFile);
509
510  // Sniff for the signature.
511  if (InStream.Read(8) != 'C' ||
512      InStream.Read(8) != 'P' ||
513      InStream.Read(8) != 'C' ||
514      InStream.Read(8) != 'H') {
515    return true;
516  }
517
518  // Record this module file and assign it a unique ID (if it doesn't have
519  // one already).
520  unsigned ID = getModuleFileInfo(File).ID;
521
522  // Search for the blocks and records we care about.
523  enum { Other, ControlBlock, ASTBlock } State = Other;
524  bool Done = false;
525  while (!Done) {
526    llvm::BitstreamEntry Entry = InStream.advance();
527    switch (Entry.Kind) {
528    case llvm::BitstreamEntry::Error:
529      Done = true;
530      continue;
531
532    case llvm::BitstreamEntry::Record:
533      // In the 'other' state, just skip the record. We don't care.
534      if (State == Other) {
535        InStream.skipRecord(Entry.ID);
536        continue;
537      }
538
539      // Handle potentially-interesting records below.
540      break;
541
542    case llvm::BitstreamEntry::SubBlock:
543      if (Entry.ID == CONTROL_BLOCK_ID) {
544        if (InStream.EnterSubBlock(CONTROL_BLOCK_ID))
545          return true;
546
547        // Found the control block.
548        State = ControlBlock;
549        continue;
550      }
551
552      if (Entry.ID == AST_BLOCK_ID) {
553        if (InStream.EnterSubBlock(AST_BLOCK_ID))
554          return true;
555
556        // Found the AST block.
557        State = ASTBlock;
558        continue;
559      }
560
561      if (InStream.SkipBlock())
562        return true;
563
564      continue;
565
566    case llvm::BitstreamEntry::EndBlock:
567      State = Other;
568      continue;
569    }
570
571    // Read the given record.
572    SmallVector<uint64_t, 64> Record;
573    StringRef Blob;
574    unsigned Code = InStream.readRecord(Entry.ID, Record, &Blob);
575
576    // Handle module dependencies.
577    if (State == ControlBlock && Code == IMPORTS) {
578      // Load each of the imported PCH files.
579      unsigned Idx = 0, N = Record.size();
580      while (Idx < N) {
581        // Read information about the AST file.
582
583        // Skip the imported kind
584        ++Idx;
585
586        // Skip the import location
587        ++Idx;
588
589        // Load stored size/modification time.
590        off_t StoredSize = (off_t)Record[Idx++];
591        time_t StoredModTime = (time_t)Record[Idx++];
592
593        // Retrieve the imported file name.
594        unsigned Length = Record[Idx++];
595        SmallString<128> ImportedFile(Record.begin() + Idx,
596                                      Record.begin() + Idx + Length);
597        Idx += Length;
598
599        // Find the imported module file.
600        const FileEntry *DependsOnFile
601          = FileMgr.getFile(ImportedFile, /*openFile=*/false,
602                            /*cacheFailure=*/false);
603        if (!DependsOnFile ||
604            (StoredSize != DependsOnFile->getSize()) ||
605            (StoredModTime != DependsOnFile->getModificationTime()))
606          return true;
607
608        // Record the dependency.
609        unsigned DependsOnID = getModuleFileInfo(DependsOnFile).ID;
610        getModuleFileInfo(File).Dependencies.push_back(DependsOnID);
611      }
612
613      continue;
614    }
615
616    // Handle the identifier table
617    if (State == ASTBlock && Code == IDENTIFIER_TABLE && Record[0] > 0) {
618      typedef llvm::OnDiskIterableChainedHashTable<
619          InterestingASTIdentifierLookupTrait> InterestingIdentifierTable;
620      std::unique_ptr<InterestingIdentifierTable> Table(
621          InterestingIdentifierTable::Create(
622              (const unsigned char *)Blob.data() + Record[0],
623              (const unsigned char *)Blob.data() + sizeof(uint32_t),
624              (const unsigned char *)Blob.data()));
625      for (InterestingIdentifierTable::data_iterator D = Table->data_begin(),
626                                                     DEnd = Table->data_end();
627           D != DEnd; ++D) {
628        std::pair<StringRef, bool> Ident = *D;
629        if (Ident.second)
630          InterestingIdentifiers[Ident.first].push_back(ID);
631        else
632          (void)InterestingIdentifiers[Ident.first];
633      }
634    }
635
636    // We don't care about this record.
637  }
638
639  return false;
640}
641
642namespace {
643
644/// \brief Trait used to generate the identifier index as an on-disk hash
645/// table.
646class IdentifierIndexWriterTrait {
647public:
648  typedef StringRef key_type;
649  typedef StringRef key_type_ref;
650  typedef SmallVector<unsigned, 2> data_type;
651  typedef const SmallVector<unsigned, 2> &data_type_ref;
652  typedef unsigned hash_value_type;
653  typedef unsigned offset_type;
654
655  static hash_value_type ComputeHash(key_type_ref Key) {
656    return llvm::HashString(Key);
657  }
658
659  std::pair<unsigned,unsigned>
660  EmitKeyDataLength(raw_ostream& Out, key_type_ref Key, data_type_ref Data) {
661    using namespace llvm::support;
662    endian::Writer<little> LE(Out);
663    unsigned KeyLen = Key.size();
664    unsigned DataLen = Data.size() * 4;
665    LE.write<uint16_t>(KeyLen);
666    LE.write<uint16_t>(DataLen);
667    return std::make_pair(KeyLen, DataLen);
668  }
669
670  void EmitKey(raw_ostream& Out, key_type_ref Key, unsigned KeyLen) {
671    Out.write(Key.data(), KeyLen);
672  }
673
674  void EmitData(raw_ostream& Out, key_type_ref Key, data_type_ref Data,
675                unsigned DataLen) {
676    using namespace llvm::support;
677    for (unsigned I = 0, N = Data.size(); I != N; ++I)
678      endian::Writer<little>(Out).write<uint32_t>(Data[I]);
679  }
680};
681
682}
683
684void GlobalModuleIndexBuilder::writeIndex(llvm::BitstreamWriter &Stream) {
685  using namespace llvm;
686
687  // Emit the file header.
688  Stream.Emit((unsigned)'B', 8);
689  Stream.Emit((unsigned)'C', 8);
690  Stream.Emit((unsigned)'G', 8);
691  Stream.Emit((unsigned)'I', 8);
692
693  // Write the block-info block, which describes the records in this bitcode
694  // file.
695  emitBlockInfoBlock(Stream);
696
697  Stream.EnterSubblock(GLOBAL_INDEX_BLOCK_ID, 3);
698
699  // Write the metadata.
700  SmallVector<uint64_t, 2> Record;
701  Record.push_back(CurrentVersion);
702  Stream.EmitRecord(INDEX_METADATA, Record);
703
704  // Write the set of known module files.
705  for (ModuleFilesMap::iterator M = ModuleFiles.begin(),
706                                MEnd = ModuleFiles.end();
707       M != MEnd; ++M) {
708    Record.clear();
709    Record.push_back(M->second.ID);
710    Record.push_back(M->first->getSize());
711    Record.push_back(M->first->getModificationTime());
712
713    // File name
714    StringRef Name(M->first->getName());
715    Record.push_back(Name.size());
716    Record.append(Name.begin(), Name.end());
717
718    // Dependencies
719    Record.push_back(M->second.Dependencies.size());
720    Record.append(M->second.Dependencies.begin(), M->second.Dependencies.end());
721    Stream.EmitRecord(MODULE, Record);
722  }
723
724  // Write the identifier -> module file mapping.
725  {
726    llvm::OnDiskChainedHashTableGenerator<IdentifierIndexWriterTrait> Generator;
727    IdentifierIndexWriterTrait Trait;
728
729    // Populate the hash table.
730    for (InterestingIdentifierMap::iterator I = InterestingIdentifiers.begin(),
731                                            IEnd = InterestingIdentifiers.end();
732         I != IEnd; ++I) {
733      Generator.insert(I->first(), I->second, Trait);
734    }
735
736    // Create the on-disk hash table in a buffer.
737    SmallString<4096> IdentifierTable;
738    uint32_t BucketOffset;
739    {
740      using namespace llvm::support;
741      llvm::raw_svector_ostream Out(IdentifierTable);
742      // Make sure that no bucket is at offset 0
743      endian::Writer<little>(Out).write<uint32_t>(0);
744      BucketOffset = Generator.Emit(Out, Trait);
745    }
746
747    // Create a blob abbreviation
748    BitCodeAbbrev *Abbrev = new BitCodeAbbrev();
749    Abbrev->Add(BitCodeAbbrevOp(IDENTIFIER_INDEX));
750    Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
751    Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Blob));
752    unsigned IDTableAbbrev = Stream.EmitAbbrev(Abbrev);
753
754    // Write the identifier table
755    Record.clear();
756    Record.push_back(IDENTIFIER_INDEX);
757    Record.push_back(BucketOffset);
758    Stream.EmitRecordWithBlob(IDTableAbbrev, Record, IdentifierTable.str());
759  }
760
761  Stream.ExitBlock();
762}
763
764GlobalModuleIndex::ErrorCode
765GlobalModuleIndex::writeIndex(FileManager &FileMgr, StringRef Path) {
766  llvm::SmallString<128> IndexPath;
767  IndexPath += Path;
768  llvm::sys::path::append(IndexPath, IndexFileName);
769
770  // Coordinate building the global index file with other processes that might
771  // try to do the same.
772  llvm::LockFileManager Locked(IndexPath);
773  switch (Locked) {
774  case llvm::LockFileManager::LFS_Error:
775    return EC_IOError;
776
777  case llvm::LockFileManager::LFS_Owned:
778    // We're responsible for building the index ourselves. Do so below.
779    break;
780
781  case llvm::LockFileManager::LFS_Shared:
782    // Someone else is responsible for building the index. We don't care
783    // when they finish, so we're done.
784    return EC_Building;
785  }
786
787  // The module index builder.
788  GlobalModuleIndexBuilder Builder(FileMgr);
789
790  // Load each of the module files.
791  llvm::error_code EC;
792  for (llvm::sys::fs::directory_iterator D(Path, EC), DEnd;
793       D != DEnd && !EC;
794       D.increment(EC)) {
795    // If this isn't a module file, we don't care.
796    if (llvm::sys::path::extension(D->path()) != ".pcm") {
797      // ... unless it's a .pcm.lock file, which indicates that someone is
798      // in the process of rebuilding a module. They'll rebuild the index
799      // at the end of that translation unit, so we don't have to.
800      if (llvm::sys::path::extension(D->path()) == ".pcm.lock")
801        return EC_Building;
802
803      continue;
804    }
805
806    // If we can't find the module file, skip it.
807    const FileEntry *ModuleFile = FileMgr.getFile(D->path());
808    if (!ModuleFile)
809      continue;
810
811    // Load this module file.
812    if (Builder.loadModuleFile(ModuleFile))
813      return EC_IOError;
814  }
815
816  // The output buffer, into which the global index will be written.
817  SmallVector<char, 16> OutputBuffer;
818  {
819    llvm::BitstreamWriter OutputStream(OutputBuffer);
820    Builder.writeIndex(OutputStream);
821  }
822
823  // Write the global index file to a temporary file.
824  llvm::SmallString<128> IndexTmpPath;
825  int TmpFD;
826  if (llvm::sys::fs::createUniqueFile(IndexPath + "-%%%%%%%%", TmpFD,
827                                      IndexTmpPath))
828    return EC_IOError;
829
830  // Open the temporary global index file for output.
831  llvm::raw_fd_ostream Out(TmpFD, true);
832  if (Out.has_error())
833    return EC_IOError;
834
835  // Write the index.
836  Out.write(OutputBuffer.data(), OutputBuffer.size());
837  Out.close();
838  if (Out.has_error())
839    return EC_IOError;
840
841  // Remove the old index file. It isn't relevant any more.
842  llvm::sys::fs::remove(IndexPath.str());
843
844  // Rename the newly-written index file to the proper name.
845  if (llvm::sys::fs::rename(IndexTmpPath.str(), IndexPath.str())) {
846    // Rename failed; just remove the
847    llvm::sys::fs::remove(IndexTmpPath.str());
848    return EC_IOError;
849  }
850
851  // We're done.
852  return EC_None;
853}
854
855namespace {
856  class GlobalIndexIdentifierIterator : public IdentifierIterator {
857    /// \brief The current position within the identifier lookup table.
858    IdentifierIndexTable::key_iterator Current;
859
860    /// \brief The end position within the identifier lookup table.
861    IdentifierIndexTable::key_iterator End;
862
863  public:
864    explicit GlobalIndexIdentifierIterator(IdentifierIndexTable &Idx) {
865      Current = Idx.key_begin();
866      End = Idx.key_end();
867    }
868
869    StringRef Next() override {
870      if (Current == End)
871        return StringRef();
872
873      StringRef Result = *Current;
874      ++Current;
875      return Result;
876    }
877  };
878}
879
880IdentifierIterator *GlobalModuleIndex::createIdentifierIterator() const {
881  IdentifierIndexTable &Table =
882    *static_cast<IdentifierIndexTable *>(IdentifierIndex);
883  return new GlobalIndexIdentifierIterator(Table);
884}
885