GlobalModuleIndex.cpp revision 651f13cea278ec967336033dd032faef0e9fc2ec 
1//===--- GlobalModuleIndex.cpp - Global Module Index ------------*- C++ -*-===//
2//
3//                     The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10// This file implements the GlobalModuleIndex class.
11//
12//===----------------------------------------------------------------------===//
13
14#include "ASTReaderInternals.h"
15#include "clang/Basic/FileManager.h"
16#include "clang/Basic/OnDiskHashTable.h"
17#include "clang/Serialization/ASTBitCodes.h"
18#include "clang/Serialization/GlobalModuleIndex.h"
19#include "clang/Serialization/Module.h"
20#include "llvm/ADT/DenseMap.h"
21#include "llvm/ADT/MapVector.h"
22#include "llvm/ADT/SmallString.h"
23#include "llvm/ADT/StringExtras.h"
24#include "llvm/Bitcode/BitstreamReader.h"
25#include "llvm/Bitcode/BitstreamWriter.h"
26#include "llvm/Support/FileSystem.h"
27#include "llvm/Support/LockFileManager.h"
28#include "llvm/Support/MemoryBuffer.h"
29#include "llvm/Support/Path.h"
30#include <cstdio>
31using namespace clang;
32using namespace serialization;
33
34//----------------------------------------------------------------------------//
35// Shared constants
36//----------------------------------------------------------------------------//
37namespace {
38  enum {
39    /// \brief The block containing the index.
40    GLOBAL_INDEX_BLOCK_ID = llvm::bitc::FIRST_APPLICATION_BLOCKID
41  };
42
43  /// \brief Describes the record types in the index.
44  enum IndexRecordTypes {
45    /// \brief Contains version information and potentially other metadata,
46    /// used to determine if we can read this global index file.
47    INDEX_METADATA,
48    /// \brief Describes a module, including its file name and dependencies.
49    MODULE,
50    /// \brief The index for identifiers.
51    IDENTIFIER_INDEX
52  };
53}
54
55/// \brief The name of the global index file.
56static const char * const IndexFileName = "modules.idx";
57
58/// \brief The global index file version.
59static const unsigned CurrentVersion = 1;
60
61//----------------------------------------------------------------------------//
62// Global module index reader.
63//----------------------------------------------------------------------------//
64
65namespace {
66
67/// \brief Trait used to read the identifier index from the on-disk hash
68/// table.
69class IdentifierIndexReaderTrait {
70public:
71  typedef StringRef external_key_type;
72  typedef StringRef internal_key_type;
73  typedef SmallVector<unsigned, 2> data_type;
74
75  static bool EqualKey(const internal_key_type& a, const internal_key_type& b) {
76    return a == b;
77  }
78
79  static unsigned ComputeHash(const internal_key_type& a) {
80    return llvm::HashString(a);
81  }
82
83  static std::pair<unsigned, unsigned>
84  ReadKeyDataLength(const unsigned char*& d) {
85    using namespace llvm::support;
86    unsigned KeyLen = endian::readNext<uint16_t, little, unaligned>(d);
87    unsigned DataLen = endian::readNext<uint16_t, little, unaligned>(d);
88    return std::make_pair(KeyLen, DataLen);
89  }
90
91  static const internal_key_type&
92  GetInternalKey(const external_key_type& x) { return x; }
93
94  static const external_key_type&
95  GetExternalKey(const internal_key_type& x) { return x; }
96
97  static internal_key_type ReadKey(const unsigned char* d, unsigned n) {
98    return StringRef((const char *)d, n);
99  }
100
101  static data_type ReadData(const internal_key_type& k,
102                            const unsigned char* d,
103                            unsigned DataLen) {
104    using namespace llvm::support;
105
106    data_type Result;
107    while (DataLen > 0) {
108      unsigned ID = endian::readNext<uint32_t, little, unaligned>(d);
109      Result.push_back(ID);
110      DataLen -= 4;
111    }
112
113    return Result;
114  }
115};
116
117typedef OnDiskChainedHashTable<IdentifierIndexReaderTrait> IdentifierIndexTable;
118
119}
120
121GlobalModuleIndex::GlobalModuleIndex(llvm::MemoryBuffer *Buffer,
122                                     llvm::BitstreamCursor Cursor)
123  : Buffer(Buffer), IdentifierIndex(),
124    NumIdentifierLookups(), NumIdentifierLookupHits()
125{
126  // Read the global index.
127  bool InGlobalIndexBlock = false;
128  bool Done = false;
129  while (!Done) {
130    llvm::BitstreamEntry Entry = Cursor.advance();
131
132    switch (Entry.Kind) {
133    case llvm::BitstreamEntry::Error:
134      return;
135
136    case llvm::BitstreamEntry::EndBlock:
137      if (InGlobalIndexBlock) {
138        InGlobalIndexBlock = false;
139        Done = true;
140        continue;
141      }
142      return;
143
144
145    case llvm::BitstreamEntry::Record:
146      // Entries in the global index block are handled below.
147      if (InGlobalIndexBlock)
148        break;
149
150      return;
151
152    case llvm::BitstreamEntry::SubBlock:
153      if (!InGlobalIndexBlock && Entry.ID == GLOBAL_INDEX_BLOCK_ID) {
154        if (Cursor.EnterSubBlock(GLOBAL_INDEX_BLOCK_ID))
155          return;
156
157        InGlobalIndexBlock = true;
158      } else if (Cursor.SkipBlock()) {
159        return;
160      }
161      continue;
162    }
163
164    SmallVector<uint64_t, 64> Record;
165    StringRef Blob;
166    switch ((IndexRecordTypes)Cursor.readRecord(Entry.ID, Record, &Blob)) {
167    case INDEX_METADATA:
168      // Make sure that the version matches.
169      if (Record.size() < 1 || Record[0] != CurrentVersion)
170        return;
171      break;
172
173    case MODULE: {
174      unsigned Idx = 0;
175      unsigned ID = Record[Idx++];
176
177      // Make room for this module's information.
178      if (ID == Modules.size())
179        Modules.push_back(ModuleInfo());
180      else
181        Modules.resize(ID + 1);
182
183      // Size/modification time for this module file at the time the
184      // global index was built.
185      Modules[ID].Size = Record[Idx++];
186      Modules[ID].ModTime = Record[Idx++];
187
188      // File name.
189      unsigned NameLen = Record[Idx++];
190      Modules[ID].FileName.assign(Record.begin() + Idx,
191                                  Record.begin() + Idx + NameLen);
192      Idx += NameLen;
193
194      // Dependencies
195      unsigned NumDeps = Record[Idx++];
196      Modules[ID].Dependencies.insert(Modules[ID].Dependencies.end(),
197                                      Record.begin() + Idx,
198                                      Record.begin() + Idx + NumDeps);
199      Idx += NumDeps;
200
201      // Make sure we're at the end of the record.
202      assert(Idx == Record.size() && "More module info?");
203
204      // Record this module as an unresolved module.
205      UnresolvedModules[llvm::sys::path::stem(Modules[ID].FileName)] = ID;
206      break;
207    }
208
209    case IDENTIFIER_INDEX:
210      // Wire up the identifier index.
211      if (Record[0]) {
212        IdentifierIndex = IdentifierIndexTable::Create(
213                            (const unsigned char *)Blob.data() + Record[0],
214                            (const unsigned char *)Blob.data(),
215                            IdentifierIndexReaderTrait());
216      }
217      break;
218    }
219  }
220}
221
222GlobalModuleIndex::~GlobalModuleIndex() { }
223
224std::pair<GlobalModuleIndex *, GlobalModuleIndex::ErrorCode>
225GlobalModuleIndex::readIndex(StringRef Path) {
226  // Load the index file, if it's there.
227  llvm::SmallString<128> IndexPath;
228  IndexPath += Path;
229  llvm::sys::path::append(IndexPath, IndexFileName);
230
231  std::unique_ptr<llvm::MemoryBuffer> Buffer;
232  if (llvm::MemoryBuffer::getFile(IndexPath.c_str(), Buffer) !=
233      llvm::errc::success)
234    return std::make_pair((GlobalModuleIndex *)0, EC_NotFound);
235
236  /// \brief The bitstream reader from which we'll read the AST file.
237  llvm::BitstreamReader Reader((const unsigned char *)Buffer->getBufferStart(),
238                               (const unsigned char *)Buffer->getBufferEnd());
239
240  /// \brief The main bitstream cursor for the main block.
241  llvm::BitstreamCursor Cursor(Reader);
242
243  // Sniff for the signature.
244  if (Cursor.Read(8) != 'B' ||
245      Cursor.Read(8) != 'C' ||
246      Cursor.Read(8) != 'G' ||
247      Cursor.Read(8) != 'I') {
248    return std::make_pair((GlobalModuleIndex *)0, EC_IOError);
249  }
250
251  return std::make_pair(new GlobalModuleIndex(Buffer.release(), Cursor),
252                        EC_None);
253}
254
255void
256GlobalModuleIndex::getKnownModules(SmallVectorImpl<ModuleFile *> &ModuleFiles) {
257  ModuleFiles.clear();
258  for (unsigned I = 0, N = Modules.size(); I != N; ++I) {
259    if (ModuleFile *MF = Modules[I].File)
260      ModuleFiles.push_back(MF);
261  }
262}
263
264void GlobalModuleIndex::getModuleDependencies(
265       ModuleFile *File,
266       SmallVectorImpl<ModuleFile *> &Dependencies) {
267  // Look for information about this module file.
268  llvm::DenseMap<ModuleFile *, unsigned>::iterator Known
269    = ModulesByFile.find(File);
270  if (Known == ModulesByFile.end())
271    return;
272
273  // Record dependencies.
274  Dependencies.clear();
275  ArrayRef<unsigned> StoredDependencies = Modules[Known->second].Dependencies;
276  for (unsigned I = 0, N = StoredDependencies.size(); I != N; ++I) {
277    if (ModuleFile *MF = Modules[I].File)
278      Dependencies.push_back(MF);
279  }
280}
281
282bool GlobalModuleIndex::lookupIdentifier(StringRef Name, HitSet &Hits) {
283  Hits.clear();
284
285  // If there's no identifier index, there is nothing we can do.
286  if (!IdentifierIndex)
287    return false;
288
289  // Look into the identifier index.
290  ++NumIdentifierLookups;
291  IdentifierIndexTable &Table
292    = *static_cast<IdentifierIndexTable *>(IdentifierIndex);
293  IdentifierIndexTable::iterator Known = Table.find(Name);
294  if (Known == Table.end()) {
295    return true;
296  }
297
298  SmallVector<unsigned, 2> ModuleIDs = *Known;
299  for (unsigned I = 0, N = ModuleIDs.size(); I != N; ++I) {
300    if (ModuleFile *MF = Modules[ModuleIDs[I]].File)
301      Hits.insert(MF);
302  }
303
304  ++NumIdentifierLookupHits;
305  return true;
306}
307
308bool GlobalModuleIndex::loadedModuleFile(ModuleFile *File) {
309  // Look for the module in the global module index based on the module name.
310  StringRef Name = llvm::sys::path::stem(File->FileName);
311  llvm::StringMap<unsigned>::iterator Known = UnresolvedModules.find(Name);
312  if (Known == UnresolvedModules.end()) {
313    return true;
314  }
315
316  // Rectify this module with the global module index.
317  ModuleInfo &Info = Modules[Known->second];
318
319  //  If the size and modification time match what we expected, record this
320  // module file.
321  bool Failed = true;
322  if (File->File->getSize() == Info.Size &&
323      File->File->getModificationTime() == Info.ModTime) {
324    Info.File = File;
325    ModulesByFile[File] = Known->second;
326
327    Failed = false;
328  }
329
330  // One way or another, we have resolved this module file.
331  UnresolvedModules.erase(Known);
332  return Failed;
333}
334
335void GlobalModuleIndex::printStats() {
336  std::fprintf(stderr, "*** Global Module Index Statistics:\n");
337  if (NumIdentifierLookups) {
338    fprintf(stderr, "  %u / %u identifier lookups succeeded (%f%%)\n",
339            NumIdentifierLookupHits, NumIdentifierLookups,
340            (double)NumIdentifierLookupHits*100.0/NumIdentifierLookups);
341  }
342  std::fprintf(stderr, "\n");
343}
344
345//----------------------------------------------------------------------------//
346// Global module index writer.
347//----------------------------------------------------------------------------//
348
349namespace {
350  /// \brief Provides information about a specific module file.
351  struct ModuleFileInfo {
352    /// \brief The numberic ID for this module file.
353    unsigned ID;
354
355    /// \brief The set of modules on which this module depends. Each entry is
356    /// a module ID.
357    SmallVector<unsigned, 4> Dependencies;
358  };
359
360  /// \brief Builder that generates the global module index file.
361  class GlobalModuleIndexBuilder {
362    FileManager &FileMgr;
363
364    /// \brief Mapping from files to module file information.
365    typedef llvm::MapVector<const FileEntry *, ModuleFileInfo> ModuleFilesMap;
366
367    /// \brief Information about each of the known module files.
368    ModuleFilesMap ModuleFiles;
369
370    /// \brief Mapping from identifiers to the list of module file IDs that
371    /// consider this identifier to be interesting.
372    typedef llvm::StringMap<SmallVector<unsigned, 2> > InterestingIdentifierMap;
373
374    /// \brief A mapping from all interesting identifiers to the set of module
375    /// files in which those identifiers are considered interesting.
376    InterestingIdentifierMap InterestingIdentifiers;
377
378    /// \brief Write the block-info block for the global module index file.
379    void emitBlockInfoBlock(llvm::BitstreamWriter &Stream);
380
381    /// \brief Retrieve the module file information for the given file.
382    ModuleFileInfo &getModuleFileInfo(const FileEntry *File) {
383      llvm::MapVector<const FileEntry *, ModuleFileInfo>::iterator Known
384        = ModuleFiles.find(File);
385      if (Known != ModuleFiles.end())
386        return Known->second;
387
388      unsigned NewID = ModuleFiles.size();
389      ModuleFileInfo &Info = ModuleFiles[File];
390      Info.ID = NewID;
391      return Info;
392    }
393
394  public:
395    explicit GlobalModuleIndexBuilder(FileManager &FileMgr) : FileMgr(FileMgr){}
396
397    /// \brief Load the contents of the given module file into the builder.
398    ///
399    /// \returns true if an error occurred, false otherwise.
400    bool loadModuleFile(const FileEntry *File);
401
402    /// \brief Write the index to the given bitstream.
403    void writeIndex(llvm::BitstreamWriter &Stream);
404  };
405}
406
407static void emitBlockID(unsigned ID, const char *Name,
408                        llvm::BitstreamWriter &Stream,
409                        SmallVectorImpl<uint64_t> &Record) {
410  Record.clear();
411  Record.push_back(ID);
412  Stream.EmitRecord(llvm::bitc::BLOCKINFO_CODE_SETBID, Record);
413
414  // Emit the block name if present.
415  if (Name == 0 || Name[0] == 0) return;
416  Record.clear();
417  while (*Name)
418    Record.push_back(*Name++);
419  Stream.EmitRecord(llvm::bitc::BLOCKINFO_CODE_BLOCKNAME, Record);
420}
421
422static void emitRecordID(unsigned ID, const char *Name,
423                         llvm::BitstreamWriter &Stream,
424                         SmallVectorImpl<uint64_t> &Record) {
425  Record.clear();
426  Record.push_back(ID);
427  while (*Name)
428    Record.push_back(*Name++);
429  Stream.EmitRecord(llvm::bitc::BLOCKINFO_CODE_SETRECORDNAME, Record);
430}
431
432void
433GlobalModuleIndexBuilder::emitBlockInfoBlock(llvm::BitstreamWriter &Stream) {
434  SmallVector<uint64_t, 64> Record;
435  Stream.EnterSubblock(llvm::bitc::BLOCKINFO_BLOCK_ID, 3);
436
437#define BLOCK(X) emitBlockID(X ## _ID, #X, Stream, Record)
438#define RECORD(X) emitRecordID(X, #X, Stream, Record)
439  BLOCK(GLOBAL_INDEX_BLOCK);
440  RECORD(INDEX_METADATA);
441  RECORD(MODULE);
442  RECORD(IDENTIFIER_INDEX);
443#undef RECORD
444#undef BLOCK
445
446  Stream.ExitBlock();
447}
448
449namespace {
450  class InterestingASTIdentifierLookupTrait
451    : public serialization::reader::ASTIdentifierLookupTraitBase {
452
453  public:
454    /// \brief The identifier and whether it is "interesting".
455    typedef std::pair<StringRef, bool> data_type;
456
457    data_type ReadData(const internal_key_type& k,
458                       const unsigned char* d,
459                       unsigned DataLen) {
460      // The first bit indicates whether this identifier is interesting.
461      // That's all we care about.
462      using namespace llvm::support;
463      unsigned RawID = endian::readNext<uint32_t, little, unaligned>(d);
464      bool IsInteresting = RawID & 0x01;
465      return std::make_pair(k, IsInteresting);
466    }
467  };
468}
469
470bool GlobalModuleIndexBuilder::loadModuleFile(const FileEntry *File) {
471  // Open the module file.
472  std::unique_ptr<llvm::MemoryBuffer> Buffer;
473  std::string ErrorStr;
474  Buffer.reset(FileMgr.getBufferForFile(File, &ErrorStr, /*isVolatile=*/true));
475  if (!Buffer) {
476    return true;
477  }
478
479  // Initialize the input stream
480  llvm::BitstreamReader InStreamFile;
481  llvm::BitstreamCursor InStream;
482  InStreamFile.init((const unsigned char *)Buffer->getBufferStart(),
483                  (const unsigned char *)Buffer->getBufferEnd());
484  InStream.init(InStreamFile);
485
486  // Sniff for the signature.
487  if (InStream.Read(8) != 'C' ||
488      InStream.Read(8) != 'P' ||
489      InStream.Read(8) != 'C' ||
490      InStream.Read(8) != 'H') {
491    return true;
492  }
493
494  // Record this module file and assign it a unique ID (if it doesn't have
495  // one already).
496  unsigned ID = getModuleFileInfo(File).ID;
497
498  // Search for the blocks and records we care about.
499  enum { Other, ControlBlock, ASTBlock } State = Other;
500  bool Done = false;
501  while (!Done) {
502    llvm::BitstreamEntry Entry = InStream.advance();
503    switch (Entry.Kind) {
504    case llvm::BitstreamEntry::Error:
505      Done = true;
506      continue;
507
508    case llvm::BitstreamEntry::Record:
509      // In the 'other' state, just skip the record. We don't care.
510      if (State == Other) {
511        InStream.skipRecord(Entry.ID);
512        continue;
513      }
514
515      // Handle potentially-interesting records below.
516      break;
517
518    case llvm::BitstreamEntry::SubBlock:
519      if (Entry.ID == CONTROL_BLOCK_ID) {
520        if (InStream.EnterSubBlock(CONTROL_BLOCK_ID))
521          return true;
522
523        // Found the control block.
524        State = ControlBlock;
525        continue;
526      }
527
528      if (Entry.ID == AST_BLOCK_ID) {
529        if (InStream.EnterSubBlock(AST_BLOCK_ID))
530          return true;
531
532        // Found the AST block.
533        State = ASTBlock;
534        continue;
535      }
536
537      if (InStream.SkipBlock())
538        return true;
539
540      continue;
541
542    case llvm::BitstreamEntry::EndBlock:
543      State = Other;
544      continue;
545    }
546
547    // Read the given record.
548    SmallVector<uint64_t, 64> Record;
549    StringRef Blob;
550    unsigned Code = InStream.readRecord(Entry.ID, Record, &Blob);
551
552    // Handle module dependencies.
553    if (State == ControlBlock && Code == IMPORTS) {
554      // Load each of the imported PCH files.
555      unsigned Idx = 0, N = Record.size();
556      while (Idx < N) {
557        // Read information about the AST file.
558
559        // Skip the imported kind
560        ++Idx;
561
562        // Skip the import location
563        ++Idx;
564
565        // Load stored size/modification time.
566        off_t StoredSize = (off_t)Record[Idx++];
567        time_t StoredModTime = (time_t)Record[Idx++];
568
569        // Retrieve the imported file name.
570        unsigned Length = Record[Idx++];
571        SmallString<128> ImportedFile(Record.begin() + Idx,
572                                      Record.begin() + Idx + Length);
573        Idx += Length;
574
575        // Find the imported module file.
576        const FileEntry *DependsOnFile
577          = FileMgr.getFile(ImportedFile, /*openFile=*/false,
578                            /*cacheFailure=*/false);
579        if (!DependsOnFile ||
580            (StoredSize != DependsOnFile->getSize()) ||
581            (StoredModTime != DependsOnFile->getModificationTime()))
582          return true;
583
584        // Record the dependency.
585        unsigned DependsOnID = getModuleFileInfo(DependsOnFile).ID;
586        getModuleFileInfo(File).Dependencies.push_back(DependsOnID);
587      }
588
589      continue;
590    }
591
592    // Handle the identifier table
593    if (State == ASTBlock && Code == IDENTIFIER_TABLE && Record[0] > 0) {
594      typedef OnDiskChainedHashTable<InterestingASTIdentifierLookupTrait>
595        InterestingIdentifierTable;
596      std::unique_ptr<InterestingIdentifierTable> Table(
597          InterestingIdentifierTable::Create(
598              (const unsigned char *)Blob.data() + Record[0],
599              (const unsigned char *)Blob.data()));
600      for (InterestingIdentifierTable::data_iterator D = Table->data_begin(),
601                                                     DEnd = Table->data_end();
602           D != DEnd; ++D) {
603        std::pair<StringRef, bool> Ident = *D;
604        if (Ident.second)
605          InterestingIdentifiers[Ident.first].push_back(ID);
606        else
607          (void)InterestingIdentifiers[Ident.first];
608      }
609    }
610
611    // We don't care about this record.
612  }
613
614  return false;
615}
616
617namespace {
618
619/// \brief Trait used to generate the identifier index as an on-disk hash
620/// table.
621class IdentifierIndexWriterTrait {
622public:
623  typedef StringRef key_type;
624  typedef StringRef key_type_ref;
625  typedef SmallVector<unsigned, 2> data_type;
626  typedef const SmallVector<unsigned, 2> &data_type_ref;
627
628  static unsigned ComputeHash(key_type_ref Key) {
629    return llvm::HashString(Key);
630  }
631
632  std::pair<unsigned,unsigned>
633  EmitKeyDataLength(raw_ostream& Out, key_type_ref Key, data_type_ref Data) {
634    using namespace llvm::support;
635    endian::Writer<little> LE(Out);
636    unsigned KeyLen = Key.size();
637    unsigned DataLen = Data.size() * 4;
638    LE.write<uint16_t>(KeyLen);
639    LE.write<uint16_t>(DataLen);
640    return std::make_pair(KeyLen, DataLen);
641  }
642
643  void EmitKey(raw_ostream& Out, key_type_ref Key, unsigned KeyLen) {
644    Out.write(Key.data(), KeyLen);
645  }
646
647  void EmitData(raw_ostream& Out, key_type_ref Key, data_type_ref Data,
648                unsigned DataLen) {
649    using namespace llvm::support;
650    for (unsigned I = 0, N = Data.size(); I != N; ++I)
651      endian::Writer<little>(Out).write<uint32_t>(Data[I]);
652  }
653};
654
655}
656
657void GlobalModuleIndexBuilder::writeIndex(llvm::BitstreamWriter &Stream) {
658  using namespace llvm;
659
660  // Emit the file header.
661  Stream.Emit((unsigned)'B', 8);
662  Stream.Emit((unsigned)'C', 8);
663  Stream.Emit((unsigned)'G', 8);
664  Stream.Emit((unsigned)'I', 8);
665
666  // Write the block-info block, which describes the records in this bitcode
667  // file.
668  emitBlockInfoBlock(Stream);
669
670  Stream.EnterSubblock(GLOBAL_INDEX_BLOCK_ID, 3);
671
672  // Write the metadata.
673  SmallVector<uint64_t, 2> Record;
674  Record.push_back(CurrentVersion);
675  Stream.EmitRecord(INDEX_METADATA, Record);
676
677  // Write the set of known module files.
678  for (ModuleFilesMap::iterator M = ModuleFiles.begin(),
679                                MEnd = ModuleFiles.end();
680       M != MEnd; ++M) {
681    Record.clear();
682    Record.push_back(M->second.ID);
683    Record.push_back(M->first->getSize());
684    Record.push_back(M->first->getModificationTime());
685
686    // File name
687    StringRef Name(M->first->getName());
688    Record.push_back(Name.size());
689    Record.append(Name.begin(), Name.end());
690
691    // Dependencies
692    Record.push_back(M->second.Dependencies.size());
693    Record.append(M->second.Dependencies.begin(), M->second.Dependencies.end());
694    Stream.EmitRecord(MODULE, Record);
695  }
696
697  // Write the identifier -> module file mapping.
698  {
699    OnDiskChainedHashTableGenerator<IdentifierIndexWriterTrait> Generator;
700    IdentifierIndexWriterTrait Trait;
701
702    // Populate the hash table.
703    for (InterestingIdentifierMap::iterator I = InterestingIdentifiers.begin(),
704                                            IEnd = InterestingIdentifiers.end();
705         I != IEnd; ++I) {
706      Generator.insert(I->first(), I->second, Trait);
707    }
708
709    // Create the on-disk hash table in a buffer.
710    SmallString<4096> IdentifierTable;
711    uint32_t BucketOffset;
712    {
713      using namespace llvm::support;
714      llvm::raw_svector_ostream Out(IdentifierTable);
715      // Make sure that no bucket is at offset 0
716      endian::Writer<little>(Out).write<uint32_t>(0);
717      BucketOffset = Generator.Emit(Out, Trait);
718    }
719
720    // Create a blob abbreviation
721    BitCodeAbbrev *Abbrev = new BitCodeAbbrev();
722    Abbrev->Add(BitCodeAbbrevOp(IDENTIFIER_INDEX));
723    Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
724    Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Blob));
725    unsigned IDTableAbbrev = Stream.EmitAbbrev(Abbrev);
726
727    // Write the identifier table
728    Record.clear();
729    Record.push_back(IDENTIFIER_INDEX);
730    Record.push_back(BucketOffset);
731    Stream.EmitRecordWithBlob(IDTableAbbrev, Record, IdentifierTable.str());
732  }
733
734  Stream.ExitBlock();
735}
736
737GlobalModuleIndex::ErrorCode
738GlobalModuleIndex::writeIndex(FileManager &FileMgr, StringRef Path) {
739  llvm::SmallString<128> IndexPath;
740  IndexPath += Path;
741  llvm::sys::path::append(IndexPath, IndexFileName);
742
743  // Coordinate building the global index file with other processes that might
744  // try to do the same.
745  llvm::LockFileManager Locked(IndexPath);
746  switch (Locked) {
747  case llvm::LockFileManager::LFS_Error:
748    return EC_IOError;
749
750  case llvm::LockFileManager::LFS_Owned:
751    // We're responsible for building the index ourselves. Do so below.
752    break;
753
754  case llvm::LockFileManager::LFS_Shared:
755    // Someone else is responsible for building the index. We don't care
756    // when they finish, so we're done.
757    return EC_Building;
758  }
759
760  // The module index builder.
761  GlobalModuleIndexBuilder Builder(FileMgr);
762
763  // Load each of the module files.
764  llvm::error_code EC;
765  for (llvm::sys::fs::directory_iterator D(Path, EC), DEnd;
766       D != DEnd && !EC;
767       D.increment(EC)) {
768    // If this isn't a module file, we don't care.
769    if (llvm::sys::path::extension(D->path()) != ".pcm") {
770      // ... unless it's a .pcm.lock file, which indicates that someone is
771      // in the process of rebuilding a module. They'll rebuild the index
772      // at the end of that translation unit, so we don't have to.
773      if (llvm::sys::path::extension(D->path()) == ".pcm.lock")
774        return EC_Building;
775
776      continue;
777    }
778
779    // If we can't find the module file, skip it.
780    const FileEntry *ModuleFile = FileMgr.getFile(D->path());
781    if (!ModuleFile)
782      continue;
783
784    // Load this module file.
785    if (Builder.loadModuleFile(ModuleFile))
786      return EC_IOError;
787  }
788
789  // The output buffer, into which the global index will be written.
790  SmallVector<char, 16> OutputBuffer;
791  {
792    llvm::BitstreamWriter OutputStream(OutputBuffer);
793    Builder.writeIndex(OutputStream);
794  }
795
796  // Write the global index file to a temporary file.
797  llvm::SmallString<128> IndexTmpPath;
798  int TmpFD;
799  if (llvm::sys::fs::createUniqueFile(IndexPath + "-%%%%%%%%", TmpFD,
800                                      IndexTmpPath))
801    return EC_IOError;
802
803  // Open the temporary global index file for output.
804  llvm::raw_fd_ostream Out(TmpFD, true);
805  if (Out.has_error())
806    return EC_IOError;
807
808  // Write the index.
809  Out.write(OutputBuffer.data(), OutputBuffer.size());
810  Out.close();
811  if (Out.has_error())
812    return EC_IOError;
813
814  // Remove the old index file. It isn't relevant any more.
815  llvm::sys::fs::remove(IndexPath.str());
816
817  // Rename the newly-written index file to the proper name.
818  if (llvm::sys::fs::rename(IndexTmpPath.str(), IndexPath.str())) {
819    // Rename failed; just remove the
820    llvm::sys::fs::remove(IndexTmpPath.str());
821    return EC_IOError;
822  }
823
824  // We're done.
825  return EC_None;
826}
827
828namespace {
829  class GlobalIndexIdentifierIterator : public IdentifierIterator {
830    /// \brief The current position within the identifier lookup table.
831    IdentifierIndexTable::key_iterator Current;
832
833    /// \brief The end position within the identifier lookup table.
834    IdentifierIndexTable::key_iterator End;
835
836  public:
837    explicit GlobalIndexIdentifierIterator(IdentifierIndexTable &Idx) {
838      Current = Idx.key_begin();
839      End = Idx.key_end();
840    }
841
842    StringRef Next() override {
843      if (Current == End)
844        return StringRef();
845
846      StringRef Result = *Current;
847      ++Current;
848      return Result;
849    }
850  };
851}
852
853IdentifierIterator *GlobalModuleIndex::createIdentifierIterator() const {
854  IdentifierIndexTable &Table =
855    *static_cast<IdentifierIndexTable *>(IdentifierIndex);
856  return new GlobalIndexIdentifierIterator(Table);
857}
858