FileManager.cpp revision 10e286aa8d39fb51a21412850265d9dae74613ee 
1//===--- FileManager.cpp - File System Probing and Caching ----------------===//
2//
3//                     The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10//  This file implements the FileManager interface.
11//
12//===----------------------------------------------------------------------===//
13//
14// TODO: This should index all interesting directories with dirent calls.
15//  getdirentries ?
16//  opendir/readdir_r/closedir ?
17//
18//===----------------------------------------------------------------------===//
19
20#include "clang/Basic/FileManager.h"
21#include "clang/Basic/FileSystemStatCache.h"
22#include "llvm/ADT/SmallString.h"
23#include "llvm/ADT/StringExtras.h"
24#include "llvm/Support/MemoryBuffer.h"
25#include "llvm/Support/raw_ostream.h"
26#include "llvm/System/Path.h"
27#include "llvm/Config/config.h"
28#include <map>
29#include <set>
30#include <string>
31using namespace clang;
32
33// FIXME: Enhance libsystem to support inode and other fields.
34#include <sys/stat.h>
35
36#if defined(_MSC_VER)
37#define S_ISDIR(s) (_S_IFDIR & s)
38#endif
39
40/// NON_EXISTENT_DIR - A special value distinct from null that is used to
41/// represent a dir name that doesn't exist on the disk.
42#define NON_EXISTENT_DIR reinterpret_cast<DirectoryEntry*>((intptr_t)-1)
43
44//===----------------------------------------------------------------------===//
45// Windows.
46//===----------------------------------------------------------------------===//
47
48#ifdef LLVM_ON_WIN32
49
50#define IS_DIR_SEPARATOR_CHAR(x) ((x) == '/' || (x) == '\\')
51
52namespace {
53  static std::string GetFullPath(const char *relPath) {
54    char *absPathStrPtr = _fullpath(NULL, relPath, 0);
55    assert(absPathStrPtr && "_fullpath() returned NULL!");
56
57    std::string absPath(absPathStrPtr);
58
59    free(absPathStrPtr);
60    return absPath;
61  }
62}
63
64class FileManager::UniqueDirContainer {
65  /// UniqueDirs - Cache from full path to existing directories/files.
66  ///
67  llvm::StringMap<DirectoryEntry> UniqueDirs;
68
69public:
70  DirectoryEntry &getDirectory(const char *Name, struct stat &StatBuf) {
71    std::string FullPath(GetFullPath(Name));
72    return UniqueDirs.GetOrCreateValue(
73                              FullPath.c_str(),
74                              FullPath.c_str() + FullPath.size()
75                                                                ).getValue();
76  }
77
78  size_t size() { return UniqueDirs.size(); }
79};
80
81class FileManager::UniqueFileContainer {
82  /// UniqueFiles - Cache from full path to existing directories/files.
83  ///
84  llvm::StringMap<FileEntry, llvm::BumpPtrAllocator> UniqueFiles;
85
86public:
87  FileEntry &getFile(const char *Name, struct stat &StatBuf) {
88    std::string FullPath(GetFullPath(Name));
89
90    // LowercaseString because Windows filesystem is case insensitive.
91    FullPath = llvm::LowercaseString(FullPath);
92    return UniqueFiles.GetOrCreateValue(
93                               FullPath.c_str(),
94                               FullPath.c_str() + FullPath.size()
95                                                                 ).getValue();
96  }
97
98  size_t size() { return UniqueFiles.size(); }
99};
100
101//===----------------------------------------------------------------------===//
102// Unix-like Systems.
103//===----------------------------------------------------------------------===//
104
105#else
106
107#define IS_DIR_SEPARATOR_CHAR(x) ((x) == '/')
108
109class FileManager::UniqueDirContainer {
110  /// UniqueDirs - Cache from ID's to existing directories/files.
111  ///
112  std::map<std::pair<dev_t, ino_t>, DirectoryEntry> UniqueDirs;
113
114public:
115  DirectoryEntry &getDirectory(const char *Name, struct stat &StatBuf) {
116    return UniqueDirs[std::make_pair(StatBuf.st_dev, StatBuf.st_ino)];
117  }
118
119  size_t size() { return UniqueDirs.size(); }
120};
121
122class FileManager::UniqueFileContainer {
123  /// UniqueFiles - Cache from ID's to existing directories/files.
124  ///
125  std::set<FileEntry> UniqueFiles;
126
127public:
128  FileEntry &getFile(const char *Name, struct stat &StatBuf) {
129    return
130      const_cast<FileEntry&>(
131                    *UniqueFiles.insert(FileEntry(StatBuf.st_dev,
132                                                  StatBuf.st_ino,
133                                                  StatBuf.st_mode)).first);
134  }
135
136  size_t size() { return UniqueFiles.size(); }
137};
138
139#endif
140
141//===----------------------------------------------------------------------===//
142// Common logic.
143//===----------------------------------------------------------------------===//
144
145FileManager::FileManager(const FileSystemOptions &FSO)
146  : FileSystemOpts(FSO),
147    UniqueDirs(*new UniqueDirContainer),
148    UniqueFiles(*new UniqueFileContainer),
149    DirEntries(64), FileEntries(64), NextFileUID(0) {
150  NumDirLookups = NumFileLookups = 0;
151  NumDirCacheMisses = NumFileCacheMisses = 0;
152}
153
154FileManager::~FileManager() {
155  delete &UniqueDirs;
156  delete &UniqueFiles;
157  for (llvm::SmallVectorImpl<FileEntry *>::iterator
158         V = VirtualFileEntries.begin(),
159         VEnd = VirtualFileEntries.end();
160       V != VEnd;
161       ++V)
162    delete *V;
163}
164
165void FileManager::addStatCache(FileSystemStatCache *statCache,
166                               bool AtBeginning) {
167  assert(statCache && "No stat cache provided?");
168  if (AtBeginning || StatCache.get() == 0) {
169    statCache->setNextStatCache(StatCache.take());
170    StatCache.reset(statCache);
171    return;
172  }
173
174  FileSystemStatCache *LastCache = StatCache.get();
175  while (LastCache->getNextStatCache())
176    LastCache = LastCache->getNextStatCache();
177
178  LastCache->setNextStatCache(statCache);
179}
180
181void FileManager::removeStatCache(FileSystemStatCache *statCache) {
182  if (!statCache)
183    return;
184
185  if (StatCache.get() == statCache) {
186    // This is the first stat cache.
187    StatCache.reset(StatCache->takeNextStatCache());
188    return;
189  }
190
191  // Find the stat cache in the list.
192  FileSystemStatCache *PrevCache = StatCache.get();
193  while (PrevCache && PrevCache->getNextStatCache() != statCache)
194    PrevCache = PrevCache->getNextStatCache();
195  if (PrevCache)
196    PrevCache->setNextStatCache(statCache->getNextStatCache());
197  else
198    assert(false && "Stat cache not found for removal");
199}
200
201/// \brief Retrieve the directory that the given file name resides in.
202static const DirectoryEntry *getDirectoryFromFile(FileManager &FileMgr,
203                                                  llvm::StringRef Filename) {
204  // Figure out what directory it is in.   If the string contains a / in it,
205  // strip off everything after it.
206  // FIXME: this logic should be in sys::Path.
207  size_t SlashPos = Filename.size();
208  while (SlashPos != 0 && !IS_DIR_SEPARATOR_CHAR(Filename[SlashPos-1]))
209    --SlashPos;
210
211  // Use the current directory if file has no path component.
212  if (SlashPos == 0)
213    return FileMgr.getDirectory(".");
214
215  if (SlashPos == Filename.size()-1)
216    return 0;       // If filename ends with a /, it's a directory.
217
218  // Ignore repeated //'s.
219  while (SlashPos != 0 && IS_DIR_SEPARATOR_CHAR(Filename[SlashPos-1]))
220    --SlashPos;
221
222  return FileMgr.getDirectory(Filename.substr(0, SlashPos));
223}
224
225/// getDirectory - Lookup, cache, and verify the specified directory.  This
226/// returns null if the directory doesn't exist.
227///
228const DirectoryEntry *FileManager::getDirectory(llvm::StringRef Filename) {
229  // stat doesn't like trailing separators (at least on Windows).
230  if (Filename.size() > 1 && IS_DIR_SEPARATOR_CHAR(Filename.back()))
231    Filename = Filename.substr(0, Filename.size()-1);
232
233  ++NumDirLookups;
234  llvm::StringMapEntry<DirectoryEntry *> &NamedDirEnt =
235    DirEntries.GetOrCreateValue(Filename);
236
237  // See if there is already an entry in the map.
238  if (NamedDirEnt.getValue())
239    return NamedDirEnt.getValue() == NON_EXISTENT_DIR
240              ? 0 : NamedDirEnt.getValue();
241
242  ++NumDirCacheMisses;
243
244  // By default, initialize it to invalid.
245  NamedDirEnt.setValue(NON_EXISTENT_DIR);
246
247  // Get the null-terminated directory name as stored as the key of the
248  // DirEntries map.
249  const char *InterndDirName = NamedDirEnt.getKeyData();
250
251  // Check to see if the directory exists.
252  struct stat StatBuf;
253  if (getStatValue(InterndDirName, StatBuf) ||    // Error stat'ing.
254      !S_ISDIR(StatBuf.st_mode))                  // Not a directory?
255    return 0;
256
257  // It exists.  See if we have already opened a directory with the same inode.
258  // This occurs when one dir is symlinked to another, for example.
259  DirectoryEntry &UDE = UniqueDirs.getDirectory(InterndDirName, StatBuf);
260
261  NamedDirEnt.setValue(&UDE);
262  if (UDE.getName()) // Already have an entry with this inode, return it.
263    return &UDE;
264
265  // Otherwise, we don't have this directory yet, add it.  We use the string
266  // key from the DirEntries map as the string.
267  UDE.Name  = InterndDirName;
268  return &UDE;
269}
270
271/// NON_EXISTENT_FILE - A special value distinct from null that is used to
272/// represent a filename that doesn't exist on the disk.
273#define NON_EXISTENT_FILE reinterpret_cast<FileEntry*>((intptr_t)-1)
274
275/// getFile - Lookup, cache, and verify the specified file.  This returns null
276/// if the file doesn't exist.
277///
278const FileEntry *FileManager::getFile(llvm::StringRef Filename) {
279  ++NumFileLookups;
280
281  // See if there is already an entry in the map.
282  llvm::StringMapEntry<FileEntry *> &NamedFileEnt =
283    FileEntries.GetOrCreateValue(Filename);
284
285  // See if there is already an entry in the map.
286  if (NamedFileEnt.getValue())
287    return NamedFileEnt.getValue() == NON_EXISTENT_FILE
288                 ? 0 : NamedFileEnt.getValue();
289
290  ++NumFileCacheMisses;
291
292  // By default, initialize it to invalid.
293  NamedFileEnt.setValue(NON_EXISTENT_FILE);
294
295
296  // Get the null-terminated file name as stored as the key of the
297  // FileEntries map.
298  const char *InterndFileName = NamedFileEnt.getKeyData();
299
300  const DirectoryEntry *DirInfo = getDirectoryFromFile(*this, Filename);
301  if (DirInfo == 0)  // Directory doesn't exist, file can't exist.
302    return 0;
303
304  // FIXME: Use the directory info to prune this, before doing the stat syscall.
305  // FIXME: This will reduce the # syscalls.
306
307  // Nope, there isn't.  Check to see if the file exists.
308  struct stat StatBuf;
309  //llvm::errs() << "STATING: " << Filename;
310  if (getStatValue(InterndFileName, StatBuf) ||    // Error stat'ing.
311      S_ISDIR(StatBuf.st_mode)) {                  // A directory?
312    // If this file doesn't exist, we leave a null in FileEntries for this path.
313    //llvm::errs() << ": Not existing\n";
314    return 0;
315  }
316  //llvm::errs() << ": exists\n";
317
318  // It exists.  See if we have already opened a file with the same inode.
319  // This occurs when one dir is symlinked to another, for example.
320  FileEntry &UFE = UniqueFiles.getFile(InterndFileName, StatBuf);
321
322  NamedFileEnt.setValue(&UFE);
323  if (UFE.getName())  // Already have an entry with this inode, return it.
324    return &UFE;
325
326  // Otherwise, we don't have this directory yet, add it.
327  // FIXME: Change the name to be a char* that points back to the 'FileEntries'
328  // key.
329  UFE.Name    = InterndFileName;
330  UFE.Size    = StatBuf.st_size;
331  UFE.ModTime = StatBuf.st_mtime;
332  UFE.Dir     = DirInfo;
333  UFE.UID     = NextFileUID++;
334  return &UFE;
335}
336
337const FileEntry *
338FileManager::getVirtualFile(llvm::StringRef Filename, off_t Size,
339                            time_t ModificationTime) {
340  ++NumFileLookups;
341
342  // See if there is already an entry in the map.
343  llvm::StringMapEntry<FileEntry *> &NamedFileEnt =
344    FileEntries.GetOrCreateValue(Filename);
345
346  // See if there is already an entry in the map.
347  if (NamedFileEnt.getValue())
348    return NamedFileEnt.getValue() == NON_EXISTENT_FILE
349                 ? 0 : NamedFileEnt.getValue();
350
351  ++NumFileCacheMisses;
352
353  // By default, initialize it to invalid.
354  NamedFileEnt.setValue(NON_EXISTENT_FILE);
355
356  const DirectoryEntry *DirInfo = getDirectoryFromFile(*this, Filename);
357  if (DirInfo == 0)  // Directory doesn't exist, file can't exist.
358    return 0;
359
360  FileEntry *UFE = new FileEntry();
361  VirtualFileEntries.push_back(UFE);
362  NamedFileEnt.setValue(UFE);
363
364  UFE->Name    = NamedFileEnt.getKeyData();
365  UFE->Size    = Size;
366  UFE->ModTime = ModificationTime;
367  UFE->Dir     = DirInfo;
368  UFE->UID     = NextFileUID++;
369
370  // If this virtual file resolves to a file, also map that file to the
371  // newly-created file entry.
372  const char *InterndFileName = NamedFileEnt.getKeyData();
373  struct stat StatBuf;
374  if (!getStatValue(InterndFileName, StatBuf) &&
375      !S_ISDIR(StatBuf.st_mode)) {
376    llvm::sys::Path FilePath(InterndFileName);
377    FilePath.makeAbsolute();
378    FileEntries[FilePath.str()] = UFE;
379  }
380
381  return UFE;
382}
383
384void FileManager::FixupRelativePath(llvm::sys::Path &path,
385                                    const FileSystemOptions &FSOpts) {
386  if (FSOpts.WorkingDir.empty() || path.isAbsolute()) return;
387
388  llvm::sys::Path NewPath(FSOpts.WorkingDir);
389  NewPath.appendComponent(path.str());
390  path = NewPath;
391}
392
393llvm::MemoryBuffer *FileManager::
394getBufferForFile(const FileEntry *Entry, std::string *ErrorStr) {
395  llvm::StringRef Filename = Entry->getName();
396  if (FileSystemOpts.WorkingDir.empty())
397    return llvm::MemoryBuffer::getFile(Filename, ErrorStr);
398
399  llvm::sys::Path FilePath(Filename);
400  FixupRelativePath(FilePath, FileSystemOpts);
401  return llvm::MemoryBuffer::getFile(FilePath.c_str(), ErrorStr);
402}
403
404llvm::MemoryBuffer *FileManager::
405getBufferForFile(llvm::StringRef Filename, std::string *ErrorStr) {
406  if (FileSystemOpts.WorkingDir.empty())
407    return llvm::MemoryBuffer::getFile(Filename, ErrorStr);
408
409  llvm::sys::Path FilePath(Filename);
410  FixupRelativePath(FilePath, FileSystemOpts);
411  return llvm::MemoryBuffer::getFile(FilePath.c_str(), ErrorStr);
412}
413
414/// getStatValue - Get the 'stat' information for the specified path, using the
415/// cache to accellerate it if possible.  This returns true if the path does not
416/// exist or false if it exists.
417bool FileManager::getStatValue(const char *Path, struct stat &StatBuf) {
418  FileSystemStatCache::LookupResult Result = FileSystemStatCache::CacheMiss;
419
420  // FIXME: FileSystemOpts shouldn't be passed in here, all paths should be
421  // absolute!
422  if (FileSystemOpts.WorkingDir.empty()) {
423    if (StatCache.get())
424      Result = StatCache->getStat(Path, StatBuf);
425
426    if (Result == FileSystemStatCache::CacheMiss)
427      return ::stat(Path, &StatBuf);
428    return Result == FileSystemStatCache::CacheHitMissing;
429  }
430
431  llvm::sys::Path FilePath(Path);
432  FixupRelativePath(FilePath, FileSystemOpts);
433
434  if (StatCache.get())
435    Result = StatCache->getStat(FilePath.c_str(), StatBuf);
436
437  if (Result == FileSystemStatCache::CacheMiss)
438    return ::stat(FilePath.c_str(), &StatBuf);
439  return Result == FileSystemStatCache::CacheHitMissing;
440}
441
442
443
444void FileManager::PrintStats() const {
445  llvm::errs() << "\n*** File Manager Stats:\n";
446  llvm::errs() << UniqueFiles.size() << " files found, "
447               << UniqueDirs.size() << " dirs found.\n";
448  llvm::errs() << NumDirLookups << " dir lookups, "
449               << NumDirCacheMisses << " dir cache misses.\n";
450  llvm::errs() << NumFileLookups << " file lookups, "
451               << NumFileCacheMisses << " file cache misses.\n";
452
453  //llvm::errs() << PagesMapped << BytesOfPagesMapped << FSLookups;
454}
455
456