FileManager.cpp revision 2e2468e2d2ccbb9a38fe3b6b754009af7e5d39a9
1//===--- FileManager.cpp - File System Probing and Caching ----------------===// 2// 3// The LLVM Compiler Infrastructure 4// 5// This file is distributed under the University of Illinois Open Source 6// License. See LICENSE.TXT for details. 7// 8//===----------------------------------------------------------------------===// 9// 10// This file implements the FileManager interface. 11// 12//===----------------------------------------------------------------------===// 13// 14// TODO: This should index all interesting directories with dirent calls. 15// getdirentries ? 16// opendir/readdir_r/closedir ? 17// 18//===----------------------------------------------------------------------===// 19 20#include "clang/Basic/FileManager.h" 21#include "clang/Basic/FileSystemStatCache.h" 22#include "llvm/ADT/SmallString.h" 23#include "llvm/ADT/StringExtras.h" 24#include "llvm/Support/FileSystem.h" 25#include "llvm/Support/MemoryBuffer.h" 26#include "llvm/Support/raw_ostream.h" 27#include "llvm/Support/Path.h" 28#include "llvm/Support/system_error.h" 29#include "llvm/Config/config.h" 30#include <map> 31#include <set> 32#include <string> 33 34// FIXME: This is terrible, we need this for ::close. 35#if !defined(_MSC_VER) && !defined(__MINGW32__) 36#include <unistd.h> 37#include <sys/uio.h> 38#else 39#include <io.h> 40#endif 41using namespace clang; 42 43// FIXME: Enhance libsystem to support inode and other fields. 44#include <sys/stat.h> 45 46/// NON_EXISTENT_DIR - A special value distinct from null that is used to 47/// represent a dir name that doesn't exist on the disk. 48#define NON_EXISTENT_DIR reinterpret_cast<DirectoryEntry*>((intptr_t)-1) 49 50/// NON_EXISTENT_FILE - A special value distinct from null that is used to 51/// represent a filename that doesn't exist on the disk. 52#define NON_EXISTENT_FILE reinterpret_cast<FileEntry*>((intptr_t)-1) 53 54 55FileEntry::~FileEntry() { 56 // If this FileEntry owns an open file descriptor that never got used, close 57 // it. 58 if (FD != -1) ::close(FD); 59} 60 61//===----------------------------------------------------------------------===// 62// Windows. 63//===----------------------------------------------------------------------===// 64 65#ifdef LLVM_ON_WIN32 66 67namespace { 68 static std::string GetFullPath(const char *relPath) { 69 char *absPathStrPtr = _fullpath(NULL, relPath, 0); 70 assert(absPathStrPtr && "_fullpath() returned NULL!"); 71 72 std::string absPath(absPathStrPtr); 73 74 free(absPathStrPtr); 75 return absPath; 76 } 77} 78 79class FileManager::UniqueDirContainer { 80 /// UniqueDirs - Cache from full path to existing directories/files. 81 /// 82 llvm::StringMap<DirectoryEntry> UniqueDirs; 83 84public: 85 /// getDirectory - Return an existing DirectoryEntry with the given 86 /// name if there is already one; otherwise create and return a 87 /// default-constructed DirectoryEntry. 88 DirectoryEntry &getDirectory(const char *Name, 89 const struct stat & /*StatBuf*/) { 90 std::string FullPath(GetFullPath(Name)); 91 return UniqueDirs.GetOrCreateValue(FullPath).getValue(); 92 } 93 94 size_t size() const { return UniqueDirs.size(); } 95}; 96 97class FileManager::UniqueFileContainer { 98 /// UniqueFiles - Cache from full path to existing directories/files. 99 /// 100 llvm::StringMap<FileEntry, llvm::BumpPtrAllocator> UniqueFiles; 101 102public: 103 /// getFile - Return an existing FileEntry with the given name if 104 /// there is already one; otherwise create and return a 105 /// default-constructed FileEntry. 106 FileEntry &getFile(const char *Name, const struct stat & /*StatBuf*/) { 107 std::string FullPath(GetFullPath(Name)); 108 109 // LowercaseString because Windows filesystem is case insensitive. 110 FullPath = llvm::LowercaseString(FullPath); 111 return UniqueFiles.GetOrCreateValue(FullPath).getValue(); 112 } 113 114 size_t size() const { return UniqueFiles.size(); } 115}; 116 117//===----------------------------------------------------------------------===// 118// Unix-like Systems. 119//===----------------------------------------------------------------------===// 120 121#else 122 123class FileManager::UniqueDirContainer { 124 /// UniqueDirs - Cache from ID's to existing directories/files. 125 std::map<std::pair<dev_t, ino_t>, DirectoryEntry> UniqueDirs; 126 127public: 128 /// getDirectory - Return an existing DirectoryEntry with the given 129 /// ID's if there is already one; otherwise create and return a 130 /// default-constructed DirectoryEntry. 131 DirectoryEntry &getDirectory(const char * /*Name*/, 132 const struct stat &StatBuf) { 133 return UniqueDirs[std::make_pair(StatBuf.st_dev, StatBuf.st_ino)]; 134 } 135 136 size_t size() const { return UniqueDirs.size(); } 137}; 138 139class FileManager::UniqueFileContainer { 140 /// UniqueFiles - Cache from ID's to existing directories/files. 141 std::set<FileEntry> UniqueFiles; 142 143public: 144 /// getFile - Return an existing FileEntry with the given ID's if 145 /// there is already one; otherwise create and return a 146 /// default-constructed FileEntry. 147 FileEntry &getFile(const char * /*Name*/, const struct stat &StatBuf) { 148 return 149 const_cast<FileEntry&>( 150 *UniqueFiles.insert(FileEntry(StatBuf.st_dev, 151 StatBuf.st_ino, 152 StatBuf.st_mode)).first); 153 } 154 155 size_t size() const { return UniqueFiles.size(); } 156}; 157 158#endif 159 160//===----------------------------------------------------------------------===// 161// Common logic. 162//===----------------------------------------------------------------------===// 163 164FileManager::FileManager(const FileSystemOptions &FSO) 165 : FileSystemOpts(FSO), 166 UniqueRealDirs(*new UniqueDirContainer()), 167 UniqueRealFiles(*new UniqueFileContainer()), 168 SeenDirEntries(64), SeenFileEntries(64), NextFileUID(0) { 169 NumDirLookups = NumFileLookups = 0; 170 NumDirCacheMisses = NumFileCacheMisses = 0; 171} 172 173FileManager::~FileManager() { 174 delete &UniqueRealDirs; 175 delete &UniqueRealFiles; 176 for (unsigned i = 0, e = VirtualFileEntries.size(); i != e; ++i) 177 delete VirtualFileEntries[i]; 178 for (unsigned i = 0, e = VirtualDirectoryEntries.size(); i != e; ++i) 179 delete VirtualDirectoryEntries[i]; 180} 181 182void FileManager::addStatCache(FileSystemStatCache *statCache, 183 bool AtBeginning) { 184 assert(statCache && "No stat cache provided?"); 185 if (AtBeginning || StatCache.get() == 0) { 186 statCache->setNextStatCache(StatCache.take()); 187 StatCache.reset(statCache); 188 return; 189 } 190 191 FileSystemStatCache *LastCache = StatCache.get(); 192 while (LastCache->getNextStatCache()) 193 LastCache = LastCache->getNextStatCache(); 194 195 LastCache->setNextStatCache(statCache); 196} 197 198void FileManager::removeStatCache(FileSystemStatCache *statCache) { 199 if (!statCache) 200 return; 201 202 if (StatCache.get() == statCache) { 203 // This is the first stat cache. 204 StatCache.reset(StatCache->takeNextStatCache()); 205 return; 206 } 207 208 // Find the stat cache in the list. 209 FileSystemStatCache *PrevCache = StatCache.get(); 210 while (PrevCache && PrevCache->getNextStatCache() != statCache) 211 PrevCache = PrevCache->getNextStatCache(); 212 213 assert(PrevCache && "Stat cache not found for removal"); 214 PrevCache->setNextStatCache(statCache->getNextStatCache()); 215} 216 217/// \brief Retrieve the directory that the given file name resides in. 218/// Filename can point to either a real file or a virtual file. 219static const DirectoryEntry *getDirectoryFromFile(FileManager &FileMgr, 220 llvm::StringRef Filename) { 221 if (Filename.empty()) 222 return NULL; 223 224 if (llvm::sys::path::is_separator(Filename[Filename.size() - 1])) 225 return NULL; // If Filename is a directory. 226 227 llvm::StringRef DirName = llvm::sys::path::parent_path(Filename); 228 // Use the current directory if file has no path component. 229 if (DirName.empty()) 230 DirName = "."; 231 232 return FileMgr.getDirectory(DirName); 233} 234 235/// Add all ancestors of the given path (pointing to either a file or 236/// a directory) as virtual directories. 237void FileManager::addAncestorsAsVirtualDirs(llvm::StringRef Path) { 238 llvm::StringRef DirName = llvm::sys::path::parent_path(Path); 239 if (DirName.empty()) 240 return; 241 242 llvm::StringMapEntry<DirectoryEntry *> &NamedDirEnt = 243 SeenDirEntries.GetOrCreateValue(DirName); 244 245 // When caching a virtual directory, we always cache its ancestors 246 // at the same time. Therefore, if DirName is already in the cache, 247 // we don't need to recurse as its ancestors must also already be in 248 // the cache. 249 if (NamedDirEnt.getValue()) 250 return; 251 252 // Add the virtual directory to the cache. 253 DirectoryEntry *UDE = new DirectoryEntry; 254 UDE->Name = NamedDirEnt.getKeyData(); 255 NamedDirEnt.setValue(UDE); 256 VirtualDirectoryEntries.push_back(UDE); 257 258 // Recursively add the other ancestors. 259 addAncestorsAsVirtualDirs(DirName); 260} 261 262/// getDirectory - Lookup, cache, and verify the specified directory 263/// (real or virtual). This returns NULL if the directory doesn't 264/// exist. 265/// 266const DirectoryEntry *FileManager::getDirectory(llvm::StringRef DirName) { 267 // stat doesn't like trailing separators (at least on Windows). 268 if (DirName.size() > 1 && llvm::sys::path::is_separator(DirName.back())) 269 DirName = DirName.substr(0, DirName.size()-1); 270 271 ++NumDirLookups; 272 llvm::StringMapEntry<DirectoryEntry *> &NamedDirEnt = 273 SeenDirEntries.GetOrCreateValue(DirName); 274 275 // See if there was already an entry in the map. Note that the map 276 // contains both virtual and real directories. 277 if (NamedDirEnt.getValue()) 278 return NamedDirEnt.getValue() == NON_EXISTENT_DIR 279 ? 0 : NamedDirEnt.getValue(); 280 281 ++NumDirCacheMisses; 282 283 // By default, initialize it to invalid. 284 NamedDirEnt.setValue(NON_EXISTENT_DIR); 285 286 // Get the null-terminated directory name as stored as the key of the 287 // SeenDirEntries map. 288 const char *InterndDirName = NamedDirEnt.getKeyData(); 289 290 // Check to see if the directory exists. 291 struct stat StatBuf; 292 if (getStatValue(InterndDirName, StatBuf, 0/*directory lookup*/)) { 293 // There's no real directory at the given path. 294 return 0; 295 } 296 297 // It exists. See if we have already opened a directory with the 298 // same inode (this occurs on Unix-like systems when one dir is 299 // symlinked to another, for example) or the same path (on 300 // Windows). 301 DirectoryEntry &UDE = UniqueRealDirs.getDirectory(InterndDirName, StatBuf); 302 303 NamedDirEnt.setValue(&UDE); 304 if (!UDE.getName()) { 305 // We don't have this directory yet, add it. We use the string 306 // key from the SeenDirEntries map as the string. 307 UDE.Name = InterndDirName; 308 } 309 310 return &UDE; 311} 312 313/// getFile - Lookup, cache, and verify the specified file (real or 314/// virtual). This returns NULL if the file doesn't exist. 315/// 316const FileEntry *FileManager::getFile(llvm::StringRef Filename) { 317 ++NumFileLookups; 318 319 // See if there is already an entry in the map. 320 llvm::StringMapEntry<FileEntry *> &NamedFileEnt = 321 SeenFileEntries.GetOrCreateValue(Filename); 322 323 // See if there is already an entry in the map. 324 if (NamedFileEnt.getValue()) 325 return NamedFileEnt.getValue() == NON_EXISTENT_FILE 326 ? 0 : NamedFileEnt.getValue(); 327 328 ++NumFileCacheMisses; 329 330 // By default, initialize it to invalid. 331 NamedFileEnt.setValue(NON_EXISTENT_FILE); 332 333 // Get the null-terminated file name as stored as the key of the 334 // SeenFileEntries map. 335 const char *InterndFileName = NamedFileEnt.getKeyData(); 336 337 // Look up the directory for the file. When looking up something like 338 // sys/foo.h we'll discover all of the search directories that have a 'sys' 339 // subdirectory. This will let us avoid having to waste time on known-to-fail 340 // searches when we go to find sys/bar.h, because all the search directories 341 // without a 'sys' subdir will get a cached failure result. 342 const DirectoryEntry *DirInfo = getDirectoryFromFile(*this, Filename); 343 if (DirInfo == 0) // Directory doesn't exist, file can't exist. 344 return 0; 345 346 // FIXME: Use the directory info to prune this, before doing the stat syscall. 347 // FIXME: This will reduce the # syscalls. 348 349 // Nope, there isn't. Check to see if the file exists. 350 int FileDescriptor = -1; 351 struct stat StatBuf; 352 if (getStatValue(InterndFileName, StatBuf, &FileDescriptor)) { 353 // There's no real file at the given path. 354 return 0; 355 } 356 357 // It exists. See if we have already opened a file with the same inode. 358 // This occurs when one dir is symlinked to another, for example. 359 FileEntry &UFE = UniqueRealFiles.getFile(InterndFileName, StatBuf); 360 361 NamedFileEnt.setValue(&UFE); 362 if (UFE.getName()) { // Already have an entry with this inode, return it. 363 // If the stat process opened the file, close it to avoid a FD leak. 364 if (FileDescriptor != -1) 365 close(FileDescriptor); 366 367 return &UFE; 368 } 369 370 // Otherwise, we don't have this directory yet, add it. 371 // FIXME: Change the name to be a char* that points back to the 372 // 'SeenFileEntries' key. 373 UFE.Name = InterndFileName; 374 UFE.Size = StatBuf.st_size; 375 UFE.ModTime = StatBuf.st_mtime; 376 UFE.Dir = DirInfo; 377 UFE.UID = NextFileUID++; 378 UFE.FD = FileDescriptor; 379 return &UFE; 380} 381 382const FileEntry * 383FileManager::getVirtualFile(llvm::StringRef Filename, off_t Size, 384 time_t ModificationTime) { 385 ++NumFileLookups; 386 387 // See if there is already an entry in the map. 388 llvm::StringMapEntry<FileEntry *> &NamedFileEnt = 389 SeenFileEntries.GetOrCreateValue(Filename); 390 391 // See if there is already an entry in the map. 392 if (NamedFileEnt.getValue() && NamedFileEnt.getValue() != NON_EXISTENT_FILE) 393 return NamedFileEnt.getValue(); 394 395 ++NumFileCacheMisses; 396 397 // By default, initialize it to invalid. 398 NamedFileEnt.setValue(NON_EXISTENT_FILE); 399 400 addAncestorsAsVirtualDirs(Filename); 401 FileEntry *UFE = 0; 402 403 // Now that all ancestors of Filename are in the cache, the 404 // following call is guaranteed to find the DirectoryEntry from the 405 // cache. 406 const DirectoryEntry *DirInfo = getDirectoryFromFile(*this, Filename); 407 assert(DirInfo && 408 "The directory of a virtual file should already be in the cache."); 409 410 // Check to see if the file exists. If so, drop the virtual file 411 int FileDescriptor = -1; 412 struct stat StatBuf; 413 const char *InterndFileName = NamedFileEnt.getKeyData(); 414 if (getStatValue(InterndFileName, StatBuf, &FileDescriptor) == 0) { 415 // If the stat process opened the file, close it to avoid a FD leak. 416 if (FileDescriptor != -1) 417 close(FileDescriptor); 418 419 StatBuf.st_size = Size; 420 StatBuf.st_mtime = ModificationTime; 421 UFE = &UniqueRealFiles.getFile(InterndFileName, StatBuf); 422 423 NamedFileEnt.setValue(UFE); 424 425 // If we had already opened this file, close it now so we don't 426 // leak the descriptor. We're not going to use the file 427 // descriptor anyway, since this is a virtual file. 428 if (UFE->FD != -1) { 429 close(UFE->FD); 430 UFE->FD = -1; 431 } 432 433 // If we already have an entry with this inode, return it. 434 if (UFE->getName()) 435 return UFE; 436 } 437 438 if (!UFE) { 439 UFE = new FileEntry(); 440 VirtualFileEntries.push_back(UFE); 441 NamedFileEnt.setValue(UFE); 442 } 443 444 UFE->Name = InterndFileName; 445 UFE->Size = Size; 446 UFE->ModTime = ModificationTime; 447 UFE->Dir = DirInfo; 448 UFE->UID = NextFileUID++; 449 UFE->FD = -1; 450 return UFE; 451} 452 453void FileManager::FixupRelativePath(llvm::SmallVectorImpl<char> &path) const { 454 llvm::StringRef pathRef(path.data(), path.size()); 455 456 if (FileSystemOpts.WorkingDir.empty() 457 || llvm::sys::path::is_absolute(pathRef)) 458 return; 459 460 llvm::SmallString<128> NewPath(FileSystemOpts.WorkingDir); 461 llvm::sys::path::append(NewPath, pathRef); 462 path = NewPath; 463} 464 465llvm::MemoryBuffer *FileManager:: 466getBufferForFile(const FileEntry *Entry, std::string *ErrorStr) { 467 llvm::OwningPtr<llvm::MemoryBuffer> Result; 468 llvm::error_code ec; 469 if (FileSystemOpts.WorkingDir.empty()) { 470 const char *Filename = Entry->getName(); 471 // If the file is already open, use the open file descriptor. 472 if (Entry->FD != -1) { 473 ec = llvm::MemoryBuffer::getOpenFile(Entry->FD, Filename, Result, 474 Entry->getSize()); 475 if (ErrorStr) 476 *ErrorStr = ec.message(); 477 478 close(Entry->FD); 479 Entry->FD = -1; 480 return Result.take(); 481 } 482 483 // Otherwise, open the file. 484 ec = llvm::MemoryBuffer::getFile(Filename, Result, Entry->getSize()); 485 if (ec && ErrorStr) 486 *ErrorStr = ec.message(); 487 return Result.take(); 488 } 489 490 llvm::SmallString<128> FilePath(Entry->getName()); 491 FixupRelativePath(FilePath); 492 ec = llvm::MemoryBuffer::getFile(FilePath.str(), Result, Entry->getSize()); 493 if (ec && ErrorStr) 494 *ErrorStr = ec.message(); 495 return Result.take(); 496} 497 498llvm::MemoryBuffer *FileManager:: 499getBufferForFile(llvm::StringRef Filename, std::string *ErrorStr) { 500 llvm::OwningPtr<llvm::MemoryBuffer> Result; 501 llvm::error_code ec; 502 if (FileSystemOpts.WorkingDir.empty()) { 503 ec = llvm::MemoryBuffer::getFile(Filename, Result); 504 if (ec && ErrorStr) 505 *ErrorStr = ec.message(); 506 return Result.take(); 507 } 508 509 llvm::SmallString<128> FilePath(Filename); 510 FixupRelativePath(FilePath); 511 ec = llvm::MemoryBuffer::getFile(FilePath.c_str(), Result); 512 if (ec && ErrorStr) 513 *ErrorStr = ec.message(); 514 return Result.take(); 515} 516 517/// getStatValue - Get the 'stat' information for the specified path, 518/// using the cache to accelerate it if possible. This returns true 519/// if the path points to a virtual file or does not exist, or returns 520/// false if it's an existent real file. If FileDescriptor is NULL, 521/// do directory look-up instead of file look-up. 522bool FileManager::getStatValue(const char *Path, struct stat &StatBuf, 523 int *FileDescriptor) { 524 // FIXME: FileSystemOpts shouldn't be passed in here, all paths should be 525 // absolute! 526 if (FileSystemOpts.WorkingDir.empty()) 527 return FileSystemStatCache::get(Path, StatBuf, FileDescriptor, 528 StatCache.get()); 529 530 llvm::SmallString<128> FilePath(Path); 531 FixupRelativePath(FilePath); 532 533 return FileSystemStatCache::get(FilePath.c_str(), StatBuf, FileDescriptor, 534 StatCache.get()); 535} 536 537void FileManager::GetUniqueIDMapping( 538 llvm::SmallVectorImpl<const FileEntry *> &UIDToFiles) const { 539 UIDToFiles.clear(); 540 UIDToFiles.resize(NextFileUID); 541 542 // Map file entries 543 for (llvm::StringMap<FileEntry*, llvm::BumpPtrAllocator>::const_iterator 544 FE = SeenFileEntries.begin(), FEEnd = SeenFileEntries.end(); 545 FE != FEEnd; ++FE) 546 if (FE->getValue() && FE->getValue() != NON_EXISTENT_FILE) 547 UIDToFiles[FE->getValue()->getUID()] = FE->getValue(); 548 549 // Map virtual file entries 550 for (llvm::SmallVector<FileEntry*, 4>::const_iterator 551 VFE = VirtualFileEntries.begin(), VFEEnd = VirtualFileEntries.end(); 552 VFE != VFEEnd; ++VFE) 553 if (*VFE && *VFE != NON_EXISTENT_FILE) 554 UIDToFiles[(*VFE)->getUID()] = *VFE; 555} 556 557 558void FileManager::PrintStats() const { 559 llvm::errs() << "\n*** File Manager Stats:\n"; 560 llvm::errs() << UniqueRealFiles.size() << " real files found, " 561 << UniqueRealDirs.size() << " real dirs found.\n"; 562 llvm::errs() << VirtualFileEntries.size() << " virtual files found, " 563 << VirtualDirectoryEntries.size() << " virtual dirs found.\n"; 564 llvm::errs() << NumDirLookups << " dir lookups, " 565 << NumDirCacheMisses << " dir cache misses.\n"; 566 llvm::errs() << NumFileLookups << " file lookups, " 567 << NumFileCacheMisses << " file cache misses.\n"; 568 569 //llvm::errs() << PagesMapped << BytesOfPagesMapped << FSLookups; 570} 571