1//===--- FileManager.cpp - File System Probing and Caching ----------------===// 2// 3// The LLVM Compiler Infrastructure 4// 5// This file is distributed under the University of Illinois Open Source 6// License. See LICENSE.TXT for details. 7// 8//===----------------------------------------------------------------------===// 9// 10// This file implements the FileManager interface. 11// 12//===----------------------------------------------------------------------===// 13// 14// TODO: This should index all interesting directories with dirent calls. 15// getdirentries ? 16// opendir/readdir_r/closedir ? 17// 18//===----------------------------------------------------------------------===// 19 20#include "clang/Basic/FileManager.h" 21#include "clang/Basic/FileSystemStatCache.h" 22#include "llvm/ADT/SmallString.h" 23#include "llvm/Support/FileSystem.h" 24#include "llvm/Support/MemoryBuffer.h" 25#include "llvm/Support/raw_ostream.h" 26#include "llvm/Support/Path.h" 27#include "llvm/Support/system_error.h" 28#include "llvm/Config/llvm-config.h" 29#include <map> 30#include <set> 31#include <string> 32 33// FIXME: This is terrible, we need this for ::close. 34#if !defined(_MSC_VER) && !defined(__MINGW32__) 35#include <unistd.h> 36#include <sys/uio.h> 37#else 38#include <io.h> 39#endif 40using namespace clang; 41 42// FIXME: Enhance libsystem to support inode and other fields. 43#include <sys/stat.h> 44 45/// NON_EXISTENT_DIR - A special value distinct from null that is used to 46/// represent a dir name that doesn't exist on the disk. 47#define NON_EXISTENT_DIR reinterpret_cast<DirectoryEntry*>((intptr_t)-1) 48 49/// NON_EXISTENT_FILE - A special value distinct from null that is used to 50/// represent a filename that doesn't exist on the disk. 51#define NON_EXISTENT_FILE reinterpret_cast<FileEntry*>((intptr_t)-1) 52 53 54FileEntry::~FileEntry() { 55 // If this FileEntry owns an open file descriptor that never got used, close 56 // it. 57 if (FD != -1) ::close(FD); 58} 59 60//===----------------------------------------------------------------------===// 61// Windows. 62//===----------------------------------------------------------------------===// 63 64#ifdef LLVM_ON_WIN32 65 66namespace { 67 static std::string GetFullPath(const char *relPath) { 68 char *absPathStrPtr = _fullpath(NULL, relPath, 0); 69 assert(absPathStrPtr && "_fullpath() returned NULL!"); 70 71 std::string absPath(absPathStrPtr); 72 73 free(absPathStrPtr); 74 return absPath; 75 } 76} 77 78class FileManager::UniqueDirContainer { 79 /// UniqueDirs - Cache from full path to existing directories/files. 80 /// 81 llvm::StringMap<DirectoryEntry> UniqueDirs; 82 83public: 84 /// getDirectory - Return an existing DirectoryEntry with the given 85 /// name if there is already one; otherwise create and return a 86 /// default-constructed DirectoryEntry. 87 DirectoryEntry &getDirectory(const char *Name, 88 const struct stat & /*StatBuf*/) { 89 std::string FullPath(GetFullPath(Name)); 90 return UniqueDirs.GetOrCreateValue(FullPath).getValue(); 91 } 92 93 size_t size() const { return UniqueDirs.size(); } 94}; 95 96class FileManager::UniqueFileContainer { 97 /// UniqueFiles - Cache from full path to existing directories/files. 98 /// 99 llvm::StringMap<FileEntry, llvm::BumpPtrAllocator> UniqueFiles; 100 101public: 102 /// getFile - Return an existing FileEntry with the given name if 103 /// there is already one; otherwise create and return a 104 /// default-constructed FileEntry. 105 FileEntry &getFile(const char *Name, const struct stat & /*StatBuf*/) { 106 std::string FullPath(GetFullPath(Name)); 107 108 // Lowercase string because Windows filesystem is case insensitive. 109 FullPath = StringRef(FullPath).lower(); 110 return UniqueFiles.GetOrCreateValue(FullPath).getValue(); 111 } 112 113 size_t size() const { return UniqueFiles.size(); } 114 115 void erase(const FileEntry *Entry) { 116 std::string FullPath(GetFullPath(Entry->getName())); 117 118 // Lowercase string because Windows filesystem is case insensitive. 119 FullPath = StringRef(FullPath).lower(); 120 UniqueFiles.erase(FullPath); 121 } 122}; 123 124//===----------------------------------------------------------------------===// 125// Unix-like Systems. 126//===----------------------------------------------------------------------===// 127 128#else 129 130class FileManager::UniqueDirContainer { 131 /// UniqueDirs - Cache from ID's to existing directories/files. 132 std::map<std::pair<dev_t, ino_t>, DirectoryEntry> UniqueDirs; 133 134public: 135 /// getDirectory - Return an existing DirectoryEntry with the given 136 /// ID's if there is already one; otherwise create and return a 137 /// default-constructed DirectoryEntry. 138 DirectoryEntry &getDirectory(const char * /*Name*/, 139 const struct stat &StatBuf) { 140 return UniqueDirs[std::make_pair(StatBuf.st_dev, StatBuf.st_ino)]; 141 } 142 143 size_t size() const { return UniqueDirs.size(); } 144}; 145 146class FileManager::UniqueFileContainer { 147 /// UniqueFiles - Cache from ID's to existing directories/files. 148 std::set<FileEntry> UniqueFiles; 149 150public: 151 /// getFile - Return an existing FileEntry with the given ID's if 152 /// there is already one; otherwise create and return a 153 /// default-constructed FileEntry. 154 FileEntry &getFile(const char * /*Name*/, const struct stat &StatBuf) { 155 return 156 const_cast<FileEntry&>( 157 *UniqueFiles.insert(FileEntry(StatBuf.st_dev, 158 StatBuf.st_ino, 159 StatBuf.st_mode)).first); 160 } 161 162 size_t size() const { return UniqueFiles.size(); } 163 164 void erase(const FileEntry *Entry) { UniqueFiles.erase(*Entry); } 165}; 166 167#endif 168 169//===----------------------------------------------------------------------===// 170// Common logic. 171//===----------------------------------------------------------------------===// 172 173FileManager::FileManager(const FileSystemOptions &FSO) 174 : FileSystemOpts(FSO), 175 UniqueRealDirs(*new UniqueDirContainer()), 176 UniqueRealFiles(*new UniqueFileContainer()), 177 SeenDirEntries(64), SeenFileEntries(64), NextFileUID(0) { 178 NumDirLookups = NumFileLookups = 0; 179 NumDirCacheMisses = NumFileCacheMisses = 0; 180} 181 182FileManager::~FileManager() { 183 delete &UniqueRealDirs; 184 delete &UniqueRealFiles; 185 for (unsigned i = 0, e = VirtualFileEntries.size(); i != e; ++i) 186 delete VirtualFileEntries[i]; 187 for (unsigned i = 0, e = VirtualDirectoryEntries.size(); i != e; ++i) 188 delete VirtualDirectoryEntries[i]; 189} 190 191void FileManager::addStatCache(FileSystemStatCache *statCache, 192 bool AtBeginning) { 193 assert(statCache && "No stat cache provided?"); 194 if (AtBeginning || StatCache.get() == 0) { 195 statCache->setNextStatCache(StatCache.take()); 196 StatCache.reset(statCache); 197 return; 198 } 199 200 FileSystemStatCache *LastCache = StatCache.get(); 201 while (LastCache->getNextStatCache()) 202 LastCache = LastCache->getNextStatCache(); 203 204 LastCache->setNextStatCache(statCache); 205} 206 207void FileManager::removeStatCache(FileSystemStatCache *statCache) { 208 if (!statCache) 209 return; 210 211 if (StatCache.get() == statCache) { 212 // This is the first stat cache. 213 StatCache.reset(StatCache->takeNextStatCache()); 214 return; 215 } 216 217 // Find the stat cache in the list. 218 FileSystemStatCache *PrevCache = StatCache.get(); 219 while (PrevCache && PrevCache->getNextStatCache() != statCache) 220 PrevCache = PrevCache->getNextStatCache(); 221 222 assert(PrevCache && "Stat cache not found for removal"); 223 PrevCache->setNextStatCache(statCache->getNextStatCache()); 224} 225 226void FileManager::clearStatCaches() { 227 StatCache.reset(0); 228} 229 230/// \brief Retrieve the directory that the given file name resides in. 231/// Filename can point to either a real file or a virtual file. 232static const DirectoryEntry *getDirectoryFromFile(FileManager &FileMgr, 233 StringRef Filename, 234 bool CacheFailure) { 235 if (Filename.empty()) 236 return NULL; 237 238 if (llvm::sys::path::is_separator(Filename[Filename.size() - 1])) 239 return NULL; // If Filename is a directory. 240 241 StringRef DirName = llvm::sys::path::parent_path(Filename); 242 // Use the current directory if file has no path component. 243 if (DirName.empty()) 244 DirName = "."; 245 246 return FileMgr.getDirectory(DirName, CacheFailure); 247} 248 249/// Add all ancestors of the given path (pointing to either a file or 250/// a directory) as virtual directories. 251void FileManager::addAncestorsAsVirtualDirs(StringRef Path) { 252 StringRef DirName = llvm::sys::path::parent_path(Path); 253 if (DirName.empty()) 254 return; 255 256 llvm::StringMapEntry<DirectoryEntry *> &NamedDirEnt = 257 SeenDirEntries.GetOrCreateValue(DirName); 258 259 // When caching a virtual directory, we always cache its ancestors 260 // at the same time. Therefore, if DirName is already in the cache, 261 // we don't need to recurse as its ancestors must also already be in 262 // the cache. 263 if (NamedDirEnt.getValue()) 264 return; 265 266 // Add the virtual directory to the cache. 267 DirectoryEntry *UDE = new DirectoryEntry; 268 UDE->Name = NamedDirEnt.getKeyData(); 269 NamedDirEnt.setValue(UDE); 270 VirtualDirectoryEntries.push_back(UDE); 271 272 // Recursively add the other ancestors. 273 addAncestorsAsVirtualDirs(DirName); 274} 275 276const DirectoryEntry *FileManager::getDirectory(StringRef DirName, 277 bool CacheFailure) { 278 // stat doesn't like trailing separators except for root directory. 279 // At least, on Win32 MSVCRT, stat() cannot strip trailing '/'. 280 // (though it can strip '\\') 281 if (DirName.size() > 1 && 282 DirName != llvm::sys::path::root_path(DirName) && 283 llvm::sys::path::is_separator(DirName.back())) 284 DirName = DirName.substr(0, DirName.size()-1); 285 286 ++NumDirLookups; 287 llvm::StringMapEntry<DirectoryEntry *> &NamedDirEnt = 288 SeenDirEntries.GetOrCreateValue(DirName); 289 290 // See if there was already an entry in the map. Note that the map 291 // contains both virtual and real directories. 292 if (NamedDirEnt.getValue()) 293 return NamedDirEnt.getValue() == NON_EXISTENT_DIR 294 ? 0 : NamedDirEnt.getValue(); 295 296 ++NumDirCacheMisses; 297 298 // By default, initialize it to invalid. 299 NamedDirEnt.setValue(NON_EXISTENT_DIR); 300 301 // Get the null-terminated directory name as stored as the key of the 302 // SeenDirEntries map. 303 const char *InterndDirName = NamedDirEnt.getKeyData(); 304 305 // Check to see if the directory exists. 306 struct stat StatBuf; 307 if (getStatValue(InterndDirName, StatBuf, 0/*directory lookup*/)) { 308 // There's no real directory at the given path. 309 if (!CacheFailure) 310 SeenDirEntries.erase(DirName); 311 return 0; 312 } 313 314 // It exists. See if we have already opened a directory with the 315 // same inode (this occurs on Unix-like systems when one dir is 316 // symlinked to another, for example) or the same path (on 317 // Windows). 318 DirectoryEntry &UDE = UniqueRealDirs.getDirectory(InterndDirName, StatBuf); 319 320 NamedDirEnt.setValue(&UDE); 321 if (!UDE.getName()) { 322 // We don't have this directory yet, add it. We use the string 323 // key from the SeenDirEntries map as the string. 324 UDE.Name = InterndDirName; 325 } 326 327 return &UDE; 328} 329 330const FileEntry *FileManager::getFile(StringRef Filename, bool openFile, 331 bool CacheFailure) { 332 ++NumFileLookups; 333 334 // See if there is already an entry in the map. 335 llvm::StringMapEntry<FileEntry *> &NamedFileEnt = 336 SeenFileEntries.GetOrCreateValue(Filename); 337 338 // See if there is already an entry in the map. 339 if (NamedFileEnt.getValue()) 340 return NamedFileEnt.getValue() == NON_EXISTENT_FILE 341 ? 0 : NamedFileEnt.getValue(); 342 343 ++NumFileCacheMisses; 344 345 // By default, initialize it to invalid. 346 NamedFileEnt.setValue(NON_EXISTENT_FILE); 347 348 // Get the null-terminated file name as stored as the key of the 349 // SeenFileEntries map. 350 const char *InterndFileName = NamedFileEnt.getKeyData(); 351 352 // Look up the directory for the file. When looking up something like 353 // sys/foo.h we'll discover all of the search directories that have a 'sys' 354 // subdirectory. This will let us avoid having to waste time on known-to-fail 355 // searches when we go to find sys/bar.h, because all the search directories 356 // without a 'sys' subdir will get a cached failure result. 357 const DirectoryEntry *DirInfo = getDirectoryFromFile(*this, Filename, 358 CacheFailure); 359 if (DirInfo == 0) { // Directory doesn't exist, file can't exist. 360 if (!CacheFailure) 361 SeenFileEntries.erase(Filename); 362 363 return 0; 364 } 365 366 // FIXME: Use the directory info to prune this, before doing the stat syscall. 367 // FIXME: This will reduce the # syscalls. 368 369 // Nope, there isn't. Check to see if the file exists. 370 int FileDescriptor = -1; 371 struct stat StatBuf; 372 if (getStatValue(InterndFileName, StatBuf, &FileDescriptor)) { 373 // There's no real file at the given path. 374 if (!CacheFailure) 375 SeenFileEntries.erase(Filename); 376 377 return 0; 378 } 379 380 if (FileDescriptor != -1 && !openFile) { 381 close(FileDescriptor); 382 FileDescriptor = -1; 383 } 384 385 // It exists. See if we have already opened a file with the same inode. 386 // This occurs when one dir is symlinked to another, for example. 387 FileEntry &UFE = UniqueRealFiles.getFile(InterndFileName, StatBuf); 388 389 NamedFileEnt.setValue(&UFE); 390 if (UFE.getName()) { // Already have an entry with this inode, return it. 391 // If the stat process opened the file, close it to avoid a FD leak. 392 if (FileDescriptor != -1) 393 close(FileDescriptor); 394 395 return &UFE; 396 } 397 398 // Otherwise, we don't have this directory yet, add it. 399 // FIXME: Change the name to be a char* that points back to the 400 // 'SeenFileEntries' key. 401 UFE.Name = InterndFileName; 402 UFE.Size = StatBuf.st_size; 403 UFE.ModTime = StatBuf.st_mtime; 404 UFE.Dir = DirInfo; 405 UFE.UID = NextFileUID++; 406 UFE.FD = FileDescriptor; 407 return &UFE; 408} 409 410const FileEntry * 411FileManager::getVirtualFile(StringRef Filename, off_t Size, 412 time_t ModificationTime) { 413 ++NumFileLookups; 414 415 // See if there is already an entry in the map. 416 llvm::StringMapEntry<FileEntry *> &NamedFileEnt = 417 SeenFileEntries.GetOrCreateValue(Filename); 418 419 // See if there is already an entry in the map. 420 if (NamedFileEnt.getValue() && NamedFileEnt.getValue() != NON_EXISTENT_FILE) 421 return NamedFileEnt.getValue(); 422 423 ++NumFileCacheMisses; 424 425 // By default, initialize it to invalid. 426 NamedFileEnt.setValue(NON_EXISTENT_FILE); 427 428 addAncestorsAsVirtualDirs(Filename); 429 FileEntry *UFE = 0; 430 431 // Now that all ancestors of Filename are in the cache, the 432 // following call is guaranteed to find the DirectoryEntry from the 433 // cache. 434 const DirectoryEntry *DirInfo = getDirectoryFromFile(*this, Filename, 435 /*CacheFailure=*/true); 436 assert(DirInfo && 437 "The directory of a virtual file should already be in the cache."); 438 439 // Check to see if the file exists. If so, drop the virtual file 440 int FileDescriptor = -1; 441 struct stat StatBuf; 442 const char *InterndFileName = NamedFileEnt.getKeyData(); 443 if (getStatValue(InterndFileName, StatBuf, &FileDescriptor) == 0) { 444 // If the stat process opened the file, close it to avoid a FD leak. 445 if (FileDescriptor != -1) 446 close(FileDescriptor); 447 448 StatBuf.st_size = Size; 449 StatBuf.st_mtime = ModificationTime; 450 UFE = &UniqueRealFiles.getFile(InterndFileName, StatBuf); 451 452 NamedFileEnt.setValue(UFE); 453 454 // If we had already opened this file, close it now so we don't 455 // leak the descriptor. We're not going to use the file 456 // descriptor anyway, since this is a virtual file. 457 if (UFE->FD != -1) { 458 close(UFE->FD); 459 UFE->FD = -1; 460 } 461 462 // If we already have an entry with this inode, return it. 463 if (UFE->getName()) 464 return UFE; 465 } 466 467 if (!UFE) { 468 UFE = new FileEntry(); 469 VirtualFileEntries.push_back(UFE); 470 NamedFileEnt.setValue(UFE); 471 } 472 473 UFE->Name = InterndFileName; 474 UFE->Size = Size; 475 UFE->ModTime = ModificationTime; 476 UFE->Dir = DirInfo; 477 UFE->UID = NextFileUID++; 478 UFE->FD = -1; 479 return UFE; 480} 481 482void FileManager::FixupRelativePath(SmallVectorImpl<char> &path) const { 483 StringRef pathRef(path.data(), path.size()); 484 485 if (FileSystemOpts.WorkingDir.empty() 486 || llvm::sys::path::is_absolute(pathRef)) 487 return; 488 489 SmallString<128> NewPath(FileSystemOpts.WorkingDir); 490 llvm::sys::path::append(NewPath, pathRef); 491 path = NewPath; 492} 493 494llvm::MemoryBuffer *FileManager:: 495getBufferForFile(const FileEntry *Entry, std::string *ErrorStr, 496 bool isVolatile) { 497 OwningPtr<llvm::MemoryBuffer> Result; 498 llvm::error_code ec; 499 500 uint64_t FileSize = Entry->getSize(); 501 // If there's a high enough chance that the file have changed since we 502 // got its size, force a stat before opening it. 503 if (isVolatile) 504 FileSize = -1; 505 506 const char *Filename = Entry->getName(); 507 // If the file is already open, use the open file descriptor. 508 if (Entry->FD != -1) { 509 ec = llvm::MemoryBuffer::getOpenFile(Entry->FD, Filename, Result, FileSize); 510 if (ErrorStr) 511 *ErrorStr = ec.message(); 512 513 close(Entry->FD); 514 Entry->FD = -1; 515 return Result.take(); 516 } 517 518 // Otherwise, open the file. 519 520 if (FileSystemOpts.WorkingDir.empty()) { 521 ec = llvm::MemoryBuffer::getFile(Filename, Result, FileSize); 522 if (ec && ErrorStr) 523 *ErrorStr = ec.message(); 524 return Result.take(); 525 } 526 527 SmallString<128> FilePath(Entry->getName()); 528 FixupRelativePath(FilePath); 529 ec = llvm::MemoryBuffer::getFile(FilePath.str(), Result, FileSize); 530 if (ec && ErrorStr) 531 *ErrorStr = ec.message(); 532 return Result.take(); 533} 534 535llvm::MemoryBuffer *FileManager:: 536getBufferForFile(StringRef Filename, std::string *ErrorStr) { 537 OwningPtr<llvm::MemoryBuffer> Result; 538 llvm::error_code ec; 539 if (FileSystemOpts.WorkingDir.empty()) { 540 ec = llvm::MemoryBuffer::getFile(Filename, Result); 541 if (ec && ErrorStr) 542 *ErrorStr = ec.message(); 543 return Result.take(); 544 } 545 546 SmallString<128> FilePath(Filename); 547 FixupRelativePath(FilePath); 548 ec = llvm::MemoryBuffer::getFile(FilePath.c_str(), Result); 549 if (ec && ErrorStr) 550 *ErrorStr = ec.message(); 551 return Result.take(); 552} 553 554/// getStatValue - Get the 'stat' information for the specified path, 555/// using the cache to accelerate it if possible. This returns true 556/// if the path points to a virtual file or does not exist, or returns 557/// false if it's an existent real file. If FileDescriptor is NULL, 558/// do directory look-up instead of file look-up. 559bool FileManager::getStatValue(const char *Path, struct stat &StatBuf, 560 int *FileDescriptor) { 561 // FIXME: FileSystemOpts shouldn't be passed in here, all paths should be 562 // absolute! 563 if (FileSystemOpts.WorkingDir.empty()) 564 return FileSystemStatCache::get(Path, StatBuf, FileDescriptor, 565 StatCache.get()); 566 567 SmallString<128> FilePath(Path); 568 FixupRelativePath(FilePath); 569 570 return FileSystemStatCache::get(FilePath.c_str(), StatBuf, FileDescriptor, 571 StatCache.get()); 572} 573 574bool FileManager::getNoncachedStatValue(StringRef Path, 575 struct stat &StatBuf) { 576 SmallString<128> FilePath(Path); 577 FixupRelativePath(FilePath); 578 579 return ::stat(FilePath.c_str(), &StatBuf) != 0; 580} 581 582void FileManager::invalidateCache(const FileEntry *Entry) { 583 assert(Entry && "Cannot invalidate a NULL FileEntry"); 584 585 SeenFileEntries.erase(Entry->getName()); 586 587 // FileEntry invalidation should not block future optimizations in the file 588 // caches. Possible alternatives are cache truncation (invalidate last N) or 589 // invalidation of the whole cache. 590 UniqueRealFiles.erase(Entry); 591} 592 593 594void FileManager::GetUniqueIDMapping( 595 SmallVectorImpl<const FileEntry *> &UIDToFiles) const { 596 UIDToFiles.clear(); 597 UIDToFiles.resize(NextFileUID); 598 599 // Map file entries 600 for (llvm::StringMap<FileEntry*, llvm::BumpPtrAllocator>::const_iterator 601 FE = SeenFileEntries.begin(), FEEnd = SeenFileEntries.end(); 602 FE != FEEnd; ++FE) 603 if (FE->getValue() && FE->getValue() != NON_EXISTENT_FILE) 604 UIDToFiles[FE->getValue()->getUID()] = FE->getValue(); 605 606 // Map virtual file entries 607 for (SmallVector<FileEntry*, 4>::const_iterator 608 VFE = VirtualFileEntries.begin(), VFEEnd = VirtualFileEntries.end(); 609 VFE != VFEEnd; ++VFE) 610 if (*VFE && *VFE != NON_EXISTENT_FILE) 611 UIDToFiles[(*VFE)->getUID()] = *VFE; 612} 613 614void FileManager::modifyFileEntry(FileEntry *File, 615 off_t Size, time_t ModificationTime) { 616 File->Size = Size; 617 File->ModTime = ModificationTime; 618} 619 620 621void FileManager::PrintStats() const { 622 llvm::errs() << "\n*** File Manager Stats:\n"; 623 llvm::errs() << UniqueRealFiles.size() << " real files found, " 624 << UniqueRealDirs.size() << " real dirs found.\n"; 625 llvm::errs() << VirtualFileEntries.size() << " virtual files found, " 626 << VirtualDirectoryEntries.size() << " virtual dirs found.\n"; 627 llvm::errs() << NumDirLookups << " dir lookups, " 628 << NumDirCacheMisses << " dir cache misses.\n"; 629 llvm::errs() << NumFileLookups << " file lookups, " 630 << NumFileCacheMisses << " file cache misses.\n"; 631 632 //llvm::errs() << PagesMapped << BytesOfPagesMapped << FSLookups; 633} 634