SourceManager.cpp revision 71e443ac92dfcabc34e8c1a9ac12a7ab602b9d37
1//===--- SourceManager.cpp - Track and cache source files -----------------===// 2// 3// The LLVM Compiler Infrastructure 4// 5// This file is distributed under the University of Illinois Open Source 6// License. See LICENSE.TXT for details. 7// 8//===----------------------------------------------------------------------===// 9// 10// This file implements the SourceManager interface. 11// 12//===----------------------------------------------------------------------===// 13 14#include "clang/Basic/SourceManager.h" 15#include "clang/Basic/FileManager.h" 16#include "llvm/Support/Compiler.h" 17#include "llvm/Support/MemoryBuffer.h" 18#include "llvm/System/Path.h" 19#include "llvm/Bitcode/Serialize.h" 20#include "llvm/Bitcode/Deserialize.h" 21#include "llvm/Support/Streams.h" 22#include <algorithm> 23using namespace clang; 24using namespace SrcMgr; 25using llvm::MemoryBuffer; 26 27// This (temporary) directive toggles between lazy and eager creation of 28// MemBuffers. This directive is not permanent, and is here to test a few 29// potential optimizations in PTH. Once it is clear whether eager or lazy 30// creation of MemBuffers is better this directive will get removed. 31#define LAZY 32 33ContentCache::~ContentCache() { 34 delete Buffer; 35 delete [] SourceLineCache; 36} 37 38/// getSizeBytesMapped - Returns the number of bytes actually mapped for 39/// this ContentCache. This can be 0 if the MemBuffer was not actually 40/// instantiated. 41unsigned ContentCache::getSizeBytesMapped() const { 42 return Buffer ? Buffer->getBufferSize() : 0; 43} 44 45/// getSize - Returns the size of the content encapsulated by this ContentCache. 46/// This can be the size of the source file or the size of an arbitrary 47/// scratch buffer. If the ContentCache encapsulates a source file, that 48/// file is not lazily brought in from disk to satisfy this query. 49unsigned ContentCache::getSize() const { 50 return Entry ? Entry->getSize() : Buffer->getBufferSize(); 51} 52 53const llvm::MemoryBuffer* ContentCache::getBuffer() const { 54#ifdef LAZY 55 // Lazily create the Buffer for ContentCaches that wrap files. 56 if (!Buffer && Entry) { 57 // FIXME: Should we support a way to not have to do this check over 58 // and over if we cannot open the file? 59 Buffer = MemoryBuffer::getFile(Entry->getName(), 0, Entry->getSize()); 60 } 61#endif 62 return Buffer; 63} 64 65 66/// getFileInfo - Create or return a cached FileInfo for the specified file. 67/// 68const ContentCache* SourceManager::getContentCache(const FileEntry *FileEnt) { 69 70 assert(FileEnt && "Didn't specify a file entry to use?"); 71 // Do we already have information about this file? 72 std::set<ContentCache>::iterator I = 73 FileInfos.lower_bound(ContentCache(FileEnt)); 74 75 if (I != FileInfos.end() && I->Entry == FileEnt) 76 return &*I; 77 78 // Nope, get information. 79#ifndef LAZY 80 const MemoryBuffer *File = 81 MemoryBuffer::getFile(FileEnt->getName(), 0, FileEnt->getSize()); 82 if (File == 0) 83 return 0; 84#endif 85 86 ContentCache& Entry = const_cast<ContentCache&>(*FileInfos.insert(I,FileEnt)); 87#ifndef LAZY 88 Entry.setBuffer(File); 89#endif 90 Entry.SourceLineCache = 0; 91 Entry.NumLines = 0; 92 return &Entry; 93} 94 95 96/// createMemBufferContentCache - Create a new ContentCache for the specified 97/// memory buffer. This does no caching. 98const ContentCache* 99SourceManager::createMemBufferContentCache(const MemoryBuffer *Buffer) { 100 // Add a new ContentCache to the MemBufferInfos list and return it. We 101 // must default construct the object first that the instance actually 102 // stored within MemBufferInfos actually owns the Buffer, and not any 103 // temporary we would use in the call to "push_back". 104 MemBufferInfos.push_back(ContentCache()); 105 ContentCache& Entry = const_cast<ContentCache&>(MemBufferInfos.back()); 106 Entry.setBuffer(Buffer); 107 return &Entry; 108} 109 110 111/// createFileID - Create a new fileID for the specified ContentCache and 112/// include position. This works regardless of whether the ContentCache 113/// corresponds to a file or some other input source. 114FileID SourceManager::createFileID(const ContentCache *File, 115 SourceLocation IncludePos, 116 SrcMgr::CharacteristicKind FileCharacter) { 117 // If FileEnt is really large (e.g. it's a large .i file), we may not be able 118 // to fit an arbitrary position in the file in the FilePos field. To handle 119 // this, we create one FileID for each chunk of the file that fits in a 120 // FilePos field. 121 unsigned FileSize = File->getSize(); 122 if (FileSize+1 < (1 << SourceLocation::FilePosBits)) { 123 FileIDs.push_back(FileIDInfo::get(IncludePos, 0, File, FileCharacter)); 124 assert(FileIDs.size() < (1 << SourceLocation::ChunkIDBits) && 125 "Ran out of file ID's!"); 126 return FileID::Create(FileIDs.size()); 127 } 128 129 // Create one FileID for each chunk of the file. 130 unsigned Result = FileIDs.size()+1; 131 132 unsigned ChunkNo = 0; 133 while (1) { 134 FileIDs.push_back(FileIDInfo::get(IncludePos, ChunkNo++, File, 135 FileCharacter)); 136 137 if (FileSize+1 < (1 << SourceLocation::FilePosBits)) break; 138 FileSize -= (1 << SourceLocation::FilePosBits); 139 } 140 141 assert(FileIDs.size() < (1 << SourceLocation::ChunkIDBits) && 142 "Ran out of file ID's!"); 143 return FileID::Create(Result); 144} 145 146/// getInstantiationLoc - Return a new SourceLocation that encodes the fact 147/// that a token from SpellingLoc should actually be referenced from 148/// InstantiationLoc. 149SourceLocation SourceManager::getInstantiationLoc(SourceLocation SpellingLoc, 150 SourceLocation InstantLoc) { 151 // The specified source location may be a mapped location, due to a macro 152 // instantiation or #line directive. Strip off this information to find out 153 // where the characters are actually located. 154 SpellingLoc = getSpellingLoc(SpellingLoc); 155 156 // Resolve InstantLoc down to a real instantiation location. 157 InstantLoc = getInstantiationLoc(InstantLoc); 158 159 160 // If the last macro id is close to the currently requested location, try to 161 // reuse it. This implements a small cache. 162 for (int i = MacroIDs.size()-1, e = MacroIDs.size()-6; i >= 0 && i != e; --i){ 163 MacroIDInfo &LastOne = MacroIDs[i]; 164 165 // The instanitation point and source SpellingLoc have to exactly match to 166 // reuse (for now). We could allow "nearby" instantiations in the future. 167 if (LastOne.getInstantiationLoc() != InstantLoc || 168 LastOne.getSpellingLoc().getChunkID() != SpellingLoc.getChunkID()) 169 continue; 170 171 // Check to see if the spellloc of the token came from near enough to reuse. 172 int SpellDelta = SpellingLoc.getRawFilePos() - 173 LastOne.getSpellingLoc().getRawFilePos(); 174 if (SourceLocation::isValidMacroSpellingOffs(SpellDelta)) 175 return SourceLocation::getMacroLoc(i, SpellDelta); 176 } 177 178 179 MacroIDs.push_back(MacroIDInfo::get(InstantLoc, SpellingLoc)); 180 return SourceLocation::getMacroLoc(MacroIDs.size()-1, 0); 181} 182 183/// getBufferData - Return a pointer to the start and end of the source buffer 184/// data for the specified FileID. 185std::pair<const char*, const char*> 186SourceManager::getBufferData(FileID FID) const { 187 const llvm::MemoryBuffer *Buf = getBuffer(FID); 188 return std::make_pair(Buf->getBufferStart(), Buf->getBufferEnd()); 189} 190 191 192 193/// getCharacterData - Return a pointer to the start of the specified location 194/// in the appropriate MemoryBuffer. 195const char *SourceManager::getCharacterData(SourceLocation SL) const { 196 // Note that this is a hot function in the getSpelling() path, which is 197 // heavily used by -E mode. 198 SL = getSpellingLoc(SL); 199 200 std::pair<FileID, unsigned> LocInfo = getDecomposedFileLoc(SL); 201 202 // Note that calling 'getBuffer()' may lazily page in a source file. 203 return getContentCache(LocInfo.first)->getBuffer()->getBufferStart() + 204 LocInfo.second; 205} 206 207 208/// getColumnNumber - Return the column # for the specified file position. 209/// this is significantly cheaper to compute than the line number. This returns 210/// zero if the column number isn't known. 211unsigned SourceManager::getColumnNumber(SourceLocation Loc) const { 212 if (Loc.getChunkID() == 0) return 0; 213 214 std::pair<FileID, unsigned> LocInfo = getDecomposedFileLoc(Loc); 215 unsigned FilePos = LocInfo.second; 216 217 const char *Buf = getBuffer(LocInfo.first)->getBufferStart(); 218 219 unsigned LineStart = FilePos; 220 while (LineStart && Buf[LineStart-1] != '\n' && Buf[LineStart-1] != '\r') 221 --LineStart; 222 return FilePos-LineStart+1; 223} 224 225/// getSourceName - This method returns the name of the file or buffer that 226/// the SourceLocation specifies. This can be modified with #line directives, 227/// etc. 228const char *SourceManager::getSourceName(SourceLocation Loc) const { 229 if (Loc.getChunkID() == 0) return ""; 230 231 // To get the source name, first consult the FileEntry (if one exists) before 232 // the MemBuffer as this will avoid unnecessarily paging in the MemBuffer. 233 const SrcMgr::ContentCache *C = getContentCacheForLoc(Loc); 234 return C->Entry ? C->Entry->getName() : C->getBuffer()->getBufferIdentifier(); 235} 236 237static void ComputeLineNumbers(ContentCache* FI) DISABLE_INLINE; 238static void ComputeLineNumbers(ContentCache* FI) { 239 // Note that calling 'getBuffer()' may lazily page in the file. 240 const MemoryBuffer *Buffer = FI->getBuffer(); 241 242 // Find the file offsets of all of the *physical* source lines. This does 243 // not look at trigraphs, escaped newlines, or anything else tricky. 244 std::vector<unsigned> LineOffsets; 245 246 // Line #1 starts at char 0. 247 LineOffsets.push_back(0); 248 249 const unsigned char *Buf = (const unsigned char *)Buffer->getBufferStart(); 250 const unsigned char *End = (const unsigned char *)Buffer->getBufferEnd(); 251 unsigned Offs = 0; 252 while (1) { 253 // Skip over the contents of the line. 254 // TODO: Vectorize this? This is very performance sensitive for programs 255 // with lots of diagnostics and in -E mode. 256 const unsigned char *NextBuf = (const unsigned char *)Buf; 257 while (*NextBuf != '\n' && *NextBuf != '\r' && *NextBuf != '\0') 258 ++NextBuf; 259 Offs += NextBuf-Buf; 260 Buf = NextBuf; 261 262 if (Buf[0] == '\n' || Buf[0] == '\r') { 263 // If this is \n\r or \r\n, skip both characters. 264 if ((Buf[1] == '\n' || Buf[1] == '\r') && Buf[0] != Buf[1]) 265 ++Offs, ++Buf; 266 ++Offs, ++Buf; 267 LineOffsets.push_back(Offs); 268 } else { 269 // Otherwise, this is a null. If end of file, exit. 270 if (Buf == End) break; 271 // Otherwise, skip the null. 272 ++Offs, ++Buf; 273 } 274 } 275 276 // Copy the offsets into the FileInfo structure. 277 FI->NumLines = LineOffsets.size(); 278 FI->SourceLineCache = new unsigned[LineOffsets.size()]; 279 std::copy(LineOffsets.begin(), LineOffsets.end(), FI->SourceLineCache); 280} 281 282/// getLineNumber - Given a SourceLocation, return the spelling line number 283/// for the position indicated. This requires building and caching a table of 284/// line offsets for the MemoryBuffer, so this is not cheap: use only when 285/// about to emit a diagnostic. 286unsigned SourceManager::getLineNumber(SourceLocation Loc) const { 287 if (Loc.getChunkID() == 0) return 0; 288 289 ContentCache *Content; 290 291 std::pair<FileID, unsigned> LocInfo = getDecomposedFileLoc(Loc); 292 293 if (LastLineNoFileIDQuery == LocInfo.first) 294 Content = LastLineNoContentCache; 295 else 296 Content = const_cast<ContentCache*>(getContentCache(LocInfo.first)); 297 298 // If this is the first use of line information for this buffer, compute the 299 /// SourceLineCache for it on demand. 300 if (Content->SourceLineCache == 0) 301 ComputeLineNumbers(Content); 302 303 // Okay, we know we have a line number table. Do a binary search to find the 304 // line number that this character position lands on. 305 unsigned *SourceLineCache = Content->SourceLineCache; 306 unsigned *SourceLineCacheStart = SourceLineCache; 307 unsigned *SourceLineCacheEnd = SourceLineCache + Content->NumLines; 308 309 unsigned QueriedFilePos = LocInfo.second+1; 310 311 // If the previous query was to the same file, we know both the file pos from 312 // that query and the line number returned. This allows us to narrow the 313 // search space from the entire file to something near the match. 314 if (LastLineNoFileIDQuery == LocInfo.first) { 315 if (QueriedFilePos >= LastLineNoFilePos) { 316 SourceLineCache = SourceLineCache+LastLineNoResult-1; 317 318 // The query is likely to be nearby the previous one. Here we check to 319 // see if it is within 5, 10 or 20 lines. It can be far away in cases 320 // where big comment blocks and vertical whitespace eat up lines but 321 // contribute no tokens. 322 if (SourceLineCache+5 < SourceLineCacheEnd) { 323 if (SourceLineCache[5] > QueriedFilePos) 324 SourceLineCacheEnd = SourceLineCache+5; 325 else if (SourceLineCache+10 < SourceLineCacheEnd) { 326 if (SourceLineCache[10] > QueriedFilePos) 327 SourceLineCacheEnd = SourceLineCache+10; 328 else if (SourceLineCache+20 < SourceLineCacheEnd) { 329 if (SourceLineCache[20] > QueriedFilePos) 330 SourceLineCacheEnd = SourceLineCache+20; 331 } 332 } 333 } 334 } else { 335 SourceLineCacheEnd = SourceLineCache+LastLineNoResult+1; 336 } 337 } 338 339 // If the spread is large, do a "radix" test as our initial guess, based on 340 // the assumption that lines average to approximately the same length. 341 // NOTE: This is currently disabled, as it does not appear to be profitable in 342 // initial measurements. 343 if (0 && SourceLineCacheEnd-SourceLineCache > 20) { 344 unsigned FileLen = Content->SourceLineCache[Content->NumLines-1]; 345 346 // Take a stab at guessing where it is. 347 unsigned ApproxPos = Content->NumLines*QueriedFilePos / FileLen; 348 349 // Check for -10 and +10 lines. 350 unsigned LowerBound = std::max(int(ApproxPos-10), 0); 351 unsigned UpperBound = std::min(ApproxPos+10, FileLen); 352 353 // If the computed lower bound is less than the query location, move it in. 354 if (SourceLineCache < SourceLineCacheStart+LowerBound && 355 SourceLineCacheStart[LowerBound] < QueriedFilePos) 356 SourceLineCache = SourceLineCacheStart+LowerBound; 357 358 // If the computed upper bound is greater than the query location, move it. 359 if (SourceLineCacheEnd > SourceLineCacheStart+UpperBound && 360 SourceLineCacheStart[UpperBound] >= QueriedFilePos) 361 SourceLineCacheEnd = SourceLineCacheStart+UpperBound; 362 } 363 364 unsigned *Pos 365 = std::lower_bound(SourceLineCache, SourceLineCacheEnd, QueriedFilePos); 366 unsigned LineNo = Pos-SourceLineCacheStart; 367 368 LastLineNoFileIDQuery = LocInfo.first; 369 LastLineNoContentCache = Content; 370 LastLineNoFilePos = QueriedFilePos; 371 LastLineNoResult = LineNo; 372 return LineNo; 373} 374 375/// PrintStats - Print statistics to stderr. 376/// 377void SourceManager::PrintStats() const { 378 llvm::cerr << "\n*** Source Manager Stats:\n"; 379 llvm::cerr << FileInfos.size() << " files mapped, " << MemBufferInfos.size() 380 << " mem buffers mapped, " << FileIDs.size() 381 << " file ID's allocated.\n"; 382 llvm::cerr << " " << FileIDs.size() << " normal buffer FileID's, " 383 << MacroIDs.size() << " macro expansion FileID's.\n"; 384 385 unsigned NumLineNumsComputed = 0; 386 unsigned NumFileBytesMapped = 0; 387 for (std::set<ContentCache>::const_iterator I = 388 FileInfos.begin(), E = FileInfos.end(); I != E; ++I) { 389 NumLineNumsComputed += I->SourceLineCache != 0; 390 NumFileBytesMapped += I->getSizeBytesMapped(); 391 } 392 393 llvm::cerr << NumFileBytesMapped << " bytes of files mapped, " 394 << NumLineNumsComputed << " files with line #'s computed.\n"; 395} 396 397//===----------------------------------------------------------------------===// 398// Serialization. 399//===----------------------------------------------------------------------===// 400 401void ContentCache::Emit(llvm::Serializer& S) const { 402 S.FlushRecord(); 403 S.EmitPtr(this); 404 405 if (Entry) { 406 llvm::sys::Path Fname(Buffer->getBufferIdentifier()); 407 408 if (Fname.isAbsolute()) 409 S.EmitCStr(Fname.c_str()); 410 else { 411 // Create an absolute path. 412 // FIXME: This will potentially contain ".." and "." in the path. 413 llvm::sys::Path path = llvm::sys::Path::GetCurrentDirectory(); 414 path.appendComponent(Fname.c_str()); 415 S.EmitCStr(path.c_str()); 416 } 417 } 418 else { 419 const char* p = Buffer->getBufferStart(); 420 const char* e = Buffer->getBufferEnd(); 421 422 S.EmitInt(e-p); 423 424 for ( ; p != e; ++p) 425 S.EmitInt(*p); 426 } 427 428 S.FlushRecord(); 429} 430 431void ContentCache::ReadToSourceManager(llvm::Deserializer& D, 432 SourceManager& SMgr, 433 FileManager* FMgr, 434 std::vector<char>& Buf) { 435 if (FMgr) { 436 llvm::SerializedPtrID PtrID = D.ReadPtrID(); 437 D.ReadCStr(Buf,false); 438 439 // Create/fetch the FileEntry. 440 const char* start = &Buf[0]; 441 const FileEntry* E = FMgr->getFile(start,start+Buf.size()); 442 443 // FIXME: Ideally we want a lazy materialization of the ContentCache 444 // anyway, because we don't want to read in source files unless this 445 // is absolutely needed. 446 if (!E) 447 D.RegisterPtr(PtrID,NULL); 448 else 449 // Get the ContextCache object and register it with the deserializer. 450 D.RegisterPtr(PtrID,SMgr.getContentCache(E)); 451 } 452 else { 453 // Register the ContextCache object with the deserializer. 454 SMgr.MemBufferInfos.push_back(ContentCache()); 455 ContentCache& Entry = const_cast<ContentCache&>(SMgr.MemBufferInfos.back()); 456 D.RegisterPtr(&Entry); 457 458 // Create the buffer. 459 unsigned Size = D.ReadInt(); 460 Entry.Buffer = MemoryBuffer::getNewUninitMemBuffer(Size); 461 462 // Read the contents of the buffer. 463 char* p = const_cast<char*>(Entry.Buffer->getBufferStart()); 464 for (unsigned i = 0; i < Size ; ++i) 465 p[i] = D.ReadInt(); 466 } 467} 468 469void FileIDInfo::Emit(llvm::Serializer& S) const { 470 S.Emit(IncludeLoc); 471 S.EmitInt(ChunkNo); 472 S.EmitPtr(Content); 473} 474 475FileIDInfo FileIDInfo::ReadVal(llvm::Deserializer& D) { 476 FileIDInfo I; 477 I.IncludeLoc = SourceLocation::ReadVal(D); 478 I.ChunkNo = D.ReadInt(); 479 D.ReadPtr(I.Content,false); 480 return I; 481} 482 483void MacroIDInfo::Emit(llvm::Serializer& S) const { 484 S.Emit(InstantiationLoc); 485 S.Emit(SpellingLoc); 486} 487 488MacroIDInfo MacroIDInfo::ReadVal(llvm::Deserializer& D) { 489 MacroIDInfo I; 490 I.InstantiationLoc = SourceLocation::ReadVal(D); 491 I.SpellingLoc = SourceLocation::ReadVal(D); 492 return I; 493} 494 495void SourceManager::Emit(llvm::Serializer& S) const { 496 S.EnterBlock(); 497 S.EmitPtr(this); 498 S.EmitInt(MainFileID.getOpaqueValue()); 499 500 // Emit: FileInfos. Just emit the file name. 501 S.EnterBlock(); 502 503 std::for_each(FileInfos.begin(),FileInfos.end(), 504 S.MakeEmitter<ContentCache>()); 505 506 S.ExitBlock(); 507 508 // Emit: MemBufferInfos 509 S.EnterBlock(); 510 511 std::for_each(MemBufferInfos.begin(), MemBufferInfos.end(), 512 S.MakeEmitter<ContentCache>()); 513 514 S.ExitBlock(); 515 516 // Emit: FileIDs 517 S.EmitInt(FileIDs.size()); 518 std::for_each(FileIDs.begin(), FileIDs.end(), S.MakeEmitter<FileIDInfo>()); 519 520 // Emit: MacroIDs 521 S.EmitInt(MacroIDs.size()); 522 std::for_each(MacroIDs.begin(), MacroIDs.end(), S.MakeEmitter<MacroIDInfo>()); 523 524 S.ExitBlock(); 525} 526 527SourceManager* 528SourceManager::CreateAndRegister(llvm::Deserializer& D, FileManager& FMgr){ 529 SourceManager *M = new SourceManager(); 530 D.RegisterPtr(M); 531 532 // Read: the FileID of the main source file of the translation unit. 533 M->MainFileID = FileID::Create(D.ReadInt()); 534 535 std::vector<char> Buf; 536 537 { // Read: FileInfos. 538 llvm::Deserializer::Location BLoc = D.getCurrentBlockLocation(); 539 while (!D.FinishedBlock(BLoc)) 540 ContentCache::ReadToSourceManager(D,*M,&FMgr,Buf); 541 } 542 543 { // Read: MemBufferInfos. 544 llvm::Deserializer::Location BLoc = D.getCurrentBlockLocation(); 545 while (!D.FinishedBlock(BLoc)) 546 ContentCache::ReadToSourceManager(D,*M,NULL,Buf); 547 } 548 549 // Read: FileIDs. 550 unsigned Size = D.ReadInt(); 551 M->FileIDs.reserve(Size); 552 for (; Size > 0 ; --Size) 553 M->FileIDs.push_back(FileIDInfo::ReadVal(D)); 554 555 // Read: MacroIDs. 556 Size = D.ReadInt(); 557 M->MacroIDs.reserve(Size); 558 for (; Size > 0 ; --Size) 559 M->MacroIDs.push_back(MacroIDInfo::ReadVal(D)); 560 561 return M; 562} 563