PTHLexer.cpp revision 6353bebfc2d6d19e15f1b2c673507cdb6985270f
1//===--- PTHLexer.cpp - Lex from a token stream ---------------------------===// 2// 3// The LLVM Compiler Infrastructure 4// 5// This file is distributed under the University of Illinois Open Source 6// License. See LICENSE.TXT for details. 7// 8//===----------------------------------------------------------------------===// 9// 10// This file implements the PTHLexer interface. 11// 12//===----------------------------------------------------------------------===// 13 14#include "clang/Basic/TokenKinds.h" 15#include "clang/Basic/FileManager.h" 16#include "clang/Basic/IdentifierTable.h" 17#include "clang/Lex/PTHLexer.h" 18#include "clang/Lex/Preprocessor.h" 19#include "clang/Lex/PTHManager.h" 20#include "clang/Lex/Token.h" 21#include "clang/Lex/Preprocessor.h" 22#include "llvm/Support/Compiler.h" 23#include "llvm/Support/MemoryBuffer.h" 24#include "llvm/ADT/StringMap.h" 25#include "llvm/ADT/OwningPtr.h" 26using namespace clang; 27 28#define DISK_TOKEN_SIZE (1+1+2+4+4) 29 30//===----------------------------------------------------------------------===// 31// Utility methods for reading from the mmap'ed PTH file. 32//===----------------------------------------------------------------------===// 33 34static inline uint8_t Read8(const unsigned char *&Data) { 35 uint8_t V = Data[0]; 36 Data += 1; 37 return V; 38} 39 40static inline uint16_t Read16(const unsigned char *&Data) { 41// Targets that directly support unaligned little-endian 16-bit loads can just 42// use them. 43#if defined(__i386__) || defined(__x86_64__) 44 uint16_t V = *((uint16_t*)Data); 45#else 46 uint16_t V = ((uint16_t)Data[0] << 0) | 47 ((uint16_t)Data[1] << 8); 48#endif 49 Data += 2; 50 return V; 51} 52 53static inline uint32_t Read24(const unsigned char *&Data) { 54// Targets that directly support unaligned little-endian 16-bit loads can just 55// use them. 56#if defined(__i386__) || defined(__x86_64__) 57 uint32_t V = ((uint16_t*)Data)[0] | 58 ((uint32_t)Data[2] << 16); 59#else 60 uint32_t V = ((uint32_t)Data[0] << 0) | 61 ((uint32_t)Data[1] << 8) | 62 ((uint32_t)Data[2] << 16); 63#endif 64 65 Data += 3; 66 return V; 67} 68 69static inline uint32_t Read32(const unsigned char *&Data) { 70// Targets that directly support unaligned little-endian 32-bit loads can just 71// use them. 72#if defined(__i386__) || defined(__x86_64__) 73 uint32_t V = *((uint32_t*)Data); 74#else 75 uint32_t V = ((uint32_t)Data[0] << 0) | 76 ((uint32_t)Data[1] << 8) | 77 ((uint32_t)Data[2] << 16) | 78 ((uint32_t)Data[3] << 24); 79#endif 80 Data += 4; 81 return V; 82} 83 84 85//===----------------------------------------------------------------------===// 86// PTHLexer methods. 87//===----------------------------------------------------------------------===// 88 89PTHLexer::PTHLexer(Preprocessor &PP, FileID FID, const unsigned char *D, 90 const unsigned char *ppcond, 91 PTHSpellingSearch &mySpellingSrch, PTHManager &PM) 92 : PreprocessorLexer(&PP, FID), TokBuf(D), CurPtr(D), LastHashTokPtr(0), 93 PPCond(ppcond), CurPPCondPtr(ppcond), MySpellingSrch(mySpellingSrch), 94 PTHMgr(PM) { 95 96 FileStartLoc = PP.getSourceManager().getLocForStartOfFile(FID); 97} 98 99void PTHLexer::Lex(Token& Tok) { 100LexNextToken: 101 102 //===--------------------------------------==// 103 // Read the raw token data. 104 //===--------------------------------------==// 105 106 // Shadow CurPtr into an automatic variable. 107 const unsigned char *CurPtrShadow = CurPtr; 108 109 // Read in the data for the token. 110 unsigned Word0 = Read32(CurPtrShadow); 111 uint32_t IdentifierID = Read32(CurPtrShadow); 112 uint32_t FileOffset = Read32(CurPtrShadow); 113 114 tok::TokenKind TKind = (tok::TokenKind) (Word0 & 0xFF); 115 Token::TokenFlags TFlags = (Token::TokenFlags) ((Word0 >> 8) & 0xFF); 116 uint32_t Len = Word0 >> 16; 117 118 CurPtr = CurPtrShadow; 119 120 //===--------------------------------------==// 121 // Construct the token itself. 122 //===--------------------------------------==// 123 124 Tok.startToken(); 125 Tok.setKind(TKind); 126 Tok.setFlag(TFlags); 127 assert(!LexingRawMode); 128 if (IdentifierID) 129 Tok.setIdentifierInfo(PTHMgr.GetIdentifierInfo(IdentifierID-1)); 130 Tok.setLocation(FileStartLoc.getFileLocWithOffset(FileOffset)); 131 Tok.setLength(Len); 132 133 //===--------------------------------------==// 134 // Process the token. 135 //===--------------------------------------==// 136#if 0 137 SourceManager& SM = PP->getSourceManager(); 138 llvm::cerr << SM.getFileEntryForID(FileID)->getName() 139 << ':' << SM.getLogicalLineNumber(Tok.getLocation()) 140 << ':' << SM.getLogicalColumnNumber(Tok.getLocation()) 141 << '\n'; 142#endif 143 144 if (TKind == tok::identifier) { 145 MIOpt.ReadToken(); 146 return PP->HandleIdentifier(Tok); 147 } 148 149 if (TKind == tok::eof) { 150 // Save the end-of-file token. 151 EofToken = Tok; 152 153 Preprocessor *PPCache = PP; 154 155 assert(!ParsingPreprocessorDirective); 156 assert(!LexingRawMode); 157 158 // FIXME: Issue diagnostics similar to Lexer. 159 if (PP->HandleEndOfFile(Tok, false)) 160 return; 161 162 assert(PPCache && "Raw buffer::LexEndOfFile should return a token"); 163 return PPCache->Lex(Tok); 164 } 165 166 if (TKind == tok::hash && Tok.isAtStartOfLine()) { 167 LastHashTokPtr = CurPtr - DISK_TOKEN_SIZE; 168 assert(!LexingRawMode); 169 PP->HandleDirective(Tok); 170 171 if (PP->isCurrentLexer(this)) 172 goto LexNextToken; 173 174 return PP->Lex(Tok); 175 } 176 177 if (TKind == tok::eom) { 178 assert(ParsingPreprocessorDirective); 179 ParsingPreprocessorDirective = false; 180 return; 181 } 182 183 MIOpt.ReadToken(); 184} 185 186// FIXME: We can just grab the last token instead of storing a copy 187// into EofToken. 188void PTHLexer::getEOF(Token& Tok) { 189 assert(EofToken.is(tok::eof)); 190 Tok = EofToken; 191} 192 193void PTHLexer::DiscardToEndOfLine() { 194 assert(ParsingPreprocessorDirective && ParsingFilename == false && 195 "Must be in a preprocessing directive!"); 196 197 // We assume that if the preprocessor wishes to discard to the end of 198 // the line that it also means to end the current preprocessor directive. 199 ParsingPreprocessorDirective = false; 200 201 // Skip tokens by only peeking at their token kind and the flags. 202 // We don't need to actually reconstruct full tokens from the token buffer. 203 // This saves some copies and it also reduces IdentifierInfo* lookup. 204 const unsigned char* p = CurPtr; 205 while (1) { 206 // Read the token kind. Are we at the end of the file? 207 tok::TokenKind x = (tok::TokenKind) (uint8_t) *p; 208 if (x == tok::eof) break; 209 210 // Read the token flags. Are we at the start of the next line? 211 Token::TokenFlags y = (Token::TokenFlags) (uint8_t) p[1]; 212 if (y & Token::StartOfLine) break; 213 214 // Skip to the next token. 215 p += DISK_TOKEN_SIZE; 216 } 217 218 CurPtr = p; 219} 220 221/// SkipBlock - Used by Preprocessor to skip the current conditional block. 222bool PTHLexer::SkipBlock() { 223 assert(CurPPCondPtr && "No cached PP conditional information."); 224 assert(LastHashTokPtr && "No known '#' token."); 225 226 const unsigned char* HashEntryI = 0; 227 uint32_t Offset; 228 uint32_t TableIdx; 229 230 do { 231 // Read the token offset from the side-table. 232 Offset = Read32(CurPPCondPtr); 233 234 // Read the target table index from the side-table. 235 TableIdx = Read32(CurPPCondPtr); 236 237 // Compute the actual memory address of the '#' token data for this entry. 238 HashEntryI = TokBuf + Offset; 239 240 // Optmization: "Sibling jumping". #if...#else...#endif blocks can 241 // contain nested blocks. In the side-table we can jump over these 242 // nested blocks instead of doing a linear search if the next "sibling" 243 // entry is not at a location greater than LastHashTokPtr. 244 if (HashEntryI < LastHashTokPtr && TableIdx) { 245 // In the side-table we are still at an entry for a '#' token that 246 // is earlier than the last one we saw. Check if the location we would 247 // stride gets us closer. 248 const unsigned char* NextPPCondPtr = 249 PPCond + TableIdx*(sizeof(uint32_t)*2); 250 assert(NextPPCondPtr >= CurPPCondPtr); 251 // Read where we should jump to. 252 uint32_t TmpOffset = Read32(NextPPCondPtr); 253 const unsigned char* HashEntryJ = TokBuf + TmpOffset; 254 255 if (HashEntryJ <= LastHashTokPtr) { 256 // Jump directly to the next entry in the side table. 257 HashEntryI = HashEntryJ; 258 Offset = TmpOffset; 259 TableIdx = Read32(NextPPCondPtr); 260 CurPPCondPtr = NextPPCondPtr; 261 } 262 } 263 } 264 while (HashEntryI < LastHashTokPtr); 265 assert(HashEntryI == LastHashTokPtr && "No PP-cond entry found for '#'"); 266 assert(TableIdx && "No jumping from #endifs."); 267 268 // Update our side-table iterator. 269 const unsigned char* NextPPCondPtr = PPCond + TableIdx*(sizeof(uint32_t)*2); 270 assert(NextPPCondPtr >= CurPPCondPtr); 271 CurPPCondPtr = NextPPCondPtr; 272 273 // Read where we should jump to. 274 HashEntryI = TokBuf + Read32(NextPPCondPtr); 275 uint32_t NextIdx = Read32(NextPPCondPtr); 276 277 // By construction NextIdx will be zero if this is a #endif. This is useful 278 // to know to obviate lexing another token. 279 bool isEndif = NextIdx == 0; 280 281 // This case can occur when we see something like this: 282 // 283 // #if ... 284 // /* a comment or nothing */ 285 // #elif 286 // 287 // If we are skipping the first #if block it will be the case that CurPtr 288 // already points 'elif'. Just return. 289 290 if (CurPtr > HashEntryI) { 291 assert(CurPtr == HashEntryI + DISK_TOKEN_SIZE); 292 // Did we reach a #endif? If so, go ahead and consume that token as well. 293 if (isEndif) 294 CurPtr += DISK_TOKEN_SIZE*2; 295 else 296 LastHashTokPtr = HashEntryI; 297 298 return isEndif; 299 } 300 301 // Otherwise, we need to advance. Update CurPtr to point to the '#' token. 302 CurPtr = HashEntryI; 303 304 // Update the location of the last observed '#'. This is useful if we 305 // are skipping multiple blocks. 306 LastHashTokPtr = CurPtr; 307 308 // Skip the '#' token. 309 assert(((tok::TokenKind)*CurPtr) == tok::hash); 310 CurPtr += DISK_TOKEN_SIZE; 311 312 // Did we reach a #endif? If so, go ahead and consume that token as well. 313 if (isEndif) { CurPtr += DISK_TOKEN_SIZE*2; } 314 315 return isEndif; 316} 317 318SourceLocation PTHLexer::getSourceLocation() { 319 // getSourceLocation is not on the hot path. It is used to get the location 320 // of the next token when transitioning back to this lexer when done 321 // handling a #included file. Just read the necessary data from the token 322 // data buffer to construct the SourceLocation object. 323 // NOTE: This is a virtual function; hence it is defined out-of-line. 324 const unsigned char *OffsetPtr = CurPtr + (1 + 1 + 3); 325 uint32_t Offset = Read32(OffsetPtr); 326 return FileStartLoc.getFileLocWithOffset(Offset); 327} 328 329//===----------------------------------------------------------------------===// 330// getSpelling() - Use cached data in PTH files for getSpelling(). 331//===----------------------------------------------------------------------===// 332 333unsigned PTHManager::getSpelling(FileID FID, unsigned FPos, 334 const char *&Buffer) { 335 llvm::DenseMap<FileID, PTHSpellingSearch*>::iterator I =SpellingMap.find(FID); 336 337 if (I == SpellingMap.end()) 338 return 0; 339 340 return I->second->getSpellingBinarySearch(FPos, Buffer); 341} 342 343unsigned PTHManager::getSpelling(SourceLocation Loc, const char *&Buffer) { 344 SourceManager &SM = PP->getSourceManager(); 345 Loc = SM.getSpellingLoc(Loc); 346 std::pair<FileID, unsigned> LocInfo = SM.getDecomposedFileLoc(Loc); 347 return getSpelling(LocInfo.first, LocInfo.second, Buffer); 348} 349 350unsigned PTHManager::getSpellingAtPTHOffset(unsigned PTHOffset, 351 const char *&Buffer) { 352 assert(PTHOffset < Buf->getBufferSize()); 353 const unsigned char* Ptr = 354 (const unsigned char*)Buf->getBufferStart() + PTHOffset; 355 356 // The string is prefixed by 16 bits for its length, followed by the string 357 // itself. 358 unsigned Len = Read16(Ptr); 359 Buffer = (const char *)Ptr; 360 return Len; 361} 362 363unsigned PTHSpellingSearch::getSpellingLinearSearch(unsigned FPos, 364 const char *&Buffer) { 365 const unsigned char *Ptr = LinearItr; 366 unsigned Len = 0; 367 368 if (Ptr == TableEnd) 369 return getSpellingBinarySearch(FPos, Buffer); 370 371 do { 372 uint32_t TokOffset = Read32(Ptr); 373 374 if (TokOffset > FPos) 375 return getSpellingBinarySearch(FPos, Buffer); 376 377 // Did we find a matching token offset for this spelling? 378 if (TokOffset == FPos) { 379 uint32_t SpellingPTHOffset = Read32(Ptr); 380 Len = PTHMgr.getSpellingAtPTHOffset(SpellingPTHOffset, Buffer); 381 break; 382 } 383 } while (Ptr != TableEnd); 384 385 LinearItr = Ptr; 386 return Len; 387} 388 389 390unsigned PTHSpellingSearch::getSpellingBinarySearch(unsigned FPos, 391 const char *&Buffer) { 392 393 assert((TableEnd - TableBeg) % SpellingEntrySize == 0); 394 assert(TableEnd >= TableBeg); 395 396 if (TableEnd == TableBeg) 397 return 0; 398 399 unsigned min = 0; 400 const unsigned char *tb = TableBeg; 401 unsigned max = NumSpellings; 402 403 do { 404 unsigned i = (max - min) / 2 + min; 405 const unsigned char *Ptr = tb + (i * SpellingEntrySize); 406 407 uint32_t TokOffset = Read32(Ptr); 408 if (TokOffset > FPos) { 409 max = i; 410 assert(!(max == min) || (min == i)); 411 continue; 412 } 413 414 if (TokOffset < FPos) { 415 if (i == min) 416 break; 417 418 min = i; 419 continue; 420 } 421 422 uint32_t SpellingPTHOffset = Read32(Ptr); 423 return PTHMgr.getSpellingAtPTHOffset(SpellingPTHOffset, Buffer); 424 } 425 while (min != max); 426 427 return 0; 428} 429 430unsigned PTHLexer::getSpelling(SourceLocation Loc, const char *&Buffer) { 431 SourceManager &SM = PP->getSourceManager(); 432 Loc = SM.getSpellingLoc(Loc); 433 std::pair<FileID, unsigned> LocInfo = SM.getDecomposedFileLoc(Loc); 434 435 FileID FID = LocInfo.first; 436 unsigned FPos = LocInfo.second; 437 438 if (FID == getFileID()) 439 return MySpellingSrch.getSpellingLinearSearch(FPos, Buffer); 440 return PTHMgr.getSpelling(FID, FPos, Buffer); 441} 442 443//===----------------------------------------------------------------------===// 444// Internal Data Structures for PTH file lookup and resolving identifiers. 445//===----------------------------------------------------------------------===// 446 447 448/// PTHFileLookup - This internal data structure is used by the PTHManager 449/// to map from FileEntry objects managed by FileManager to offsets within 450/// the PTH file. 451namespace { 452class VISIBILITY_HIDDEN PTHFileLookup { 453public: 454 class Val { 455 uint32_t TokenOff; 456 uint32_t PPCondOff; 457 uint32_t SpellingOff; 458 public: 459 Val() : TokenOff(~0) {} 460 Val(uint32_t toff, uint32_t poff, uint32_t soff) 461 : TokenOff(toff), PPCondOff(poff), SpellingOff(soff) {} 462 463 bool isValid() const { return TokenOff != ~((uint32_t)0); } 464 465 uint32_t getTokenOffset() const { 466 assert(isValid() && "PTHFileLookup entry initialized."); 467 return TokenOff; 468 } 469 470 uint32_t getPPCondOffset() const { 471 assert(isValid() && "PTHFileLookup entry initialized."); 472 return PPCondOff; 473 } 474 475 uint32_t getSpellingOffset() const { 476 assert(isValid() && "PTHFileLookup entry initialized."); 477 return SpellingOff; 478 } 479 }; 480 481private: 482 llvm::StringMap<Val> FileMap; 483 484public: 485 PTHFileLookup() {}; 486 487 Val Lookup(const FileEntry* FE) { 488 const char* s = FE->getName(); 489 unsigned size = strlen(s); 490 return FileMap.GetOrCreateValue(s, s+size).getValue(); 491 } 492 493 void ReadTable(const unsigned char* D) { 494 uint32_t N = Read32(D); // Read the length of the table. 495 496 for ( ; N > 0; --N) { // The rest of the data is the table itself. 497 uint32_t Len = Read32(D); 498 const char* s = (const char *)D; 499 D += Len; 500 501 uint32_t TokenOff = Read32(D); 502 uint32_t PPCondOff = Read32(D); 503 uint32_t SpellingOff = Read32(D); 504 505 FileMap.GetOrCreateValue(s, s+Len).getValue() = 506 Val(TokenOff, PPCondOff, SpellingOff); 507 } 508 } 509}; 510} // end anonymous namespace 511 512//===----------------------------------------------------------------------===// 513// PTHManager methods. 514//===----------------------------------------------------------------------===// 515 516PTHManager::PTHManager(const llvm::MemoryBuffer* buf, void* fileLookup, 517 const unsigned char* idDataTable, 518 IdentifierInfo** perIDCache, 519 const unsigned char* sortedIdTable, unsigned numIds) 520: Buf(buf), PerIDCache(perIDCache), FileLookup(fileLookup), 521 IdDataTable(idDataTable), SortedIdTable(sortedIdTable), 522 NumIds(numIds), PP(0) {} 523 524PTHManager::~PTHManager() { 525 delete Buf; 526 delete (PTHFileLookup*) FileLookup; 527 free(PerIDCache); 528} 529 530PTHManager* PTHManager::Create(const std::string& file) { 531 // Memory map the PTH file. 532 llvm::OwningPtr<llvm::MemoryBuffer> 533 File(llvm::MemoryBuffer::getFile(file.c_str())); 534 535 if (!File) 536 return 0; 537 538 // Get the buffer ranges and check if there are at least three 32-bit 539 // words at the end of the file. 540 const unsigned char* BufBeg = (unsigned char*)File->getBufferStart(); 541 const unsigned char* BufEnd = (unsigned char*)File->getBufferEnd(); 542 543 if(!(BufEnd > BufBeg + sizeof(uint32_t)*3)) { 544 assert(false && "Invalid PTH file."); 545 return 0; // FIXME: Proper error diagnostic? 546 } 547 548 // Compute the address of the index table at the end of the PTH file. 549 // This table contains the offset of the file lookup table, the 550 // persistent ID -> identifer data table. 551 // FIXME: We should just embed this offset in the PTH file. 552 const unsigned char* EndTable = BufEnd - sizeof(uint32_t)*4; 553 554 // Construct the file lookup table. This will be used for mapping from 555 // FileEntry*'s to cached tokens. 556 const unsigned char* FileTableOffset = EndTable + sizeof(uint32_t)*3; 557 const unsigned char* FileTable = BufBeg + Read32(FileTableOffset); 558 559 if (!(FileTable > BufBeg && FileTable < BufEnd)) { 560 assert(false && "Invalid PTH file."); 561 return 0; // FIXME: Proper error diagnostic? 562 } 563 564 llvm::OwningPtr<PTHFileLookup> FL(new PTHFileLookup()); 565 FL->ReadTable(FileTable); 566 567 // Get the location of the table mapping from persistent ids to the 568 // data needed to reconstruct identifiers. 569 const unsigned char* IDTableOffset = EndTable + sizeof(uint32_t)*1; 570 const unsigned char* IData = BufBeg + Read32(IDTableOffset); 571 if (!(IData > BufBeg && IData < BufEnd)) { 572 assert(false && "Invalid PTH file."); 573 return 0; // FIXME: Proper error diagnostic? 574 } 575 576 // Get the location of the lexigraphically-sorted table of persistent IDs. 577 const unsigned char* SortedIdTableOffset = EndTable + sizeof(uint32_t)*2; 578 const unsigned char* SortedIdTable = BufBeg + Read32(SortedIdTableOffset); 579 if (!(SortedIdTable > BufBeg && SortedIdTable < BufEnd)) { 580 assert(false && "Invalid PTH file."); 581 return 0; // FIXME: Proper error diagnostic? 582 } 583 584 // Get the number of IdentifierInfos and pre-allocate the identifier cache. 585 uint32_t NumIds = Read32(IData); 586 587 // Pre-allocate the peristent ID -> IdentifierInfo* cache. We use calloc() 588 // so that we in the best case only zero out memory once when the OS returns 589 // us new pages. 590 IdentifierInfo** PerIDCache = 591 (IdentifierInfo**)calloc(NumIds, sizeof(*PerIDCache)); 592 593 if (!PerIDCache) { 594 assert(false && "Could not allocate Persistent ID cache."); 595 return 0; 596 } 597 598 // Create the new PTHManager. 599 return new PTHManager(File.take(), FL.take(), IData, PerIDCache, 600 SortedIdTable, NumIds); 601} 602IdentifierInfo* PTHManager::LazilyCreateIdentifierInfo(unsigned PersistentID) { 603 // Look in the PTH file for the string data for the IdentifierInfo object. 604 const unsigned char* TableEntry = IdDataTable + sizeof(uint32_t)*PersistentID; 605 const unsigned char* IDData = 606 (const unsigned char*)Buf->getBufferStart() + Read32(TableEntry); 607 assert(IDData < (const unsigned char*)Buf->getBufferEnd()); 608 609 // Allocate the object. 610 std::pair<IdentifierInfo,const unsigned char*> *Mem = 611 Alloc.Allocate<std::pair<IdentifierInfo,const unsigned char*> >(); 612 613 Mem->second = IDData; 614 IdentifierInfo *II = new ((void*) Mem) IdentifierInfo(); 615 616 // Store the new IdentifierInfo in the cache. 617 PerIDCache[PersistentID] = II; 618 return II; 619} 620 621IdentifierInfo* PTHManager::get(const char *NameStart, const char *NameEnd) { 622 unsigned min = 0; 623 unsigned max = NumIds; 624 unsigned Len = NameEnd - NameStart; 625 626 do { 627 unsigned i = (max - min) / 2 + min; 628 const unsigned char *Ptr = SortedIdTable + (i * 4); 629 630 // Read the persistentID. 631 unsigned perID = Read32(Ptr); 632 633 // Get the IdentifierInfo. 634 IdentifierInfo* II = GetIdentifierInfo(perID); 635 636 // First compare the lengths. 637 unsigned IILen = II->getLength(); 638 if (Len < IILen) goto IsLess; 639 if (Len > IILen) goto IsGreater; 640 641 // Now compare the strings! 642 { 643 signed comp = strncmp(NameStart, II->getName(), Len); 644 if (comp < 0) goto IsLess; 645 if (comp > 0) goto IsGreater; 646 } 647 // We found a match! 648 return II; 649 650 IsGreater: 651 if (i == min) break; 652 min = i; 653 continue; 654 655 IsLess: 656 max = i; 657 assert(!(max == min) || (min == i)); 658 } 659 while (min != max); 660 661 return 0; 662} 663 664 665PTHLexer *PTHManager::CreateLexer(FileID FID) { 666 const FileEntry *FE = PP->getSourceManager().getFileEntryForID(FID); 667 if (!FE) 668 return 0; 669 670 // Lookup the FileEntry object in our file lookup data structure. It will 671 // return a variant that indicates whether or not there is an offset within 672 // the PTH file that contains cached tokens. 673 PTHFileLookup::Val FileData = ((PTHFileLookup*)FileLookup)->Lookup(FE); 674 675 if (!FileData.isValid()) // No tokens available. 676 return 0; 677 678 const unsigned char *BufStart = (const unsigned char *)Buf->getBufferStart(); 679 // Compute the offset of the token data within the buffer. 680 const unsigned char* data = BufStart + FileData.getTokenOffset(); 681 682 // Get the location of pp-conditional table. 683 const unsigned char* ppcond = BufStart + FileData.getPPCondOffset(); 684 uint32_t Len = Read32(ppcond); 685 if (Len == 0) ppcond = 0; 686 687 // Get the location of the spelling table. 688 const unsigned char* spellingTable = BufStart + FileData.getSpellingOffset(); 689 690 Len = Read32(spellingTable); 691 if (Len == 0) spellingTable = 0; 692 693 assert(data < (const unsigned char*)Buf->getBufferEnd()); 694 695 // Create the SpellingSearch object for this FileID. 696 PTHSpellingSearch* ss = new PTHSpellingSearch(*this, Len, spellingTable); 697 SpellingMap[FID] = ss; 698 699 assert(PP && "No preprocessor set yet!"); 700 return new PTHLexer(*PP, FID, data, ppcond, *ss, *this); 701} 702