PTHLexer.cpp revision 30a12ec2a7f331d9e08acabe7cda853aaa7ba54b
1274b20863a728cc6a31ee75c670e3733600c1531Ted Kremenek//===--- PTHLexer.cpp - Lex from a token stream ---------------------------===// 2274b20863a728cc6a31ee75c670e3733600c1531Ted Kremenek// 3274b20863a728cc6a31ee75c670e3733600c1531Ted Kremenek// The LLVM Compiler Infrastructure 4274b20863a728cc6a31ee75c670e3733600c1531Ted Kremenek// 5274b20863a728cc6a31ee75c670e3733600c1531Ted Kremenek// This file is distributed under the University of Illinois Open Source 6274b20863a728cc6a31ee75c670e3733600c1531Ted Kremenek// License. See LICENSE.TXT for details. 7274b20863a728cc6a31ee75c670e3733600c1531Ted Kremenek// 8274b20863a728cc6a31ee75c670e3733600c1531Ted Kremenek//===----------------------------------------------------------------------===// 9274b20863a728cc6a31ee75c670e3733600c1531Ted Kremenek// 10274b20863a728cc6a31ee75c670e3733600c1531Ted Kremenek// This file implements the PTHLexer interface. 11274b20863a728cc6a31ee75c670e3733600c1531Ted Kremenek// 12274b20863a728cc6a31ee75c670e3733600c1531Ted Kremenek//===----------------------------------------------------------------------===// 13274b20863a728cc6a31ee75c670e3733600c1531Ted Kremenek 140c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek#include "clang/Basic/TokenKinds.h" 150c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek#include "clang/Basic/FileManager.h" 160c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek#include "clang/Basic/IdentifierTable.h" 17274b20863a728cc6a31ee75c670e3733600c1531Ted Kremenek#include "clang/Lex/PTHLexer.h" 18274b20863a728cc6a31ee75c670e3733600c1531Ted Kremenek#include "clang/Lex/Preprocessor.h" 190c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek#include "clang/Lex/PTHManager.h" 200c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek#include "clang/Lex/Token.h" 210c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek#include "clang/Lex/Preprocessor.h" 220c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek#include "llvm/Support/Compiler.h" 230c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek#include "llvm/Support/MemoryBuffer.h" 240c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek#include "llvm/ADT/StringMap.h" 250c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek#include "llvm/ADT/OwningPtr.h" 260c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek 27274b20863a728cc6a31ee75c670e3733600c1531Ted Kremenekusing namespace clang; 28274b20863a728cc6a31ee75c670e3733600c1531Ted Kremenek 29268ee7016a2811803989487c0ad3799486092c63Ted Kremenek#define DISK_TOKEN_SIZE (2+3*4) 30268ee7016a2811803989487c0ad3799486092c63Ted Kremenek 310c6a77bc1f52f282a969538f139ebde429076ed3Ted KremenekPTHLexer::PTHLexer(Preprocessor& pp, SourceLocation fileloc, const char* D, 32268ee7016a2811803989487c0ad3799486092c63Ted Kremenek const char* ppcond, PTHManager& PM) 33cd223444d1680290efe11da657faafc9a1ac14baTed Kremenek : PreprocessorLexer(&pp, fileloc), TokBuf(D), CurPtr(D), LastHashTokPtr(0), 34268ee7016a2811803989487c0ad3799486092c63Ted Kremenek PPCond(ppcond), CurPPCondPtr(ppcond), PTHMgr(PM), NeedsFetching(true) { 350c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek // Make sure the EofToken is completely clean. 360c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek EofToken.startToken(); 370c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek } 38274b20863a728cc6a31ee75c670e3733600c1531Ted Kremenek 390c6a77bc1f52f282a969538f139ebde429076ed3Ted KremenekToken PTHLexer::GetToken() { 400c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek // Read the next token, or if we haven't advanced yet, get the last 410c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek // token read. 420c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek if (NeedsFetching) { 430c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek NeedsFetching = false; 440c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek ReadToken(LastFetched); 450c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek } 460c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek 470c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek Token Tok = LastFetched; 4889d7ee9619d2dbdfa8d956a695c612a104a92cadTed Kremenek 4989d7ee9619d2dbdfa8d956a695c612a104a92cadTed Kremenek // If we are in raw mode, zero out identifier pointers. This is 5089d7ee9619d2dbdfa8d956a695c612a104a92cadTed Kremenek // needed for 'pragma poison'. Note that this requires that the Preprocessor 5189d7ee9619d2dbdfa8d956a695c612a104a92cadTed Kremenek // can go back to the original source when it calls getSpelling(). 5289d7ee9619d2dbdfa8d956a695c612a104a92cadTed Kremenek if (LexingRawMode && Tok.is(tok::identifier)) 5389d7ee9619d2dbdfa8d956a695c612a104a92cadTed Kremenek Tok.setIdentifierInfo(0); 5489d7ee9619d2dbdfa8d956a695c612a104a92cadTed Kremenek 5589d7ee9619d2dbdfa8d956a695c612a104a92cadTed Kremenek return Tok; 5689d7ee9619d2dbdfa8d956a695c612a104a92cadTed Kremenek} 5789d7ee9619d2dbdfa8d956a695c612a104a92cadTed Kremenek 58274b20863a728cc6a31ee75c670e3733600c1531Ted Kremenekvoid PTHLexer::Lex(Token& Tok) { 59d6f53dc4951aace69014619761760addac9e59ecTed KremenekLexNextToken: 60cd4e2aecde5bb7810715d5d5a88ac63ce7946f34Ted Kremenek Tok = GetToken(); 61cd4e2aecde5bb7810715d5d5a88ac63ce7946f34Ted Kremenek 6231aba425a01c8c957e662ccfaa71f923d0f0932aTed Kremenek if (AtLastToken()) { 63cd4e2aecde5bb7810715d5d5a88ac63ce7946f34Ted Kremenek Preprocessor *PPCache = PP; 64cd4e2aecde5bb7810715d5d5a88ac63ce7946f34Ted Kremenek 65cd4e2aecde5bb7810715d5d5a88ac63ce7946f34Ted Kremenek if (LexEndOfFile(Tok)) 66d6f53dc4951aace69014619761760addac9e59ecTed Kremenek return; 67274b20863a728cc6a31ee75c670e3733600c1531Ted Kremenek 68cd4e2aecde5bb7810715d5d5a88ac63ce7946f34Ted Kremenek assert(PPCache && "Raw buffer::LexEndOfFile should return a token"); 69cd4e2aecde5bb7810715d5d5a88ac63ce7946f34Ted Kremenek return PPCache->Lex(Tok); 70cd4e2aecde5bb7810715d5d5a88ac63ce7946f34Ted Kremenek } 71274b20863a728cc6a31ee75c670e3733600c1531Ted Kremenek 72d6f53dc4951aace69014619761760addac9e59ecTed Kremenek // Don't advance to the next token yet. Check if we are at the 73d6f53dc4951aace69014619761760addac9e59ecTed Kremenek // start of a new line and we're processing a directive. If so, we 74d6f53dc4951aace69014619761760addac9e59ecTed Kremenek // consume this token twice, once as an tok::eom. 75d6f53dc4951aace69014619761760addac9e59ecTed Kremenek if (Tok.isAtStartOfLine() && ParsingPreprocessorDirective) { 76d6f53dc4951aace69014619761760addac9e59ecTed Kremenek ParsingPreprocessorDirective = false; 77d6f53dc4951aace69014619761760addac9e59ecTed Kremenek Tok.setKind(tok::eom); 78274b20863a728cc6a31ee75c670e3733600c1531Ted Kremenek MIOpt.ReadToken(); 79274b20863a728cc6a31ee75c670e3733600c1531Ted Kremenek return; 80274b20863a728cc6a31ee75c670e3733600c1531Ted Kremenek } 81d6f53dc4951aace69014619761760addac9e59ecTed Kremenek 82d6f53dc4951aace69014619761760addac9e59ecTed Kremenek // Advance to the next token. 8331aba425a01c8c957e662ccfaa71f923d0f0932aTed Kremenek AdvanceToken(); 84d6f53dc4951aace69014619761760addac9e59ecTed Kremenek 85d6f53dc4951aace69014619761760addac9e59ecTed Kremenek if (Tok.is(tok::hash)) { 86268ee7016a2811803989487c0ad3799486092c63Ted Kremenek if (Tok.isAtStartOfLine()) { 87268ee7016a2811803989487c0ad3799486092c63Ted Kremenek LastHashTokPtr = CurPtr - DISK_TOKEN_SIZE; 88268ee7016a2811803989487c0ad3799486092c63Ted Kremenek if (!LexingRawMode) { 89268ee7016a2811803989487c0ad3799486092c63Ted Kremenek PP->HandleDirective(Tok); 90d6f53dc4951aace69014619761760addac9e59ecTed Kremenek 91268ee7016a2811803989487c0ad3799486092c63Ted Kremenek if (PP->isCurrentLexer(this)) 92268ee7016a2811803989487c0ad3799486092c63Ted Kremenek goto LexNextToken; 93268ee7016a2811803989487c0ad3799486092c63Ted Kremenek 94268ee7016a2811803989487c0ad3799486092c63Ted Kremenek return PP->Lex(Tok); 95268ee7016a2811803989487c0ad3799486092c63Ted Kremenek } 96d6f53dc4951aace69014619761760addac9e59ecTed Kremenek } 97274b20863a728cc6a31ee75c670e3733600c1531Ted Kremenek } 98d6f53dc4951aace69014619761760addac9e59ecTed Kremenek 99274b20863a728cc6a31ee75c670e3733600c1531Ted Kremenek MIOpt.ReadToken(); 100d6f53dc4951aace69014619761760addac9e59ecTed Kremenek 101d6f53dc4951aace69014619761760addac9e59ecTed Kremenek if (Tok.is(tok::identifier)) { 102d6f53dc4951aace69014619761760addac9e59ecTed Kremenek if (LexingRawMode) return; 103d6f53dc4951aace69014619761760addac9e59ecTed Kremenek return PP->HandleIdentifier(Tok); 104d6f53dc4951aace69014619761760addac9e59ecTed Kremenek } 105274b20863a728cc6a31ee75c670e3733600c1531Ted Kremenek} 106274b20863a728cc6a31ee75c670e3733600c1531Ted Kremenek 107cd4e2aecde5bb7810715d5d5a88ac63ce7946f34Ted Kremenekbool PTHLexer::LexEndOfFile(Token &Tok) { 108cd4e2aecde5bb7810715d5d5a88ac63ce7946f34Ted Kremenek 109cd4e2aecde5bb7810715d5d5a88ac63ce7946f34Ted Kremenek if (ParsingPreprocessorDirective) { 110cd4e2aecde5bb7810715d5d5a88ac63ce7946f34Ted Kremenek ParsingPreprocessorDirective = false; 111cd4e2aecde5bb7810715d5d5a88ac63ce7946f34Ted Kremenek Tok.setKind(tok::eom); 112cd4e2aecde5bb7810715d5d5a88ac63ce7946f34Ted Kremenek MIOpt.ReadToken(); 113cd4e2aecde5bb7810715d5d5a88ac63ce7946f34Ted Kremenek return true; // Have a token. 114cd4e2aecde5bb7810715d5d5a88ac63ce7946f34Ted Kremenek } 115cd4e2aecde5bb7810715d5d5a88ac63ce7946f34Ted Kremenek 116cd4e2aecde5bb7810715d5d5a88ac63ce7946f34Ted Kremenek if (LexingRawMode) { 117cd4e2aecde5bb7810715d5d5a88ac63ce7946f34Ted Kremenek MIOpt.ReadToken(); 118cd4e2aecde5bb7810715d5d5a88ac63ce7946f34Ted Kremenek return true; // Have an eof token. 119cd4e2aecde5bb7810715d5d5a88ac63ce7946f34Ted Kremenek } 120cd4e2aecde5bb7810715d5d5a88ac63ce7946f34Ted Kremenek 121cd4e2aecde5bb7810715d5d5a88ac63ce7946f34Ted Kremenek // FIXME: Issue diagnostics similar to Lexer. 122cd4e2aecde5bb7810715d5d5a88ac63ce7946f34Ted Kremenek return PP->HandleEndOfFile(Tok, false); 123cd4e2aecde5bb7810715d5d5a88ac63ce7946f34Ted Kremenek} 124cd4e2aecde5bb7810715d5d5a88ac63ce7946f34Ted Kremenek 125274b20863a728cc6a31ee75c670e3733600c1531Ted Kremenekvoid PTHLexer::setEOF(Token& Tok) { 1260c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek assert(!EofToken.is(tok::eof)); 1270c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek Tok = EofToken; 128274b20863a728cc6a31ee75c670e3733600c1531Ted Kremenek} 12917ff58a63197b398ae52697b088dc0fb8b255519Ted Kremenek 13017ff58a63197b398ae52697b088dc0fb8b255519Ted Kremenekvoid PTHLexer::DiscardToEndOfLine() { 13117ff58a63197b398ae52697b088dc0fb8b255519Ted Kremenek assert(ParsingPreprocessorDirective && ParsingFilename == false && 13217ff58a63197b398ae52697b088dc0fb8b255519Ted Kremenek "Must be in a preprocessing directive!"); 1334d35da2e41941965bbee8ed7e8c30e7c21000d71Ted Kremenek 1344d35da2e41941965bbee8ed7e8c30e7c21000d71Ted Kremenek // Already at end-of-file? 13531aba425a01c8c957e662ccfaa71f923d0f0932aTed Kremenek if (AtLastToken()) 1364d35da2e41941965bbee8ed7e8c30e7c21000d71Ted Kremenek return; 1374d35da2e41941965bbee8ed7e8c30e7c21000d71Ted Kremenek 1384d35da2e41941965bbee8ed7e8c30e7c21000d71Ted Kremenek // Find the first token that is not the start of the *current* line. 139d2bdeed0727c291a1a0037dc4fa80379dafa0687Ted Kremenek Token T; 140d2bdeed0727c291a1a0037dc4fa80379dafa0687Ted Kremenek for (Lex(T); !AtLastToken(); Lex(T)) 14131aba425a01c8c957e662ccfaa71f923d0f0932aTed Kremenek if (GetToken().isAtStartOfLine()) 1424d35da2e41941965bbee8ed7e8c30e7c21000d71Ted Kremenek return; 14317ff58a63197b398ae52697b088dc0fb8b255519Ted Kremenek} 1440c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek 1450c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek//===----------------------------------------------------------------------===// 1460c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek// Utility methods for reading from the mmap'ed PTH file. 1470c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek//===----------------------------------------------------------------------===// 1480c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek 1490c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenekstatic inline uint8_t Read8(const char*& data) { 1500c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek return (uint8_t) *(data++); 1510c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek} 1520c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek 1530c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenekstatic inline uint32_t Read32(const char*& data) { 1540c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek uint32_t V = (uint32_t) Read8(data); 1550c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek V |= (((uint32_t) Read8(data)) << 8); 1560c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek V |= (((uint32_t) Read8(data)) << 16); 1570c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek V |= (((uint32_t) Read8(data)) << 24); 1580c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek return V; 1590c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek} 1600c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek 161268ee7016a2811803989487c0ad3799486092c63Ted Kremenek/// SkipBlock - Used by Preprocessor to skip the current conditional block. 162268ee7016a2811803989487c0ad3799486092c63Ted Kremenekbool PTHLexer::SkipBlock() { 163268ee7016a2811803989487c0ad3799486092c63Ted Kremenek assert(CurPPCondPtr && "No cached PP conditional information."); 164268ee7016a2811803989487c0ad3799486092c63Ted Kremenek assert(LastHashTokPtr && "No known '#' token."); 165268ee7016a2811803989487c0ad3799486092c63Ted Kremenek 16641a2660377d215d004fe413c03874bd066b5384cTed Kremenek const char* HashEntryI = 0; 167268ee7016a2811803989487c0ad3799486092c63Ted Kremenek uint32_t Offset; 168268ee7016a2811803989487c0ad3799486092c63Ted Kremenek uint32_t TableIdx; 169268ee7016a2811803989487c0ad3799486092c63Ted Kremenek 170268ee7016a2811803989487c0ad3799486092c63Ted Kremenek do { 17141a2660377d215d004fe413c03874bd066b5384cTed Kremenek // Read the token offset from the side-table. 172268ee7016a2811803989487c0ad3799486092c63Ted Kremenek Offset = Read32(CurPPCondPtr); 17341a2660377d215d004fe413c03874bd066b5384cTed Kremenek 17441a2660377d215d004fe413c03874bd066b5384cTed Kremenek // Read the target table index from the side-table. 175268ee7016a2811803989487c0ad3799486092c63Ted Kremenek TableIdx = Read32(CurPPCondPtr); 17641a2660377d215d004fe413c03874bd066b5384cTed Kremenek 17741a2660377d215d004fe413c03874bd066b5384cTed Kremenek // Compute the actual memory address of the '#' token data for this entry. 17841a2660377d215d004fe413c03874bd066b5384cTed Kremenek HashEntryI = TokBuf + Offset; 17941a2660377d215d004fe413c03874bd066b5384cTed Kremenek 18041a2660377d215d004fe413c03874bd066b5384cTed Kremenek // Optmization: "Sibling jumping". #if...#else...#endif blocks can 18141a2660377d215d004fe413c03874bd066b5384cTed Kremenek // contain nested blocks. In the side-table we can jump over these 18241a2660377d215d004fe413c03874bd066b5384cTed Kremenek // nested blocks instead of doing a linear search if the next "sibling" 18341a2660377d215d004fe413c03874bd066b5384cTed Kremenek // entry is not at a location greater than LastHashTokPtr. 18441a2660377d215d004fe413c03874bd066b5384cTed Kremenek if (HashEntryI < LastHashTokPtr && TableIdx) { 18541a2660377d215d004fe413c03874bd066b5384cTed Kremenek // In the side-table we are still at an entry for a '#' token that 18641a2660377d215d004fe413c03874bd066b5384cTed Kremenek // is earlier than the last one we saw. Check if the location we would 18741a2660377d215d004fe413c03874bd066b5384cTed Kremenek // stride gets us closer. 18841a2660377d215d004fe413c03874bd066b5384cTed Kremenek const char* NextPPCondPtr = PPCond + TableIdx*(sizeof(uint32_t)*2); 18941a2660377d215d004fe413c03874bd066b5384cTed Kremenek assert(NextPPCondPtr >= CurPPCondPtr); 19041a2660377d215d004fe413c03874bd066b5384cTed Kremenek // Read where we should jump to. 19141a2660377d215d004fe413c03874bd066b5384cTed Kremenek uint32_t TmpOffset = Read32(NextPPCondPtr); 19241a2660377d215d004fe413c03874bd066b5384cTed Kremenek const char* HashEntryJ = TokBuf + TmpOffset; 19341a2660377d215d004fe413c03874bd066b5384cTed Kremenek 19441a2660377d215d004fe413c03874bd066b5384cTed Kremenek if (HashEntryJ <= LastHashTokPtr) { 19541a2660377d215d004fe413c03874bd066b5384cTed Kremenek // Jump directly to the next entry in the side table. 19641a2660377d215d004fe413c03874bd066b5384cTed Kremenek HashEntryI = HashEntryJ; 19741a2660377d215d004fe413c03874bd066b5384cTed Kremenek Offset = TmpOffset; 19841a2660377d215d004fe413c03874bd066b5384cTed Kremenek TableIdx = Read32(NextPPCondPtr); 19941a2660377d215d004fe413c03874bd066b5384cTed Kremenek CurPPCondPtr = NextPPCondPtr; 20041a2660377d215d004fe413c03874bd066b5384cTed Kremenek } 20141a2660377d215d004fe413c03874bd066b5384cTed Kremenek } 202268ee7016a2811803989487c0ad3799486092c63Ted Kremenek } 20341a2660377d215d004fe413c03874bd066b5384cTed Kremenek while (HashEntryI < LastHashTokPtr); 20441a2660377d215d004fe413c03874bd066b5384cTed Kremenek assert(HashEntryI == LastHashTokPtr && "No PP-cond entry found for '#'"); 205268ee7016a2811803989487c0ad3799486092c63Ted Kremenek assert(TableIdx && "No jumping from #endifs."); 206268ee7016a2811803989487c0ad3799486092c63Ted Kremenek 207268ee7016a2811803989487c0ad3799486092c63Ted Kremenek // Update our side-table iterator. 208268ee7016a2811803989487c0ad3799486092c63Ted Kremenek const char* NextPPCondPtr = PPCond + TableIdx*(sizeof(uint32_t)*2); 209268ee7016a2811803989487c0ad3799486092c63Ted Kremenek assert(NextPPCondPtr >= CurPPCondPtr); 210268ee7016a2811803989487c0ad3799486092c63Ted Kremenek CurPPCondPtr = NextPPCondPtr; 211268ee7016a2811803989487c0ad3799486092c63Ted Kremenek 212268ee7016a2811803989487c0ad3799486092c63Ted Kremenek // Read where we should jump to. 21341a2660377d215d004fe413c03874bd066b5384cTed Kremenek HashEntryI = TokBuf + Read32(NextPPCondPtr); 214268ee7016a2811803989487c0ad3799486092c63Ted Kremenek uint32_t NextIdx = Read32(NextPPCondPtr); 215268ee7016a2811803989487c0ad3799486092c63Ted Kremenek 216268ee7016a2811803989487c0ad3799486092c63Ted Kremenek // By construction NextIdx will be zero if this is a #endif. This is useful 217268ee7016a2811803989487c0ad3799486092c63Ted Kremenek // to know to obviate lexing another token. 218268ee7016a2811803989487c0ad3799486092c63Ted Kremenek bool isEndif = NextIdx == 0; 219268ee7016a2811803989487c0ad3799486092c63Ted Kremenek NeedsFetching = true; 220268ee7016a2811803989487c0ad3799486092c63Ted Kremenek 221268ee7016a2811803989487c0ad3799486092c63Ted Kremenek // This case can occur when we see something like this: 222268ee7016a2811803989487c0ad3799486092c63Ted Kremenek // 223268ee7016a2811803989487c0ad3799486092c63Ted Kremenek // #if ... 224268ee7016a2811803989487c0ad3799486092c63Ted Kremenek // /* a comment or nothing */ 225268ee7016a2811803989487c0ad3799486092c63Ted Kremenek // #elif 226268ee7016a2811803989487c0ad3799486092c63Ted Kremenek // 227268ee7016a2811803989487c0ad3799486092c63Ted Kremenek // If we are skipping the first #if block it will be the case that CurPtr 228268ee7016a2811803989487c0ad3799486092c63Ted Kremenek // already points 'elif'. Just return. 229268ee7016a2811803989487c0ad3799486092c63Ted Kremenek 23041a2660377d215d004fe413c03874bd066b5384cTed Kremenek if (CurPtr > HashEntryI) { 23141a2660377d215d004fe413c03874bd066b5384cTed Kremenek assert(CurPtr == HashEntryI + DISK_TOKEN_SIZE); 232268ee7016a2811803989487c0ad3799486092c63Ted Kremenek // Did we reach a #endif? If so, go ahead and consume that token as well. 233268ee7016a2811803989487c0ad3799486092c63Ted Kremenek if (isEndif) 234268ee7016a2811803989487c0ad3799486092c63Ted Kremenek CurPtr += DISK_TOKEN_SIZE; 235268ee7016a2811803989487c0ad3799486092c63Ted Kremenek else 23641a2660377d215d004fe413c03874bd066b5384cTed Kremenek LastHashTokPtr = HashEntryI; 237268ee7016a2811803989487c0ad3799486092c63Ted Kremenek 238268ee7016a2811803989487c0ad3799486092c63Ted Kremenek return isEndif; 239268ee7016a2811803989487c0ad3799486092c63Ted Kremenek } 240268ee7016a2811803989487c0ad3799486092c63Ted Kremenek 241268ee7016a2811803989487c0ad3799486092c63Ted Kremenek // Otherwise, we need to advance. Update CurPtr to point to the '#' token. 24241a2660377d215d004fe413c03874bd066b5384cTed Kremenek CurPtr = HashEntryI; 243268ee7016a2811803989487c0ad3799486092c63Ted Kremenek 244268ee7016a2811803989487c0ad3799486092c63Ted Kremenek // Update the location of the last observed '#'. This is useful if we 245268ee7016a2811803989487c0ad3799486092c63Ted Kremenek // are skipping multiple blocks. 246268ee7016a2811803989487c0ad3799486092c63Ted Kremenek LastHashTokPtr = CurPtr; 247268ee7016a2811803989487c0ad3799486092c63Ted Kremenek 248268ee7016a2811803989487c0ad3799486092c63Ted Kremenek#ifndef DEBUG 249268ee7016a2811803989487c0ad3799486092c63Ted Kremenek // In a debug build we should verify that the token is really a '#' that 250268ee7016a2811803989487c0ad3799486092c63Ted Kremenek // appears at the start of the line. 251268ee7016a2811803989487c0ad3799486092c63Ted Kremenek Token Tok; 252268ee7016a2811803989487c0ad3799486092c63Ted Kremenek ReadToken(Tok); 253268ee7016a2811803989487c0ad3799486092c63Ted Kremenek assert(Tok.isAtStartOfLine() && Tok.is(tok::hash)); 254268ee7016a2811803989487c0ad3799486092c63Ted Kremenek#else 255268ee7016a2811803989487c0ad3799486092c63Ted Kremenek // In a full release build we can just skip the token entirely. 256268ee7016a2811803989487c0ad3799486092c63Ted Kremenek CurPtr += DISK_TOKEN_SIZE; 257268ee7016a2811803989487c0ad3799486092c63Ted Kremenek#endif 258268ee7016a2811803989487c0ad3799486092c63Ted Kremenek 259268ee7016a2811803989487c0ad3799486092c63Ted Kremenek // Did we reach a #endif? If so, go ahead and consume that token as well. 260268ee7016a2811803989487c0ad3799486092c63Ted Kremenek if (isEndif) { CurPtr += DISK_TOKEN_SIZE; } 261268ee7016a2811803989487c0ad3799486092c63Ted Kremenek 262268ee7016a2811803989487c0ad3799486092c63Ted Kremenek return isEndif; 263268ee7016a2811803989487c0ad3799486092c63Ted Kremenek} 264268ee7016a2811803989487c0ad3799486092c63Ted Kremenek 26530a12ec2a7f331d9e08acabe7cda853aaa7ba54bTed KremenekSourceLocation PTHLexer::getSourceLocation() { 26630a12ec2a7f331d9e08acabe7cda853aaa7ba54bTed Kremenek // getLocation is not on the hot path. It is used to get the location of 26730a12ec2a7f331d9e08acabe7cda853aaa7ba54bTed Kremenek // the next token when transitioning back to this lexer when done 26830a12ec2a7f331d9e08acabe7cda853aaa7ba54bTed Kremenek // handling a #included file. Just read the necessary data from the token 26930a12ec2a7f331d9e08acabe7cda853aaa7ba54bTed Kremenek // data buffer to construct the SourceLocation object. 27030a12ec2a7f331d9e08acabe7cda853aaa7ba54bTed Kremenek // NOTE: This is a virtual function; hence it is defined out-of-line. 27130a12ec2a7f331d9e08acabe7cda853aaa7ba54bTed Kremenek const char* p = CurPtr + (1 + 1 + 4); 27230a12ec2a7f331d9e08acabe7cda853aaa7ba54bTed Kremenek uint32_t offset = 27330a12ec2a7f331d9e08acabe7cda853aaa7ba54bTed Kremenek ((uint32_t) ((uint8_t) p[0])) 27430a12ec2a7f331d9e08acabe7cda853aaa7ba54bTed Kremenek | (((uint32_t) ((uint8_t) p[1])) << 8) 27530a12ec2a7f331d9e08acabe7cda853aaa7ba54bTed Kremenek | (((uint32_t) ((uint8_t) p[2])) << 16) 27630a12ec2a7f331d9e08acabe7cda853aaa7ba54bTed Kremenek | (((uint32_t) ((uint8_t) p[3])) << 24); 27730a12ec2a7f331d9e08acabe7cda853aaa7ba54bTed Kremenek return SourceLocation::getFileLoc(FileID, offset); 27830a12ec2a7f331d9e08acabe7cda853aaa7ba54bTed Kremenek} 27930a12ec2a7f331d9e08acabe7cda853aaa7ba54bTed Kremenek 2800c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek//===----------------------------------------------------------------------===// 2810c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek// Token reconstruction from the PTH file. 2820c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek//===----------------------------------------------------------------------===// 2830c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek 2840c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenekvoid PTHLexer::ReadToken(Token& T) { 2850c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek // Clear the token. 2860c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek // FIXME: Setting the flags directly should obviate this step. 2870c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek T.startToken(); 2880c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek 2891c13c4f87ebdf4309d9b9b75336f15301b8ec8ceTed Kremenek // Shadow CurPtr into an automatic variable so that Read8 doesn't load and 2901c13c4f87ebdf4309d9b9b75336f15301b8ec8ceTed Kremenek // store back into the instance variable. 2911c13c4f87ebdf4309d9b9b75336f15301b8ec8ceTed Kremenek const char *CurPtrShadow = CurPtr; 2921c13c4f87ebdf4309d9b9b75336f15301b8ec8ceTed Kremenek 2930c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek // Read the type of the token. 2941c13c4f87ebdf4309d9b9b75336f15301b8ec8ceTed Kremenek T.setKind((tok::TokenKind) Read8(CurPtrShadow)); 2950c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek 2960c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek // Set flags. This is gross, since we are really setting multiple flags. 2971c13c4f87ebdf4309d9b9b75336f15301b8ec8ceTed Kremenek T.setFlag((Token::TokenFlags) Read8(CurPtrShadow)); 2980c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek 2990c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek // Set the IdentifierInfo* (if any). 3001c13c4f87ebdf4309d9b9b75336f15301b8ec8ceTed Kremenek T.setIdentifierInfo(PTHMgr.ReadIdentifierInfo(CurPtrShadow)); 3010c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek 3020c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek // Set the SourceLocation. Since all tokens are constructed using a 303b60d7999d621fce608e03d39e82c0e7eda750054Chris Lattner // raw lexer, they will all be offseted from the same FileID. 3041c13c4f87ebdf4309d9b9b75336f15301b8ec8ceTed Kremenek T.setLocation(SourceLocation::getFileLoc(FileID, Read32(CurPtrShadow))); 3050c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek 3060c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek // Finally, read and set the length of the token. 3071c13c4f87ebdf4309d9b9b75336f15301b8ec8ceTed Kremenek T.setLength(Read32(CurPtrShadow)); 3081c13c4f87ebdf4309d9b9b75336f15301b8ec8ceTed Kremenek 3091c13c4f87ebdf4309d9b9b75336f15301b8ec8ceTed Kremenek CurPtr = CurPtrShadow; 3100c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek} 3110c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek 3120c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek//===----------------------------------------------------------------------===// 3130c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek// Internal Data Structures for PTH file lookup and resolving identifiers. 3140c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek//===----------------------------------------------------------------------===// 3150c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek 3160c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek 3170c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek/// PTHFileLookup - This internal data structure is used by the PTHManager 3180c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek/// to map from FileEntry objects managed by FileManager to offsets within 3190c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek/// the PTH file. 3200c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremeneknamespace { 3210c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenekclass VISIBILITY_HIDDEN PTHFileLookup { 3220c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenekpublic: 3230c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek class Val { 324fb645b6547b75ddc2e3c7ab2126ad8beeefca62dTed Kremenek uint32_t TokenOff; 325fb645b6547b75ddc2e3c7ab2126ad8beeefca62dTed Kremenek uint32_t PPCondOff; 3260c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek 3270c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek public: 328fb645b6547b75ddc2e3c7ab2126ad8beeefca62dTed Kremenek Val() : TokenOff(~0) {} 329fb645b6547b75ddc2e3c7ab2126ad8beeefca62dTed Kremenek Val(uint32_t toff, uint32_t poff) : TokenOff(toff), PPCondOff(poff) {} 3300c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek 331fb645b6547b75ddc2e3c7ab2126ad8beeefca62dTed Kremenek uint32_t getTokenOffset() const { 332fb645b6547b75ddc2e3c7ab2126ad8beeefca62dTed Kremenek assert(TokenOff != ~((uint32_t)0) && "PTHFileLookup entry initialized."); 333fb645b6547b75ddc2e3c7ab2126ad8beeefca62dTed Kremenek return TokenOff; 3340c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek } 3350c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek 336fb645b6547b75ddc2e3c7ab2126ad8beeefca62dTed Kremenek uint32_t gettPPCondOffset() const { 337fb645b6547b75ddc2e3c7ab2126ad8beeefca62dTed Kremenek assert(TokenOff != ~((uint32_t)0) && "PTHFileLookup entry initialized."); 338fb645b6547b75ddc2e3c7ab2126ad8beeefca62dTed Kremenek return PPCondOff; 339fb645b6547b75ddc2e3c7ab2126ad8beeefca62dTed Kremenek } 340fb645b6547b75ddc2e3c7ab2126ad8beeefca62dTed Kremenek 341fb645b6547b75ddc2e3c7ab2126ad8beeefca62dTed Kremenek bool isValid() const { return TokenOff != ~((uint32_t)0); } 3420c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek }; 3430c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek 3440c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenekprivate: 3450c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek llvm::StringMap<Val> FileMap; 3460c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek 3470c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenekpublic: 3480c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek PTHFileLookup() {}; 3490c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek 3500c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek Val Lookup(const FileEntry* FE) { 3510c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek const char* s = FE->getName(); 3520c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek unsigned size = strlen(s); 3530c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek return FileMap.GetOrCreateValue(s, s+size).getValue(); 3540c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek } 3550c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek 3560c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek void ReadTable(const char* D) { 3570c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek uint32_t N = Read32(D); // Read the length of the table. 3580c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek 3590c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek for ( ; N > 0; --N) { // The rest of the data is the table itself. 3600c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek uint32_t len = Read32(D); 3610c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek const char* s = D; 3620c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek D += len; 363fb645b6547b75ddc2e3c7ab2126ad8beeefca62dTed Kremenek uint32_t TokenOff = Read32(D); 364fb645b6547b75ddc2e3c7ab2126ad8beeefca62dTed Kremenek FileMap.GetOrCreateValue(s, s+len).getValue() = Val(TokenOff, Read32(D)); 3650c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek } 3660c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek } 3670c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek}; 3680c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek} // end anonymous namespace 3690c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek 3700c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek//===----------------------------------------------------------------------===// 3710c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek// PTHManager methods. 3720c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek//===----------------------------------------------------------------------===// 3730c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek 3740c6a77bc1f52f282a969538f139ebde429076ed3Ted KremenekPTHManager::PTHManager(const llvm::MemoryBuffer* buf, void* fileLookup, 375cf58e6249c6b018508e34bcb76202caa42d2451aTed Kremenek const char* idDataTable, IdentifierInfo** perIDCache, 3766183e4815a4019e97ad01bd880f12355599b75fdTed Kremenek Preprocessor& pp) 3776183e4815a4019e97ad01bd880f12355599b75fdTed Kremenek: Buf(buf), PerIDCache(perIDCache), FileLookup(fileLookup), 3786183e4815a4019e97ad01bd880f12355599b75fdTed Kremenek IdDataTable(idDataTable), ITable(pp.getIdentifierTable()), PP(pp) {} 3790c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek 3800c6a77bc1f52f282a969538f139ebde429076ed3Ted KremenekPTHManager::~PTHManager() { 3810c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek delete Buf; 3820c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek delete (PTHFileLookup*) FileLookup; 3830e50b6e7c104d00614baa3d80df62f1630a94d9cTed Kremenek free(PerIDCache); 3840c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek} 3850c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek 3860c6a77bc1f52f282a969538f139ebde429076ed3Ted KremenekPTHManager* PTHManager::Create(const std::string& file, Preprocessor& PP) { 3870c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek 3880c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek // Memory map the PTH file. 3890c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek llvm::OwningPtr<llvm::MemoryBuffer> 3900c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek File(llvm::MemoryBuffer::getFile(file.c_str())); 3910c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek 3920c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek if (!File) 3930c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek return 0; 3940c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek 3950c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek // Get the buffer ranges and check if there are at least three 32-bit 3960c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek // words at the end of the file. 3970c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek const char* BufBeg = File->getBufferStart(); 3980c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek const char* BufEnd = File->getBufferEnd(); 3990c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek 4000c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek if(!(BufEnd > BufBeg + sizeof(uint32_t)*3)) { 4010c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek assert(false && "Invalid PTH file."); 4020c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek return 0; // FIXME: Proper error diagnostic? 4030c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek } 4040c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek 4050c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek // Compute the address of the index table at the end of the PTH file. 4060c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek // This table contains the offset of the file lookup table, the 4070c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek // persistent ID -> identifer data table. 4080c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek const char* EndTable = BufEnd - sizeof(uint32_t)*3; 4090c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek 4100c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek // Construct the file lookup table. This will be used for mapping from 4110c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek // FileEntry*'s to cached tokens. 4120c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek const char* FileTableOffset = EndTable + sizeof(uint32_t)*2; 4130c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek const char* FileTable = BufBeg + Read32(FileTableOffset); 4140c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek 4150c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek if (!(FileTable > BufBeg && FileTable < BufEnd)) { 4160c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek assert(false && "Invalid PTH file."); 4170c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek return 0; // FIXME: Proper error diagnostic? 4180c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek } 4190c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek 4200c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek llvm::OwningPtr<PTHFileLookup> FL(new PTHFileLookup()); 4210c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek FL->ReadTable(FileTable); 4220c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek 4230c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek // Get the location of the table mapping from persistent ids to the 4240c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek // data needed to reconstruct identifiers. 4250c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek const char* IDTableOffset = EndTable + sizeof(uint32_t)*1; 4260c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek const char* IData = BufBeg + Read32(IDTableOffset); 4270c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek if (!(IData > BufBeg && IData < BufEnd)) { 4280c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek assert(false && "Invalid PTH file."); 4290c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek return 0; // FIXME: Proper error diagnostic? 4300c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek } 4310c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek 4326183e4815a4019e97ad01bd880f12355599b75fdTed Kremenek // Get the number of IdentifierInfos and pre-allocate the identifier cache. 4336183e4815a4019e97ad01bd880f12355599b75fdTed Kremenek uint32_t NumIds = Read32(IData); 4346183e4815a4019e97ad01bd880f12355599b75fdTed Kremenek 4356183e4815a4019e97ad01bd880f12355599b75fdTed Kremenek // Pre-allocate the peristent ID -> IdentifierInfo* cache. We use calloc() 4366183e4815a4019e97ad01bd880f12355599b75fdTed Kremenek // so that we in the best case only zero out memory once when the OS returns 4376183e4815a4019e97ad01bd880f12355599b75fdTed Kremenek // us new pages. 4386183e4815a4019e97ad01bd880f12355599b75fdTed Kremenek IdentifierInfo** PerIDCache = 4396183e4815a4019e97ad01bd880f12355599b75fdTed Kremenek (IdentifierInfo**) calloc(NumIds, sizeof(*PerIDCache)); 4406183e4815a4019e97ad01bd880f12355599b75fdTed Kremenek 4416183e4815a4019e97ad01bd880f12355599b75fdTed Kremenek if (!PerIDCache) { 4426183e4815a4019e97ad01bd880f12355599b75fdTed Kremenek assert(false && "Could not allocate Persistent ID cache."); 4436183e4815a4019e97ad01bd880f12355599b75fdTed Kremenek return 0; 4446183e4815a4019e97ad01bd880f12355599b75fdTed Kremenek } 4456183e4815a4019e97ad01bd880f12355599b75fdTed Kremenek 4466183e4815a4019e97ad01bd880f12355599b75fdTed Kremenek // Create the new lexer. 4476183e4815a4019e97ad01bd880f12355599b75fdTed Kremenek return new PTHManager(File.take(), FL.take(), IData, PerIDCache, PP); 4480c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek} 4490c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek 4500c6a77bc1f52f282a969538f139ebde429076ed3Ted KremenekIdentifierInfo* PTHManager::ReadIdentifierInfo(const char*& D) { 4510c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek // Read the persistent ID from the PTH file. 4520c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek uint32_t persistentID = Read32(D); 4530c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek 4540c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek // A persistent ID of '0' always maps to NULL. 4550c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek if (!persistentID) 4560c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek return 0; 4570c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek 4580c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek // Adjust the persistent ID by subtracting '1' so that it can be used 4590c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek // as an index within a table in the PTH file. 4600c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek --persistentID; 4610c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek 4620c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek // Check if the IdentifierInfo has already been resolved. 463cf58e6249c6b018508e34bcb76202caa42d2451aTed Kremenek IdentifierInfo*& II = PerIDCache[persistentID]; 4640c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek if (II) return II; 4650c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek 4660c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek // Look in the PTH file for the string data for the IdentifierInfo object. 4670c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek const char* TableEntry = IdDataTable + sizeof(uint32_t) * persistentID; 4680c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek const char* IDData = Buf->getBufferStart() + Read32(TableEntry); 4690c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek assert(IDData < Buf->getBufferEnd()); 4700c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek 4710c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek // Read the length of the string. 4720c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek uint32_t len = Read32(IDData); 4730c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek 4740c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek // Get the IdentifierInfo* with the specified string. 4750c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek II = &ITable.get(IDData, IDData+len); 4760c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek return II; 4770c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek} 4780c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek 4790c6a77bc1f52f282a969538f139ebde429076ed3Ted KremenekPTHLexer* PTHManager::CreateLexer(unsigned FileID, const FileEntry* FE) { 4800c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek 4810c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek if (!FE) 4820c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek return 0; 4830c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek 4840c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek // Lookup the FileEntry object in our file lookup data structure. It will 4850c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek // return a variant that indicates whether or not there is an offset within 4860c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek // the PTH file that contains cached tokens. 487fb645b6547b75ddc2e3c7ab2126ad8beeefca62dTed Kremenek PTHFileLookup::Val FileData = ((PTHFileLookup*) FileLookup)->Lookup(FE); 4880c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek 489fb645b6547b75ddc2e3c7ab2126ad8beeefca62dTed Kremenek if (!FileData.isValid()) // No tokens available. 4900c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek return 0; 4910c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek 4920c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek // Compute the offset of the token data within the buffer. 493fb645b6547b75ddc2e3c7ab2126ad8beeefca62dTed Kremenek const char* data = Buf->getBufferStart() + FileData.getTokenOffset(); 494268ee7016a2811803989487c0ad3799486092c63Ted Kremenek 495268ee7016a2811803989487c0ad3799486092c63Ted Kremenek // Get the location of pp-conditional table. 496268ee7016a2811803989487c0ad3799486092c63Ted Kremenek const char* ppcond = Buf->getBufferStart() + FileData.gettPPCondOffset(); 497268ee7016a2811803989487c0ad3799486092c63Ted Kremenek uint32_t len = Read32(ppcond); 498268ee7016a2811803989487c0ad3799486092c63Ted Kremenek if (len == 0) ppcond = 0; 499268ee7016a2811803989487c0ad3799486092c63Ted Kremenek 5000c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek assert(data < Buf->getBufferEnd()); 501268ee7016a2811803989487c0ad3799486092c63Ted Kremenek return new PTHLexer(PP, SourceLocation::getFileLoc(FileID, 0), data, ppcond, 502268ee7016a2811803989487c0ad3799486092c63Ted Kremenek *this); 5030c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek} 504