PTHLexer.cpp revision fb645b6547b75ddc2e3c7ab2126ad8beeefca62d
1274b20863a728cc6a31ee75c670e3733600c1531Ted Kremenek//===--- PTHLexer.cpp - Lex from a token stream ---------------------------===// 2274b20863a728cc6a31ee75c670e3733600c1531Ted Kremenek// 3274b20863a728cc6a31ee75c670e3733600c1531Ted Kremenek// The LLVM Compiler Infrastructure 4274b20863a728cc6a31ee75c670e3733600c1531Ted Kremenek// 5274b20863a728cc6a31ee75c670e3733600c1531Ted Kremenek// This file is distributed under the University of Illinois Open Source 6274b20863a728cc6a31ee75c670e3733600c1531Ted Kremenek// License. See LICENSE.TXT for details. 7274b20863a728cc6a31ee75c670e3733600c1531Ted Kremenek// 8274b20863a728cc6a31ee75c670e3733600c1531Ted Kremenek//===----------------------------------------------------------------------===// 9274b20863a728cc6a31ee75c670e3733600c1531Ted Kremenek// 10274b20863a728cc6a31ee75c670e3733600c1531Ted Kremenek// This file implements the PTHLexer interface. 11274b20863a728cc6a31ee75c670e3733600c1531Ted Kremenek// 12274b20863a728cc6a31ee75c670e3733600c1531Ted Kremenek//===----------------------------------------------------------------------===// 13274b20863a728cc6a31ee75c670e3733600c1531Ted Kremenek 140c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek#include "clang/Basic/TokenKinds.h" 150c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek#include "clang/Basic/FileManager.h" 160c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek#include "clang/Basic/IdentifierTable.h" 17274b20863a728cc6a31ee75c670e3733600c1531Ted Kremenek#include "clang/Lex/PTHLexer.h" 18274b20863a728cc6a31ee75c670e3733600c1531Ted Kremenek#include "clang/Lex/Preprocessor.h" 190c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek#include "clang/Lex/PTHManager.h" 200c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek#include "clang/Lex/Token.h" 210c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek#include "clang/Lex/Preprocessor.h" 220c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek#include "llvm/Support/Compiler.h" 230c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek#include "llvm/Support/MemoryBuffer.h" 240c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek#include "llvm/ADT/StringMap.h" 250c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek#include "llvm/ADT/OwningPtr.h" 260c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek 27274b20863a728cc6a31ee75c670e3733600c1531Ted Kremenekusing namespace clang; 28274b20863a728cc6a31ee75c670e3733600c1531Ted Kremenek 290c6a77bc1f52f282a969538f139ebde429076ed3Ted KremenekPTHLexer::PTHLexer(Preprocessor& pp, SourceLocation fileloc, const char* D, 300c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek PTHManager& PM) 31cd223444d1680290efe11da657faafc9a1ac14baTed Kremenek : PreprocessorLexer(&pp, fileloc), TokBuf(D), CurPtr(D), LastHashTokPtr(0), 32cd223444d1680290efe11da657faafc9a1ac14baTed Kremenek PTHMgr(PM), 330c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek NeedsFetching(true) { 340c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek // Make sure the EofToken is completely clean. 350c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek EofToken.startToken(); 360c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek } 37274b20863a728cc6a31ee75c670e3733600c1531Ted Kremenek 380c6a77bc1f52f282a969538f139ebde429076ed3Ted KremenekToken PTHLexer::GetToken() { 390c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek // Read the next token, or if we haven't advanced yet, get the last 400c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek // token read. 410c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek if (NeedsFetching) { 420c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek NeedsFetching = false; 430c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek ReadToken(LastFetched); 440c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek } 450c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek 460c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek Token Tok = LastFetched; 4789d7ee9619d2dbdfa8d956a695c612a104a92cadTed Kremenek 4889d7ee9619d2dbdfa8d956a695c612a104a92cadTed Kremenek // If we are in raw mode, zero out identifier pointers. This is 4989d7ee9619d2dbdfa8d956a695c612a104a92cadTed Kremenek // needed for 'pragma poison'. Note that this requires that the Preprocessor 5089d7ee9619d2dbdfa8d956a695c612a104a92cadTed Kremenek // can go back to the original source when it calls getSpelling(). 5189d7ee9619d2dbdfa8d956a695c612a104a92cadTed Kremenek if (LexingRawMode && Tok.is(tok::identifier)) 5289d7ee9619d2dbdfa8d956a695c612a104a92cadTed Kremenek Tok.setIdentifierInfo(0); 5389d7ee9619d2dbdfa8d956a695c612a104a92cadTed Kremenek 5489d7ee9619d2dbdfa8d956a695c612a104a92cadTed Kremenek return Tok; 5589d7ee9619d2dbdfa8d956a695c612a104a92cadTed Kremenek} 5689d7ee9619d2dbdfa8d956a695c612a104a92cadTed Kremenek 57274b20863a728cc6a31ee75c670e3733600c1531Ted Kremenekvoid PTHLexer::Lex(Token& Tok) { 58d6f53dc4951aace69014619761760addac9e59ecTed KremenekLexNextToken: 59cd4e2aecde5bb7810715d5d5a88ac63ce7946f34Ted Kremenek Tok = GetToken(); 60cd4e2aecde5bb7810715d5d5a88ac63ce7946f34Ted Kremenek 6131aba425a01c8c957e662ccfaa71f923d0f0932aTed Kremenek if (AtLastToken()) { 62cd4e2aecde5bb7810715d5d5a88ac63ce7946f34Ted Kremenek Preprocessor *PPCache = PP; 63cd4e2aecde5bb7810715d5d5a88ac63ce7946f34Ted Kremenek 64cd4e2aecde5bb7810715d5d5a88ac63ce7946f34Ted Kremenek if (LexEndOfFile(Tok)) 65d6f53dc4951aace69014619761760addac9e59ecTed Kremenek return; 66274b20863a728cc6a31ee75c670e3733600c1531Ted Kremenek 67cd4e2aecde5bb7810715d5d5a88ac63ce7946f34Ted Kremenek assert(PPCache && "Raw buffer::LexEndOfFile should return a token"); 68cd4e2aecde5bb7810715d5d5a88ac63ce7946f34Ted Kremenek return PPCache->Lex(Tok); 69cd4e2aecde5bb7810715d5d5a88ac63ce7946f34Ted Kremenek } 70274b20863a728cc6a31ee75c670e3733600c1531Ted Kremenek 71d6f53dc4951aace69014619761760addac9e59ecTed Kremenek // Don't advance to the next token yet. Check if we are at the 72d6f53dc4951aace69014619761760addac9e59ecTed Kremenek // start of a new line and we're processing a directive. If so, we 73d6f53dc4951aace69014619761760addac9e59ecTed Kremenek // consume this token twice, once as an tok::eom. 74d6f53dc4951aace69014619761760addac9e59ecTed Kremenek if (Tok.isAtStartOfLine() && ParsingPreprocessorDirective) { 75d6f53dc4951aace69014619761760addac9e59ecTed Kremenek ParsingPreprocessorDirective = false; 76d6f53dc4951aace69014619761760addac9e59ecTed Kremenek Tok.setKind(tok::eom); 77274b20863a728cc6a31ee75c670e3733600c1531Ted Kremenek MIOpt.ReadToken(); 78274b20863a728cc6a31ee75c670e3733600c1531Ted Kremenek return; 79274b20863a728cc6a31ee75c670e3733600c1531Ted Kremenek } 80d6f53dc4951aace69014619761760addac9e59ecTed Kremenek 81d6f53dc4951aace69014619761760addac9e59ecTed Kremenek // Advance to the next token. 8231aba425a01c8c957e662ccfaa71f923d0f0932aTed Kremenek AdvanceToken(); 83d6f53dc4951aace69014619761760addac9e59ecTed Kremenek 84d6f53dc4951aace69014619761760addac9e59ecTed Kremenek if (Tok.is(tok::hash)) { 85d6f53dc4951aace69014619761760addac9e59ecTed Kremenek if (Tok.isAtStartOfLine() && !LexingRawMode) { 86cd223444d1680290efe11da657faafc9a1ac14baTed Kremenek LastHashTokPtr = CurPtr; 87cd223444d1680290efe11da657faafc9a1ac14baTed Kremenek 88d6f53dc4951aace69014619761760addac9e59ecTed Kremenek PP->HandleDirective(Tok); 89274b20863a728cc6a31ee75c670e3733600c1531Ted Kremenek 90d6f53dc4951aace69014619761760addac9e59ecTed Kremenek if (PP->isCurrentLexer(this)) 91d6f53dc4951aace69014619761760addac9e59ecTed Kremenek goto LexNextToken; 92d6f53dc4951aace69014619761760addac9e59ecTed Kremenek 93d6f53dc4951aace69014619761760addac9e59ecTed Kremenek return PP->Lex(Tok); 94d6f53dc4951aace69014619761760addac9e59ecTed Kremenek } 95274b20863a728cc6a31ee75c670e3733600c1531Ted Kremenek } 96d6f53dc4951aace69014619761760addac9e59ecTed Kremenek 97274b20863a728cc6a31ee75c670e3733600c1531Ted Kremenek MIOpt.ReadToken(); 98d6f53dc4951aace69014619761760addac9e59ecTed Kremenek 99d6f53dc4951aace69014619761760addac9e59ecTed Kremenek if (Tok.is(tok::identifier)) { 100d6f53dc4951aace69014619761760addac9e59ecTed Kremenek if (LexingRawMode) return; 101d6f53dc4951aace69014619761760addac9e59ecTed Kremenek return PP->HandleIdentifier(Tok); 102d6f53dc4951aace69014619761760addac9e59ecTed Kremenek } 103274b20863a728cc6a31ee75c670e3733600c1531Ted Kremenek} 104274b20863a728cc6a31ee75c670e3733600c1531Ted Kremenek 105cd4e2aecde5bb7810715d5d5a88ac63ce7946f34Ted Kremenekbool PTHLexer::LexEndOfFile(Token &Tok) { 106cd4e2aecde5bb7810715d5d5a88ac63ce7946f34Ted Kremenek 107cd4e2aecde5bb7810715d5d5a88ac63ce7946f34Ted Kremenek if (ParsingPreprocessorDirective) { 108cd4e2aecde5bb7810715d5d5a88ac63ce7946f34Ted Kremenek ParsingPreprocessorDirective = false; 109cd4e2aecde5bb7810715d5d5a88ac63ce7946f34Ted Kremenek Tok.setKind(tok::eom); 110cd4e2aecde5bb7810715d5d5a88ac63ce7946f34Ted Kremenek MIOpt.ReadToken(); 111cd4e2aecde5bb7810715d5d5a88ac63ce7946f34Ted Kremenek return true; // Have a token. 112cd4e2aecde5bb7810715d5d5a88ac63ce7946f34Ted Kremenek } 113cd4e2aecde5bb7810715d5d5a88ac63ce7946f34Ted Kremenek 114cd4e2aecde5bb7810715d5d5a88ac63ce7946f34Ted Kremenek if (LexingRawMode) { 115cd4e2aecde5bb7810715d5d5a88ac63ce7946f34Ted Kremenek MIOpt.ReadToken(); 116cd4e2aecde5bb7810715d5d5a88ac63ce7946f34Ted Kremenek return true; // Have an eof token. 117cd4e2aecde5bb7810715d5d5a88ac63ce7946f34Ted Kremenek } 118cd4e2aecde5bb7810715d5d5a88ac63ce7946f34Ted Kremenek 119cd4e2aecde5bb7810715d5d5a88ac63ce7946f34Ted Kremenek // FIXME: Issue diagnostics similar to Lexer. 120cd4e2aecde5bb7810715d5d5a88ac63ce7946f34Ted Kremenek return PP->HandleEndOfFile(Tok, false); 121cd4e2aecde5bb7810715d5d5a88ac63ce7946f34Ted Kremenek} 122cd4e2aecde5bb7810715d5d5a88ac63ce7946f34Ted Kremenek 123274b20863a728cc6a31ee75c670e3733600c1531Ted Kremenekvoid PTHLexer::setEOF(Token& Tok) { 1240c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek assert(!EofToken.is(tok::eof)); 1250c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek Tok = EofToken; 126274b20863a728cc6a31ee75c670e3733600c1531Ted Kremenek} 12717ff58a63197b398ae52697b088dc0fb8b255519Ted Kremenek 12817ff58a63197b398ae52697b088dc0fb8b255519Ted Kremenekvoid PTHLexer::DiscardToEndOfLine() { 12917ff58a63197b398ae52697b088dc0fb8b255519Ted Kremenek assert(ParsingPreprocessorDirective && ParsingFilename == false && 13017ff58a63197b398ae52697b088dc0fb8b255519Ted Kremenek "Must be in a preprocessing directive!"); 1314d35da2e41941965bbee8ed7e8c30e7c21000d71Ted Kremenek 1324d35da2e41941965bbee8ed7e8c30e7c21000d71Ted Kremenek // Already at end-of-file? 13331aba425a01c8c957e662ccfaa71f923d0f0932aTed Kremenek if (AtLastToken()) 1344d35da2e41941965bbee8ed7e8c30e7c21000d71Ted Kremenek return; 1354d35da2e41941965bbee8ed7e8c30e7c21000d71Ted Kremenek 1364d35da2e41941965bbee8ed7e8c30e7c21000d71Ted Kremenek // Find the first token that is not the start of the *current* line. 137d2bdeed0727c291a1a0037dc4fa80379dafa0687Ted Kremenek Token T; 138d2bdeed0727c291a1a0037dc4fa80379dafa0687Ted Kremenek for (Lex(T); !AtLastToken(); Lex(T)) 13931aba425a01c8c957e662ccfaa71f923d0f0932aTed Kremenek if (GetToken().isAtStartOfLine()) 1404d35da2e41941965bbee8ed7e8c30e7c21000d71Ted Kremenek return; 14117ff58a63197b398ae52697b088dc0fb8b255519Ted Kremenek} 1420c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek 1430c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek//===----------------------------------------------------------------------===// 1440c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek// Utility methods for reading from the mmap'ed PTH file. 1450c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek//===----------------------------------------------------------------------===// 1460c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek 1470c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenekstatic inline uint8_t Read8(const char*& data) { 1480c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek return (uint8_t) *(data++); 1490c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek} 1500c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek 1510c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenekstatic inline uint32_t Read32(const char*& data) { 1520c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek uint32_t V = (uint32_t) Read8(data); 1530c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek V |= (((uint32_t) Read8(data)) << 8); 1540c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek V |= (((uint32_t) Read8(data)) << 16); 1550c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek V |= (((uint32_t) Read8(data)) << 24); 1560c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek return V; 1570c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek} 1580c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek 1590c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek//===----------------------------------------------------------------------===// 1600c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek// Token reconstruction from the PTH file. 1610c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek//===----------------------------------------------------------------------===// 1620c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek 1630c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenekvoid PTHLexer::ReadToken(Token& T) { 1640c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek // Clear the token. 1650c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek // FIXME: Setting the flags directly should obviate this step. 1660c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek T.startToken(); 1670c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek 1680c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek // Read the type of the token. 169cd223444d1680290efe11da657faafc9a1ac14baTed Kremenek T.setKind((tok::TokenKind) Read8(CurPtr)); 1700c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek 1710c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek // Set flags. This is gross, since we are really setting multiple flags. 172cd223444d1680290efe11da657faafc9a1ac14baTed Kremenek T.setFlag((Token::TokenFlags) Read8(CurPtr)); 1730c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek 1740c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek // Set the IdentifierInfo* (if any). 175cd223444d1680290efe11da657faafc9a1ac14baTed Kremenek T.setIdentifierInfo(PTHMgr.ReadIdentifierInfo(CurPtr)); 1760c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek 1770c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek // Set the SourceLocation. Since all tokens are constructed using a 1780c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek // raw lexer, they will all be offseted from the same FileID. 179cd223444d1680290efe11da657faafc9a1ac14baTed Kremenek T.setLocation(SourceLocation::getFileLoc(FileID, Read32(CurPtr))); 1800c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek 1810c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek // Finally, read and set the length of the token. 182cd223444d1680290efe11da657faafc9a1ac14baTed Kremenek T.setLength(Read32(CurPtr)); 1830c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek} 1840c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek 1850c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek//===----------------------------------------------------------------------===// 1860c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek// Internal Data Structures for PTH file lookup and resolving identifiers. 1870c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek//===----------------------------------------------------------------------===// 1880c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek 1890c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek 1900c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek/// PTHFileLookup - This internal data structure is used by the PTHManager 1910c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek/// to map from FileEntry objects managed by FileManager to offsets within 1920c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek/// the PTH file. 1930c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremeneknamespace { 1940c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenekclass VISIBILITY_HIDDEN PTHFileLookup { 1950c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenekpublic: 1960c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek class Val { 197fb645b6547b75ddc2e3c7ab2126ad8beeefca62dTed Kremenek uint32_t TokenOff; 198fb645b6547b75ddc2e3c7ab2126ad8beeefca62dTed Kremenek uint32_t PPCondOff; 1990c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek 2000c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek public: 201fb645b6547b75ddc2e3c7ab2126ad8beeefca62dTed Kremenek Val() : TokenOff(~0) {} 202fb645b6547b75ddc2e3c7ab2126ad8beeefca62dTed Kremenek Val(uint32_t toff, uint32_t poff) : TokenOff(toff), PPCondOff(poff) {} 2030c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek 204fb645b6547b75ddc2e3c7ab2126ad8beeefca62dTed Kremenek uint32_t getTokenOffset() const { 205fb645b6547b75ddc2e3c7ab2126ad8beeefca62dTed Kremenek assert(TokenOff != ~((uint32_t)0) && "PTHFileLookup entry initialized."); 206fb645b6547b75ddc2e3c7ab2126ad8beeefca62dTed Kremenek return TokenOff; 2070c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek } 2080c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek 209fb645b6547b75ddc2e3c7ab2126ad8beeefca62dTed Kremenek uint32_t gettPPCondOffset() const { 210fb645b6547b75ddc2e3c7ab2126ad8beeefca62dTed Kremenek assert(TokenOff != ~((uint32_t)0) && "PTHFileLookup entry initialized."); 211fb645b6547b75ddc2e3c7ab2126ad8beeefca62dTed Kremenek return PPCondOff; 212fb645b6547b75ddc2e3c7ab2126ad8beeefca62dTed Kremenek } 213fb645b6547b75ddc2e3c7ab2126ad8beeefca62dTed Kremenek 214fb645b6547b75ddc2e3c7ab2126ad8beeefca62dTed Kremenek bool isValid() const { return TokenOff != ~((uint32_t)0); } 2150c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek }; 2160c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek 2170c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenekprivate: 2180c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek llvm::StringMap<Val> FileMap; 2190c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek 2200c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenekpublic: 2210c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek PTHFileLookup() {}; 2220c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek 2230c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek Val Lookup(const FileEntry* FE) { 2240c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek const char* s = FE->getName(); 2250c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek unsigned size = strlen(s); 2260c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek return FileMap.GetOrCreateValue(s, s+size).getValue(); 2270c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek } 2280c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek 2290c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek void ReadTable(const char* D) { 2300c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek uint32_t N = Read32(D); // Read the length of the table. 2310c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek 2320c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek for ( ; N > 0; --N) { // The rest of the data is the table itself. 2330c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek uint32_t len = Read32(D); 2340c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek const char* s = D; 2350c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek D += len; 236fb645b6547b75ddc2e3c7ab2126ad8beeefca62dTed Kremenek uint32_t TokenOff = Read32(D); 237fb645b6547b75ddc2e3c7ab2126ad8beeefca62dTed Kremenek FileMap.GetOrCreateValue(s, s+len).getValue() = Val(TokenOff, Read32(D)); 2380c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek } 2390c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek } 2400c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek}; 2410c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek} // end anonymous namespace 2420c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek 2430c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek//===----------------------------------------------------------------------===// 2440c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek// PTHManager methods. 2450c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek//===----------------------------------------------------------------------===// 2460c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek 2470c6a77bc1f52f282a969538f139ebde429076ed3Ted KremenekPTHManager::PTHManager(const llvm::MemoryBuffer* buf, void* fileLookup, 248cf58e6249c6b018508e34bcb76202caa42d2451aTed Kremenek const char* idDataTable, IdentifierInfo** perIDCache, 2496183e4815a4019e97ad01bd880f12355599b75fdTed Kremenek Preprocessor& pp) 2506183e4815a4019e97ad01bd880f12355599b75fdTed Kremenek: Buf(buf), PerIDCache(perIDCache), FileLookup(fileLookup), 2516183e4815a4019e97ad01bd880f12355599b75fdTed Kremenek IdDataTable(idDataTable), ITable(pp.getIdentifierTable()), PP(pp) {} 2520c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek 2530c6a77bc1f52f282a969538f139ebde429076ed3Ted KremenekPTHManager::~PTHManager() { 2540c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek delete Buf; 2550c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek delete (PTHFileLookup*) FileLookup; 2560e50b6e7c104d00614baa3d80df62f1630a94d9cTed Kremenek free(PerIDCache); 2570c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek} 2580c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek 2590c6a77bc1f52f282a969538f139ebde429076ed3Ted KremenekPTHManager* PTHManager::Create(const std::string& file, Preprocessor& PP) { 2600c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek 2610c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek // Memory map the PTH file. 2620c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek llvm::OwningPtr<llvm::MemoryBuffer> 2630c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek File(llvm::MemoryBuffer::getFile(file.c_str())); 2640c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek 2650c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek if (!File) 2660c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek return 0; 2670c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek 2680c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek // Get the buffer ranges and check if there are at least three 32-bit 2690c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek // words at the end of the file. 2700c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek const char* BufBeg = File->getBufferStart(); 2710c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek const char* BufEnd = File->getBufferEnd(); 2720c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek 2730c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek if(!(BufEnd > BufBeg + sizeof(uint32_t)*3)) { 2740c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek assert(false && "Invalid PTH file."); 2750c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek return 0; // FIXME: Proper error diagnostic? 2760c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek } 2770c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek 2780c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek // Compute the address of the index table at the end of the PTH file. 2790c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek // This table contains the offset of the file lookup table, the 2800c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek // persistent ID -> identifer data table. 2810c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek const char* EndTable = BufEnd - sizeof(uint32_t)*3; 2820c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek 2830c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek // Construct the file lookup table. This will be used for mapping from 2840c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek // FileEntry*'s to cached tokens. 2850c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek const char* FileTableOffset = EndTable + sizeof(uint32_t)*2; 2860c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek const char* FileTable = BufBeg + Read32(FileTableOffset); 2870c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek 2880c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek if (!(FileTable > BufBeg && FileTable < BufEnd)) { 2890c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek assert(false && "Invalid PTH file."); 2900c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek return 0; // FIXME: Proper error diagnostic? 2910c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek } 2920c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek 2930c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek llvm::OwningPtr<PTHFileLookup> FL(new PTHFileLookup()); 2940c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek FL->ReadTable(FileTable); 2950c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek 2960c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek // Get the location of the table mapping from persistent ids to the 2970c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek // data needed to reconstruct identifiers. 2980c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek const char* IDTableOffset = EndTable + sizeof(uint32_t)*1; 2990c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek const char* IData = BufBeg + Read32(IDTableOffset); 3000c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek if (!(IData > BufBeg && IData < BufEnd)) { 3010c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek assert(false && "Invalid PTH file."); 3020c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek return 0; // FIXME: Proper error diagnostic? 3030c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek } 3040c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek 3056183e4815a4019e97ad01bd880f12355599b75fdTed Kremenek // Get the number of IdentifierInfos and pre-allocate the identifier cache. 3066183e4815a4019e97ad01bd880f12355599b75fdTed Kremenek uint32_t NumIds = Read32(IData); 3076183e4815a4019e97ad01bd880f12355599b75fdTed Kremenek 3086183e4815a4019e97ad01bd880f12355599b75fdTed Kremenek // Pre-allocate the peristent ID -> IdentifierInfo* cache. We use calloc() 3096183e4815a4019e97ad01bd880f12355599b75fdTed Kremenek // so that we in the best case only zero out memory once when the OS returns 3106183e4815a4019e97ad01bd880f12355599b75fdTed Kremenek // us new pages. 3116183e4815a4019e97ad01bd880f12355599b75fdTed Kremenek IdentifierInfo** PerIDCache = 3126183e4815a4019e97ad01bd880f12355599b75fdTed Kremenek (IdentifierInfo**) calloc(NumIds, sizeof(*PerIDCache)); 3136183e4815a4019e97ad01bd880f12355599b75fdTed Kremenek 3146183e4815a4019e97ad01bd880f12355599b75fdTed Kremenek if (!PerIDCache) { 3156183e4815a4019e97ad01bd880f12355599b75fdTed Kremenek assert(false && "Could not allocate Persistent ID cache."); 3166183e4815a4019e97ad01bd880f12355599b75fdTed Kremenek return 0; 3176183e4815a4019e97ad01bd880f12355599b75fdTed Kremenek } 3186183e4815a4019e97ad01bd880f12355599b75fdTed Kremenek 3196183e4815a4019e97ad01bd880f12355599b75fdTed Kremenek // Create the new lexer. 3206183e4815a4019e97ad01bd880f12355599b75fdTed Kremenek return new PTHManager(File.take(), FL.take(), IData, PerIDCache, PP); 3210c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek} 3220c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek 3230c6a77bc1f52f282a969538f139ebde429076ed3Ted KremenekIdentifierInfo* PTHManager::ReadIdentifierInfo(const char*& D) { 3240c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek // Read the persistent ID from the PTH file. 3250c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek uint32_t persistentID = Read32(D); 3260c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek 3270c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek // A persistent ID of '0' always maps to NULL. 3280c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek if (!persistentID) 3290c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek return 0; 3300c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek 3310c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek // Adjust the persistent ID by subtracting '1' so that it can be used 3320c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek // as an index within a table in the PTH file. 3330c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek --persistentID; 3340c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek 3350c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek // Check if the IdentifierInfo has already been resolved. 336cf58e6249c6b018508e34bcb76202caa42d2451aTed Kremenek IdentifierInfo*& II = PerIDCache[persistentID]; 3370c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek if (II) return II; 3380c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek 3390c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek // Look in the PTH file for the string data for the IdentifierInfo object. 3400c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek const char* TableEntry = IdDataTable + sizeof(uint32_t) * persistentID; 3410c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek const char* IDData = Buf->getBufferStart() + Read32(TableEntry); 3420c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek assert(IDData < Buf->getBufferEnd()); 3430c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek 3440c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek // Read the length of the string. 3450c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek uint32_t len = Read32(IDData); 3460c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek 3470c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek // Get the IdentifierInfo* with the specified string. 3480c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek II = &ITable.get(IDData, IDData+len); 3490c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek return II; 3500c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek} 3510c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek 3520c6a77bc1f52f282a969538f139ebde429076ed3Ted KremenekPTHLexer* PTHManager::CreateLexer(unsigned FileID, const FileEntry* FE) { 3530c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek 3540c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek if (!FE) 3550c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek return 0; 3560c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek 3570c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek // Lookup the FileEntry object in our file lookup data structure. It will 3580c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek // return a variant that indicates whether or not there is an offset within 3590c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek // the PTH file that contains cached tokens. 360fb645b6547b75ddc2e3c7ab2126ad8beeefca62dTed Kremenek PTHFileLookup::Val FileData = ((PTHFileLookup*) FileLookup)->Lookup(FE); 3610c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek 362fb645b6547b75ddc2e3c7ab2126ad8beeefca62dTed Kremenek if (!FileData.isValid()) // No tokens available. 3630c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek return 0; 3640c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek 3650c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek // Compute the offset of the token data within the buffer. 366fb645b6547b75ddc2e3c7ab2126ad8beeefca62dTed Kremenek const char* data = Buf->getBufferStart() + FileData.getTokenOffset(); 3670c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek assert(data < Buf->getBufferEnd()); 3680c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek return new PTHLexer(PP, SourceLocation::getFileLoc(FileID, 0), data, *this); 3690c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek} 370