PTHLexer.cpp revision 8a6aec620dbec1f292fe4116c0373ac81ab90234
1274b20863a728cc6a31ee75c670e3733600c1531Ted Kremenek//===--- PTHLexer.cpp - Lex from a token stream ---------------------------===// 2274b20863a728cc6a31ee75c670e3733600c1531Ted Kremenek// 3274b20863a728cc6a31ee75c670e3733600c1531Ted Kremenek// The LLVM Compiler Infrastructure 4274b20863a728cc6a31ee75c670e3733600c1531Ted Kremenek// 5274b20863a728cc6a31ee75c670e3733600c1531Ted Kremenek// This file is distributed under the University of Illinois Open Source 6274b20863a728cc6a31ee75c670e3733600c1531Ted Kremenek// License. See LICENSE.TXT for details. 7274b20863a728cc6a31ee75c670e3733600c1531Ted Kremenek// 8274b20863a728cc6a31ee75c670e3733600c1531Ted Kremenek//===----------------------------------------------------------------------===// 9274b20863a728cc6a31ee75c670e3733600c1531Ted Kremenek// 10274b20863a728cc6a31ee75c670e3733600c1531Ted Kremenek// This file implements the PTHLexer interface. 11274b20863a728cc6a31ee75c670e3733600c1531Ted Kremenek// 12274b20863a728cc6a31ee75c670e3733600c1531Ted Kremenek//===----------------------------------------------------------------------===// 13274b20863a728cc6a31ee75c670e3733600c1531Ted Kremenek 140c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek#include "clang/Basic/TokenKinds.h" 150c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek#include "clang/Basic/FileManager.h" 160c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek#include "clang/Basic/IdentifierTable.h" 17274b20863a728cc6a31ee75c670e3733600c1531Ted Kremenek#include "clang/Lex/PTHLexer.h" 18274b20863a728cc6a31ee75c670e3733600c1531Ted Kremenek#include "clang/Lex/Preprocessor.h" 190c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek#include "clang/Lex/PTHManager.h" 200c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek#include "clang/Lex/Token.h" 210c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek#include "clang/Lex/Preprocessor.h" 220c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek#include "llvm/ADT/StringMap.h" 230c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek#include "llvm/ADT/OwningPtr.h" 246f78c3b8b9343e7e9fbf2d457cccf00df6da5d47Chris Lattner#include "llvm/Support/Compiler.h" 256f78c3b8b9343e7e9fbf2d457cccf00df6da5d47Chris Lattner#include "llvm/Support/MathExtras.h" 266f78c3b8b9343e7e9fbf2d457cccf00df6da5d47Chris Lattner#include "llvm/Support/MemoryBuffer.h" 276f78c3b8b9343e7e9fbf2d457cccf00df6da5d47Chris Lattner#include "llvm/System/Host.h" 28274b20863a728cc6a31ee75c670e3733600c1531Ted Kremenekusing namespace clang; 29274b20863a728cc6a31ee75c670e3733600c1531Ted Kremenek 307b78b7c6d96deb1e63f8d0655ee6fa53de0b65efTed Kremenek#define DISK_TOKEN_SIZE (1+1+2+4+4) 31268ee7016a2811803989487c0ad3799486092c63Ted Kremenek 32e5680f3cd678014cf0872d34726dc804b0cbbdd4Ted Kremenek//===----------------------------------------------------------------------===// 33e5680f3cd678014cf0872d34726dc804b0cbbdd4Ted Kremenek// Utility methods for reading from the mmap'ed PTH file. 34e5680f3cd678014cf0872d34726dc804b0cbbdd4Ted Kremenek//===----------------------------------------------------------------------===// 35e5680f3cd678014cf0872d34726dc804b0cbbdd4Ted Kremenek 365ff4317536dbd7f03332bb250c8b35ec04a6f5dbChris Lattnerstatic inline uint16_t ReadUnalignedLE16(const unsigned char *&Data) { 37da9d61c96c412f6babc7f824152609562f302388Chris Lattner uint16_t V = ((uint16_t)Data[0] << 0) | 38da9d61c96c412f6babc7f824152609562f302388Chris Lattner ((uint16_t)Data[1] << 8); 39da9d61c96c412f6babc7f824152609562f302388Chris Lattner Data += 2; 40da9d61c96c412f6babc7f824152609562f302388Chris Lattner return V; 41da9d61c96c412f6babc7f824152609562f302388Chris Lattner} 42da9d61c96c412f6babc7f824152609562f302388Chris Lattner 435ff4317536dbd7f03332bb250c8b35ec04a6f5dbChris Lattnerstatic inline uint32_t ReadLE32(const unsigned char *&Data) { 44fbc33388c199d6f731170bf55719d57373a09c1fChris Lattner // Hosts that directly support little-endian 32-bit loads can just 45fbc33388c199d6f731170bf55719d57373a09c1fChris Lattner // use them. Big-endian hosts need a bswap. 46f15674c680730c652a37a16a5d3f3ff429b0c308Chris Lattner uint32_t V = *((uint32_t*)Data); 476f78c3b8b9343e7e9fbf2d457cccf00df6da5d47Chris Lattner if (llvm::sys::isBigEndianHost()) 486f78c3b8b9343e7e9fbf2d457cccf00df6da5d47Chris Lattner V = llvm::ByteSwap_32(V); 49da9d61c96c412f6babc7f824152609562f302388Chris Lattner Data += 4; 50e5680f3cd678014cf0872d34726dc804b0cbbdd4Ted Kremenek return V; 51e5680f3cd678014cf0872d34726dc804b0cbbdd4Ted Kremenek} 52e5680f3cd678014cf0872d34726dc804b0cbbdd4Ted Kremenek 53da9d61c96c412f6babc7f824152609562f302388Chris Lattner 54e5680f3cd678014cf0872d34726dc804b0cbbdd4Ted Kremenek//===----------------------------------------------------------------------===// 55e5680f3cd678014cf0872d34726dc804b0cbbdd4Ted Kremenek// PTHLexer methods. 56e5680f3cd678014cf0872d34726dc804b0cbbdd4Ted Kremenek//===----------------------------------------------------------------------===// 57e5680f3cd678014cf0872d34726dc804b0cbbdd4Ted Kremenek 58da9d61c96c412f6babc7f824152609562f302388Chris LattnerPTHLexer::PTHLexer(Preprocessor &PP, FileID FID, const unsigned char *D, 59277faca30c9f8f72b79f55695cbe3395ec246e7cTed Kremenek const unsigned char *ppcond, PTHManager &PM) 602b2453a7d8fe732561795431f39ceb2b2a832d84Chris Lattner : PreprocessorLexer(&PP, FID), TokBuf(D), CurPtr(D), LastHashTokPtr(0), 61277faca30c9f8f72b79f55695cbe3395ec246e7cTed Kremenek PPCond(ppcond), CurPPCondPtr(ppcond), PTHMgr(PM) { 622b2453a7d8fe732561795431f39ceb2b2a832d84Chris Lattner 632b2453a7d8fe732561795431f39ceb2b2a832d84Chris Lattner FileStartLoc = PP.getSourceManager().getLocForStartOfFile(FID); 645f074266cc59563036c40516c814d63825723e20Ted Kremenek} 65274b20863a728cc6a31ee75c670e3733600c1531Ted Kremenek 66e5680f3cd678014cf0872d34726dc804b0cbbdd4Ted Kremenekvoid PTHLexer::Lex(Token& Tok) { 67e5680f3cd678014cf0872d34726dc804b0cbbdd4Ted KremenekLexNextToken: 68866bdf74547efe32c320554837ffce00fcc084feTed Kremenek 69866bdf74547efe32c320554837ffce00fcc084feTed Kremenek //===--------------------------------------==// 70866bdf74547efe32c320554837ffce00fcc084feTed Kremenek // Read the raw token data. 71866bdf74547efe32c320554837ffce00fcc084feTed Kremenek //===--------------------------------------==// 72e5680f3cd678014cf0872d34726dc804b0cbbdd4Ted Kremenek 73866bdf74547efe32c320554837ffce00fcc084feTed Kremenek // Shadow CurPtr into an automatic variable. 74aff6ef8e7bc3c3739f984c390e0af693e60be064Chris Lattner const unsigned char *CurPtrShadow = CurPtr; 75866bdf74547efe32c320554837ffce00fcc084feTed Kremenek 761b5285e1ba31975864da356b2ed927e87670e654Chris Lattner // Read in the data for the token. 775ff4317536dbd7f03332bb250c8b35ec04a6f5dbChris Lattner unsigned Word0 = ReadLE32(CurPtrShadow); 785ff4317536dbd7f03332bb250c8b35ec04a6f5dbChris Lattner uint32_t IdentifierID = ReadLE32(CurPtrShadow); 795ff4317536dbd7f03332bb250c8b35ec04a6f5dbChris Lattner uint32_t FileOffset = ReadLE32(CurPtrShadow); 807b78b7c6d96deb1e63f8d0655ee6fa53de0b65efTed Kremenek 817b78b7c6d96deb1e63f8d0655ee6fa53de0b65efTed Kremenek tok::TokenKind TKind = (tok::TokenKind) (Word0 & 0xFF); 827b78b7c6d96deb1e63f8d0655ee6fa53de0b65efTed Kremenek Token::TokenFlags TFlags = (Token::TokenFlags) ((Word0 >> 8) & 0xFF); 83aff6ef8e7bc3c3739f984c390e0af693e60be064Chris Lattner uint32_t Len = Word0 >> 16; 847b78b7c6d96deb1e63f8d0655ee6fa53de0b65efTed Kremenek 85aff6ef8e7bc3c3739f984c390e0af693e60be064Chris Lattner CurPtr = CurPtrShadow; 86e5680f3cd678014cf0872d34726dc804b0cbbdd4Ted Kremenek 87866bdf74547efe32c320554837ffce00fcc084feTed Kremenek //===--------------------------------------==// 88866bdf74547efe32c320554837ffce00fcc084feTed Kremenek // Construct the token itself. 89866bdf74547efe32c320554837ffce00fcc084feTed Kremenek //===--------------------------------------==// 90e5680f3cd678014cf0872d34726dc804b0cbbdd4Ted Kremenek 91866bdf74547efe32c320554837ffce00fcc084feTed Kremenek Tok.startToken(); 92898a0bb1972efb6e03cb1151412ec7392cef07deChris Lattner Tok.setKind(TKind); 93898a0bb1972efb6e03cb1151412ec7392cef07deChris Lattner Tok.setFlag(TFlags); 9459d08cb672136322375e5400578ee1fbd0947de2Ted Kremenek assert(!LexingRawMode); 952b2453a7d8fe732561795431f39ceb2b2a832d84Chris Lattner Tok.setLocation(FileStartLoc.getFileLocWithOffset(FileOffset)); 96866bdf74547efe32c320554837ffce00fcc084feTed Kremenek Tok.setLength(Len); 9789d7ee9619d2dbdfa8d956a695c612a104a92cadTed Kremenek 98d0a69696acca62798dfc8b98f97c92bfa7fa0490Chris Lattner // Handle identifiers. 99277faca30c9f8f72b79f55695cbe3395ec246e7cTed Kremenek if (Tok.isLiteral()) { 100277faca30c9f8f72b79f55695cbe3395ec246e7cTed Kremenek Tok.setLiteralData((const char*) (PTHMgr.SpellingBase + IdentifierID)); 101277faca30c9f8f72b79f55695cbe3395ec246e7cTed Kremenek } 102277faca30c9f8f72b79f55695cbe3395ec246e7cTed Kremenek else if (IdentifierID) { 103d0a69696acca62798dfc8b98f97c92bfa7fa0490Chris Lattner MIOpt.ReadToken(); 104d0a69696acca62798dfc8b98f97c92bfa7fa0490Chris Lattner IdentifierInfo *II = PTHMgr.GetIdentifierInfo(IdentifierID-1); 105863c486fcb6162495a94fddf7ac8409de2638995Chris Lattner 106d0a69696acca62798dfc8b98f97c92bfa7fa0490Chris Lattner Tok.setIdentifierInfo(II); 107863c486fcb6162495a94fddf7ac8409de2638995Chris Lattner 108863c486fcb6162495a94fddf7ac8409de2638995Chris Lattner // Change the kind of this identifier to the appropriate token kind, e.g. 109863c486fcb6162495a94fddf7ac8409de2638995Chris Lattner // turning "for" into a keyword. 110863c486fcb6162495a94fddf7ac8409de2638995Chris Lattner Tok.setKind(II->getTokenID()); 111863c486fcb6162495a94fddf7ac8409de2638995Chris Lattner 112d0a69696acca62798dfc8b98f97c92bfa7fa0490Chris Lattner if (II->isHandleIdentifierCase()) 113d0a69696acca62798dfc8b98f97c92bfa7fa0490Chris Lattner PP->HandleIdentifier(Tok); 114d0a69696acca62798dfc8b98f97c92bfa7fa0490Chris Lattner return; 115d0a69696acca62798dfc8b98f97c92bfa7fa0490Chris Lattner } 116d0a69696acca62798dfc8b98f97c92bfa7fa0490Chris Lattner 117866bdf74547efe32c320554837ffce00fcc084feTed Kremenek //===--------------------------------------==// 118866bdf74547efe32c320554837ffce00fcc084feTed Kremenek // Process the token. 119866bdf74547efe32c320554837ffce00fcc084feTed Kremenek //===--------------------------------------==// 1205f074266cc59563036c40516c814d63825723e20Ted Kremenek#if 0 1215f074266cc59563036c40516c814d63825723e20Ted Kremenek SourceManager& SM = PP->getSourceManager(); 1225f074266cc59563036c40516c814d63825723e20Ted Kremenek llvm::cerr << SM.getFileEntryForID(FileID)->getName() 1235f074266cc59563036c40516c814d63825723e20Ted Kremenek << ':' << SM.getLogicalLineNumber(Tok.getLocation()) 1245f074266cc59563036c40516c814d63825723e20Ted Kremenek << ':' << SM.getLogicalColumnNumber(Tok.getLocation()) 1255f074266cc59563036c40516c814d63825723e20Ted Kremenek << '\n'; 1265f074266cc59563036c40516c814d63825723e20Ted Kremenek#endif 12789d7ee9619d2dbdfa8d956a695c612a104a92cadTed Kremenek 128898a0bb1972efb6e03cb1151412ec7392cef07deChris Lattner if (TKind == tok::eof) { 129e5680f3cd678014cf0872d34726dc804b0cbbdd4Ted Kremenek // Save the end-of-file token. 130e5680f3cd678014cf0872d34726dc804b0cbbdd4Ted Kremenek EofToken = Tok; 131e5680f3cd678014cf0872d34726dc804b0cbbdd4Ted Kremenek 132cd4e2aecde5bb7810715d5d5a88ac63ce7946f34Ted Kremenek Preprocessor *PPCache = PP; 13359d08cb672136322375e5400578ee1fbd0947de2Ted Kremenek 13459d08cb672136322375e5400578ee1fbd0947de2Ted Kremenek assert(!ParsingPreprocessorDirective); 13559d08cb672136322375e5400578ee1fbd0947de2Ted Kremenek assert(!LexingRawMode); 13659d08cb672136322375e5400578ee1fbd0947de2Ted Kremenek 13759d08cb672136322375e5400578ee1fbd0947de2Ted Kremenek // FIXME: Issue diagnostics similar to Lexer. 13859d08cb672136322375e5400578ee1fbd0947de2Ted Kremenek if (PP->HandleEndOfFile(Tok, false)) 139d6f53dc4951aace69014619761760addac9e59ecTed Kremenek return; 14059d08cb672136322375e5400578ee1fbd0947de2Ted Kremenek 141cd4e2aecde5bb7810715d5d5a88ac63ce7946f34Ted Kremenek assert(PPCache && "Raw buffer::LexEndOfFile should return a token"); 142cd4e2aecde5bb7810715d5d5a88ac63ce7946f34Ted Kremenek return PPCache->Lex(Tok); 143cd4e2aecde5bb7810715d5d5a88ac63ce7946f34Ted Kremenek } 144d6f53dc4951aace69014619761760addac9e59ecTed Kremenek 145898a0bb1972efb6e03cb1151412ec7392cef07deChris Lattner if (TKind == tok::hash && Tok.isAtStartOfLine()) { 14659d08cb672136322375e5400578ee1fbd0947de2Ted Kremenek LastHashTokPtr = CurPtr - DISK_TOKEN_SIZE; 14759d08cb672136322375e5400578ee1fbd0947de2Ted Kremenek assert(!LexingRawMode); 14859d08cb672136322375e5400578ee1fbd0947de2Ted Kremenek PP->HandleDirective(Tok); 149d6f53dc4951aace69014619761760addac9e59ecTed Kremenek 15059d08cb672136322375e5400578ee1fbd0947de2Ted Kremenek if (PP->isCurrentLexer(this)) 15159d08cb672136322375e5400578ee1fbd0947de2Ted Kremenek goto LexNextToken; 152e5680f3cd678014cf0872d34726dc804b0cbbdd4Ted Kremenek 15359d08cb672136322375e5400578ee1fbd0947de2Ted Kremenek return PP->Lex(Tok); 154e5680f3cd678014cf0872d34726dc804b0cbbdd4Ted Kremenek } 155e5680f3cd678014cf0872d34726dc804b0cbbdd4Ted Kremenek 156898a0bb1972efb6e03cb1151412ec7392cef07deChris Lattner if (TKind == tok::eom) { 15759d08cb672136322375e5400578ee1fbd0947de2Ted Kremenek assert(ParsingPreprocessorDirective); 15859d08cb672136322375e5400578ee1fbd0947de2Ted Kremenek ParsingPreprocessorDirective = false; 15959d08cb672136322375e5400578ee1fbd0947de2Ted Kremenek return; 16059d08cb672136322375e5400578ee1fbd0947de2Ted Kremenek } 161274b20863a728cc6a31ee75c670e3733600c1531Ted Kremenek 16259d08cb672136322375e5400578ee1fbd0947de2Ted Kremenek MIOpt.ReadToken(); 163cd4e2aecde5bb7810715d5d5a88ac63ce7946f34Ted Kremenek} 164cd4e2aecde5bb7810715d5d5a88ac63ce7946f34Ted Kremenek 165e5680f3cd678014cf0872d34726dc804b0cbbdd4Ted Kremenek// FIXME: We can just grab the last token instead of storing a copy 166e5680f3cd678014cf0872d34726dc804b0cbbdd4Ted Kremenek// into EofToken. 16759d08cb672136322375e5400578ee1fbd0947de2Ted Kremenekvoid PTHLexer::getEOF(Token& Tok) { 168defb7094c835998bb821e894253287625ce8c74dTed Kremenek assert(EofToken.is(tok::eof)); 1690c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek Tok = EofToken; 170274b20863a728cc6a31ee75c670e3733600c1531Ted Kremenek} 17117ff58a63197b398ae52697b088dc0fb8b255519Ted Kremenek 17217ff58a63197b398ae52697b088dc0fb8b255519Ted Kremenekvoid PTHLexer::DiscardToEndOfLine() { 17317ff58a63197b398ae52697b088dc0fb8b255519Ted Kremenek assert(ParsingPreprocessorDirective && ParsingFilename == false && 17417ff58a63197b398ae52697b088dc0fb8b255519Ted Kremenek "Must be in a preprocessing directive!"); 1754d35da2e41941965bbee8ed7e8c30e7c21000d71Ted Kremenek 176e5680f3cd678014cf0872d34726dc804b0cbbdd4Ted Kremenek // We assume that if the preprocessor wishes to discard to the end of 177e5680f3cd678014cf0872d34726dc804b0cbbdd4Ted Kremenek // the line that it also means to end the current preprocessor directive. 178e5680f3cd678014cf0872d34726dc804b0cbbdd4Ted Kremenek ParsingPreprocessorDirective = false; 179e5680f3cd678014cf0872d34726dc804b0cbbdd4Ted Kremenek 18074c3e6e5e95af08096aab415d1ce15f15ffff02aTed Kremenek // Skip tokens by only peeking at their token kind and the flags. 18174c3e6e5e95af08096aab415d1ce15f15ffff02aTed Kremenek // We don't need to actually reconstruct full tokens from the token buffer. 18274c3e6e5e95af08096aab415d1ce15f15ffff02aTed Kremenek // This saves some copies and it also reduces IdentifierInfo* lookup. 183da9d61c96c412f6babc7f824152609562f302388Chris Lattner const unsigned char* p = CurPtr; 18474c3e6e5e95af08096aab415d1ce15f15ffff02aTed Kremenek while (1) { 18574c3e6e5e95af08096aab415d1ce15f15ffff02aTed Kremenek // Read the token kind. Are we at the end of the file? 18674c3e6e5e95af08096aab415d1ce15f15ffff02aTed Kremenek tok::TokenKind x = (tok::TokenKind) (uint8_t) *p; 18774c3e6e5e95af08096aab415d1ce15f15ffff02aTed Kremenek if (x == tok::eof) break; 188e5680f3cd678014cf0872d34726dc804b0cbbdd4Ted Kremenek 18974c3e6e5e95af08096aab415d1ce15f15ffff02aTed Kremenek // Read the token flags. Are we at the start of the next line? 19074c3e6e5e95af08096aab415d1ce15f15ffff02aTed Kremenek Token::TokenFlags y = (Token::TokenFlags) (uint8_t) p[1]; 19180d2f3059326f99ebf7c867db1c7f106ec9485f5Ted Kremenek if (y & Token::StartOfLine) break; 19274c3e6e5e95af08096aab415d1ce15f15ffff02aTed Kremenek 19374c3e6e5e95af08096aab415d1ce15f15ffff02aTed Kremenek // Skip to the next token. 19474c3e6e5e95af08096aab415d1ce15f15ffff02aTed Kremenek p += DISK_TOKEN_SIZE; 19574c3e6e5e95af08096aab415d1ce15f15ffff02aTed Kremenek } 19674c3e6e5e95af08096aab415d1ce15f15ffff02aTed Kremenek 19774c3e6e5e95af08096aab415d1ce15f15ffff02aTed Kremenek CurPtr = p; 19817ff58a63197b398ae52697b088dc0fb8b255519Ted Kremenek} 1990c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek 200268ee7016a2811803989487c0ad3799486092c63Ted Kremenek/// SkipBlock - Used by Preprocessor to skip the current conditional block. 201268ee7016a2811803989487c0ad3799486092c63Ted Kremenekbool PTHLexer::SkipBlock() { 202268ee7016a2811803989487c0ad3799486092c63Ted Kremenek assert(CurPPCondPtr && "No cached PP conditional information."); 203268ee7016a2811803989487c0ad3799486092c63Ted Kremenek assert(LastHashTokPtr && "No known '#' token."); 204268ee7016a2811803989487c0ad3799486092c63Ted Kremenek 205da9d61c96c412f6babc7f824152609562f302388Chris Lattner const unsigned char* HashEntryI = 0; 206268ee7016a2811803989487c0ad3799486092c63Ted Kremenek uint32_t Offset; 207268ee7016a2811803989487c0ad3799486092c63Ted Kremenek uint32_t TableIdx; 208268ee7016a2811803989487c0ad3799486092c63Ted Kremenek 209268ee7016a2811803989487c0ad3799486092c63Ted Kremenek do { 21041a2660377d215d004fe413c03874bd066b5384cTed Kremenek // Read the token offset from the side-table. 2115ff4317536dbd7f03332bb250c8b35ec04a6f5dbChris Lattner Offset = ReadLE32(CurPPCondPtr); 21241a2660377d215d004fe413c03874bd066b5384cTed Kremenek 21341a2660377d215d004fe413c03874bd066b5384cTed Kremenek // Read the target table index from the side-table. 2145ff4317536dbd7f03332bb250c8b35ec04a6f5dbChris Lattner TableIdx = ReadLE32(CurPPCondPtr); 21541a2660377d215d004fe413c03874bd066b5384cTed Kremenek 21641a2660377d215d004fe413c03874bd066b5384cTed Kremenek // Compute the actual memory address of the '#' token data for this entry. 21741a2660377d215d004fe413c03874bd066b5384cTed Kremenek HashEntryI = TokBuf + Offset; 21841a2660377d215d004fe413c03874bd066b5384cTed Kremenek 21941a2660377d215d004fe413c03874bd066b5384cTed Kremenek // Optmization: "Sibling jumping". #if...#else...#endif blocks can 22041a2660377d215d004fe413c03874bd066b5384cTed Kremenek // contain nested blocks. In the side-table we can jump over these 22141a2660377d215d004fe413c03874bd066b5384cTed Kremenek // nested blocks instead of doing a linear search if the next "sibling" 22241a2660377d215d004fe413c03874bd066b5384cTed Kremenek // entry is not at a location greater than LastHashTokPtr. 22341a2660377d215d004fe413c03874bd066b5384cTed Kremenek if (HashEntryI < LastHashTokPtr && TableIdx) { 22441a2660377d215d004fe413c03874bd066b5384cTed Kremenek // In the side-table we are still at an entry for a '#' token that 22541a2660377d215d004fe413c03874bd066b5384cTed Kremenek // is earlier than the last one we saw. Check if the location we would 22641a2660377d215d004fe413c03874bd066b5384cTed Kremenek // stride gets us closer. 227da9d61c96c412f6babc7f824152609562f302388Chris Lattner const unsigned char* NextPPCondPtr = 228da9d61c96c412f6babc7f824152609562f302388Chris Lattner PPCond + TableIdx*(sizeof(uint32_t)*2); 22941a2660377d215d004fe413c03874bd066b5384cTed Kremenek assert(NextPPCondPtr >= CurPPCondPtr); 23041a2660377d215d004fe413c03874bd066b5384cTed Kremenek // Read where we should jump to. 2315ff4317536dbd7f03332bb250c8b35ec04a6f5dbChris Lattner uint32_t TmpOffset = ReadLE32(NextPPCondPtr); 232da9d61c96c412f6babc7f824152609562f302388Chris Lattner const unsigned char* HashEntryJ = TokBuf + TmpOffset; 23341a2660377d215d004fe413c03874bd066b5384cTed Kremenek 23441a2660377d215d004fe413c03874bd066b5384cTed Kremenek if (HashEntryJ <= LastHashTokPtr) { 23541a2660377d215d004fe413c03874bd066b5384cTed Kremenek // Jump directly to the next entry in the side table. 23641a2660377d215d004fe413c03874bd066b5384cTed Kremenek HashEntryI = HashEntryJ; 23741a2660377d215d004fe413c03874bd066b5384cTed Kremenek Offset = TmpOffset; 2385ff4317536dbd7f03332bb250c8b35ec04a6f5dbChris Lattner TableIdx = ReadLE32(NextPPCondPtr); 23941a2660377d215d004fe413c03874bd066b5384cTed Kremenek CurPPCondPtr = NextPPCondPtr; 24041a2660377d215d004fe413c03874bd066b5384cTed Kremenek } 24141a2660377d215d004fe413c03874bd066b5384cTed Kremenek } 242268ee7016a2811803989487c0ad3799486092c63Ted Kremenek } 24341a2660377d215d004fe413c03874bd066b5384cTed Kremenek while (HashEntryI < LastHashTokPtr); 24441a2660377d215d004fe413c03874bd066b5384cTed Kremenek assert(HashEntryI == LastHashTokPtr && "No PP-cond entry found for '#'"); 245268ee7016a2811803989487c0ad3799486092c63Ted Kremenek assert(TableIdx && "No jumping from #endifs."); 246268ee7016a2811803989487c0ad3799486092c63Ted Kremenek 247268ee7016a2811803989487c0ad3799486092c63Ted Kremenek // Update our side-table iterator. 248da9d61c96c412f6babc7f824152609562f302388Chris Lattner const unsigned char* NextPPCondPtr = PPCond + TableIdx*(sizeof(uint32_t)*2); 249268ee7016a2811803989487c0ad3799486092c63Ted Kremenek assert(NextPPCondPtr >= CurPPCondPtr); 250268ee7016a2811803989487c0ad3799486092c63Ted Kremenek CurPPCondPtr = NextPPCondPtr; 251268ee7016a2811803989487c0ad3799486092c63Ted Kremenek 252268ee7016a2811803989487c0ad3799486092c63Ted Kremenek // Read where we should jump to. 2535ff4317536dbd7f03332bb250c8b35ec04a6f5dbChris Lattner HashEntryI = TokBuf + ReadLE32(NextPPCondPtr); 2545ff4317536dbd7f03332bb250c8b35ec04a6f5dbChris Lattner uint32_t NextIdx = ReadLE32(NextPPCondPtr); 255268ee7016a2811803989487c0ad3799486092c63Ted Kremenek 256268ee7016a2811803989487c0ad3799486092c63Ted Kremenek // By construction NextIdx will be zero if this is a #endif. This is useful 257268ee7016a2811803989487c0ad3799486092c63Ted Kremenek // to know to obviate lexing another token. 258268ee7016a2811803989487c0ad3799486092c63Ted Kremenek bool isEndif = NextIdx == 0; 259268ee7016a2811803989487c0ad3799486092c63Ted Kremenek 260268ee7016a2811803989487c0ad3799486092c63Ted Kremenek // This case can occur when we see something like this: 261268ee7016a2811803989487c0ad3799486092c63Ted Kremenek // 262268ee7016a2811803989487c0ad3799486092c63Ted Kremenek // #if ... 263268ee7016a2811803989487c0ad3799486092c63Ted Kremenek // /* a comment or nothing */ 264268ee7016a2811803989487c0ad3799486092c63Ted Kremenek // #elif 265268ee7016a2811803989487c0ad3799486092c63Ted Kremenek // 266268ee7016a2811803989487c0ad3799486092c63Ted Kremenek // If we are skipping the first #if block it will be the case that CurPtr 267268ee7016a2811803989487c0ad3799486092c63Ted Kremenek // already points 'elif'. Just return. 268268ee7016a2811803989487c0ad3799486092c63Ted Kremenek 26941a2660377d215d004fe413c03874bd066b5384cTed Kremenek if (CurPtr > HashEntryI) { 27041a2660377d215d004fe413c03874bd066b5384cTed Kremenek assert(CurPtr == HashEntryI + DISK_TOKEN_SIZE); 271268ee7016a2811803989487c0ad3799486092c63Ted Kremenek // Did we reach a #endif? If so, go ahead and consume that token as well. 272268ee7016a2811803989487c0ad3799486092c63Ted Kremenek if (isEndif) 273e5680f3cd678014cf0872d34726dc804b0cbbdd4Ted Kremenek CurPtr += DISK_TOKEN_SIZE*2; 274268ee7016a2811803989487c0ad3799486092c63Ted Kremenek else 27541a2660377d215d004fe413c03874bd066b5384cTed Kremenek LastHashTokPtr = HashEntryI; 276268ee7016a2811803989487c0ad3799486092c63Ted Kremenek 277268ee7016a2811803989487c0ad3799486092c63Ted Kremenek return isEndif; 278268ee7016a2811803989487c0ad3799486092c63Ted Kremenek } 279268ee7016a2811803989487c0ad3799486092c63Ted Kremenek 280268ee7016a2811803989487c0ad3799486092c63Ted Kremenek // Otherwise, we need to advance. Update CurPtr to point to the '#' token. 28141a2660377d215d004fe413c03874bd066b5384cTed Kremenek CurPtr = HashEntryI; 282268ee7016a2811803989487c0ad3799486092c63Ted Kremenek 283268ee7016a2811803989487c0ad3799486092c63Ted Kremenek // Update the location of the last observed '#'. This is useful if we 284268ee7016a2811803989487c0ad3799486092c63Ted Kremenek // are skipping multiple blocks. 285268ee7016a2811803989487c0ad3799486092c63Ted Kremenek LastHashTokPtr = CurPtr; 286268ee7016a2811803989487c0ad3799486092c63Ted Kremenek 287e5680f3cd678014cf0872d34726dc804b0cbbdd4Ted Kremenek // Skip the '#' token. 288da9d61c96c412f6babc7f824152609562f302388Chris Lattner assert(((tok::TokenKind)*CurPtr) == tok::hash); 289e5680f3cd678014cf0872d34726dc804b0cbbdd4Ted Kremenek CurPtr += DISK_TOKEN_SIZE; 290e5680f3cd678014cf0872d34726dc804b0cbbdd4Ted Kremenek 291268ee7016a2811803989487c0ad3799486092c63Ted Kremenek // Did we reach a #endif? If so, go ahead and consume that token as well. 292e5680f3cd678014cf0872d34726dc804b0cbbdd4Ted Kremenek if (isEndif) { CurPtr += DISK_TOKEN_SIZE*2; } 293268ee7016a2811803989487c0ad3799486092c63Ted Kremenek 294268ee7016a2811803989487c0ad3799486092c63Ted Kremenek return isEndif; 295268ee7016a2811803989487c0ad3799486092c63Ted Kremenek} 296268ee7016a2811803989487c0ad3799486092c63Ted Kremenek 29730a12ec2a7f331d9e08acabe7cda853aaa7ba54bTed KremenekSourceLocation PTHLexer::getSourceLocation() { 2981b5285e1ba31975864da356b2ed927e87670e654Chris Lattner // getSourceLocation is not on the hot path. It is used to get the location 2991b5285e1ba31975864da356b2ed927e87670e654Chris Lattner // of the next token when transitioning back to this lexer when done 30030a12ec2a7f331d9e08acabe7cda853aaa7ba54bTed Kremenek // handling a #included file. Just read the necessary data from the token 30130a12ec2a7f331d9e08acabe7cda853aaa7ba54bTed Kremenek // data buffer to construct the SourceLocation object. 30230a12ec2a7f331d9e08acabe7cda853aaa7ba54bTed Kremenek // NOTE: This is a virtual function; hence it is defined out-of-line. 303b248d53f2599d8e7b53b144b713e163ca521ffcaTed Kremenek const unsigned char *OffsetPtr = CurPtr + (DISK_TOKEN_SIZE - 4); 3045ff4317536dbd7f03332bb250c8b35ec04a6f5dbChris Lattner uint32_t Offset = ReadLE32(OffsetPtr); 3051b5285e1ba31975864da356b2ed927e87670e654Chris Lattner return FileStartLoc.getFileLocWithOffset(Offset); 30630a12ec2a7f331d9e08acabe7cda853aaa7ba54bTed Kremenek} 30730a12ec2a7f331d9e08acabe7cda853aaa7ba54bTed Kremenek 3085f074266cc59563036c40516c814d63825723e20Ted Kremenek//===----------------------------------------------------------------------===// 3090c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek// Internal Data Structures for PTH file lookup and resolving identifiers. 3100c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek//===----------------------------------------------------------------------===// 3110c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek 3120c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek 3130c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek/// PTHFileLookup - This internal data structure is used by the PTHManager 3140c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek/// to map from FileEntry objects managed by FileManager to offsets within 3150c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek/// the PTH file. 3160c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremeneknamespace { 3170c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenekclass VISIBILITY_HIDDEN PTHFileLookup { 3180c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenekpublic: 3190c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek class Val { 320fb645b6547b75ddc2e3c7ab2126ad8beeefca62dTed Kremenek uint32_t TokenOff; 321fb645b6547b75ddc2e3c7ab2126ad8beeefca62dTed Kremenek uint32_t PPCondOff; 3220c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek public: 323fb645b6547b75ddc2e3c7ab2126ad8beeefca62dTed Kremenek Val() : TokenOff(~0) {} 324277faca30c9f8f72b79f55695cbe3395ec246e7cTed Kremenek Val(uint32_t toff, uint32_t poff) 325277faca30c9f8f72b79f55695cbe3395ec246e7cTed Kremenek : TokenOff(toff), PPCondOff(poff) {} 3260c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek 3271b5285e1ba31975864da356b2ed927e87670e654Chris Lattner bool isValid() const { return TokenOff != ~((uint32_t)0); } 3281b5285e1ba31975864da356b2ed927e87670e654Chris Lattner 329fb645b6547b75ddc2e3c7ab2126ad8beeefca62dTed Kremenek uint32_t getTokenOffset() const { 3301b5285e1ba31975864da356b2ed927e87670e654Chris Lattner assert(isValid() && "PTHFileLookup entry initialized."); 331fb645b6547b75ddc2e3c7ab2126ad8beeefca62dTed Kremenek return TokenOff; 3320c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek } 3330c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek 334b70e3dafb9618f34017061400dc19ac5e3539a6dTed Kremenek uint32_t getPPCondOffset() const { 3351b5285e1ba31975864da356b2ed927e87670e654Chris Lattner assert(isValid() && "PTHFileLookup entry initialized."); 336fb645b6547b75ddc2e3c7ab2126ad8beeefca62dTed Kremenek return PPCondOff; 337277faca30c9f8f72b79f55695cbe3395ec246e7cTed Kremenek } 3380c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek }; 3390c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek 3400c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenekprivate: 3410c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek llvm::StringMap<Val> FileMap; 3420c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek 3430c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenekpublic: 3440c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek PTHFileLookup() {}; 3450c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek 346cdd8f2153e18796e9e2a126ebcbd4f3e1bd7135bTed Kremenek bool isEmpty() const { 347cdd8f2153e18796e9e2a126ebcbd4f3e1bd7135bTed Kremenek return FileMap.empty(); 348cdd8f2153e18796e9e2a126ebcbd4f3e1bd7135bTed Kremenek } 349cdd8f2153e18796e9e2a126ebcbd4f3e1bd7135bTed Kremenek 3500c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek Val Lookup(const FileEntry* FE) { 3510c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek const char* s = FE->getName(); 3520c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek unsigned size = strlen(s); 3530c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek return FileMap.GetOrCreateValue(s, s+size).getValue(); 3540c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek } 3550c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek 356da9d61c96c412f6babc7f824152609562f302388Chris Lattner void ReadTable(const unsigned char* D) { 3575ff4317536dbd7f03332bb250c8b35ec04a6f5dbChris Lattner uint32_t N = ReadLE32(D); // Read the length of the table. 3580c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek 3590c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek for ( ; N > 0; --N) { // The rest of the data is the table itself. 3605ff4317536dbd7f03332bb250c8b35ec04a6f5dbChris Lattner uint32_t Len = ReadLE32(D); 361da9d61c96c412f6babc7f824152609562f302388Chris Lattner const char* s = (const char *)D; 3621b5285e1ba31975864da356b2ed927e87670e654Chris Lattner D += Len; 363b70e3dafb9618f34017061400dc19ac5e3539a6dTed Kremenek 3645ff4317536dbd7f03332bb250c8b35ec04a6f5dbChris Lattner uint32_t TokenOff = ReadLE32(D); 3655ff4317536dbd7f03332bb250c8b35ec04a6f5dbChris Lattner uint32_t PPCondOff = ReadLE32(D); 366b70e3dafb9618f34017061400dc19ac5e3539a6dTed Kremenek 3671b5285e1ba31975864da356b2ed927e87670e654Chris Lattner FileMap.GetOrCreateValue(s, s+Len).getValue() = 368277faca30c9f8f72b79f55695cbe3395ec246e7cTed Kremenek Val(TokenOff, PPCondOff); 3690c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek } 3700c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek } 3710c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek}; 3720c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek} // end anonymous namespace 3730c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek 3740c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek//===----------------------------------------------------------------------===// 3750c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek// PTHManager methods. 3760c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek//===----------------------------------------------------------------------===// 3770c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek 3780c6a77bc1f52f282a969538f139ebde429076ed3Ted KremenekPTHManager::PTHManager(const llvm::MemoryBuffer* buf, void* fileLookup, 379da9d61c96c412f6babc7f824152609562f302388Chris Lattner const unsigned char* idDataTable, 380da9d61c96c412f6babc7f824152609562f302388Chris Lattner IdentifierInfo** perIDCache, 381277faca30c9f8f72b79f55695cbe3395ec246e7cTed Kremenek const unsigned char* sortedIdTable, unsigned numIds, 382277faca30c9f8f72b79f55695cbe3395ec246e7cTed Kremenek const unsigned char* spellingBase) 3836183e4815a4019e97ad01bd880f12355599b75fdTed Kremenek: Buf(buf), PerIDCache(perIDCache), FileLookup(fileLookup), 38472b1b15ee88aac0a63e2c1dc53fe22f5ab297b20Ted Kremenek IdDataTable(idDataTable), SortedIdTable(sortedIdTable), 385277faca30c9f8f72b79f55695cbe3395ec246e7cTed Kremenek NumIds(numIds), PP(0), SpellingBase(spellingBase) {} 3860c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek 3870c6a77bc1f52f282a969538f139ebde429076ed3Ted KremenekPTHManager::~PTHManager() { 3880c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek delete Buf; 3890c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek delete (PTHFileLookup*) FileLookup; 3900e50b6e7c104d00614baa3d80df62f1630a94d9cTed Kremenek free(PerIDCache); 3910c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek} 3920c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek 3938a6aec620dbec1f292fe4116c0373ac81ab90234Ted KremenekPTHManager* PTHManager::Create(const std::string& file, Diagnostic* Diags) { 3940c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek // Memory map the PTH file. 3950c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek llvm::OwningPtr<llvm::MemoryBuffer> 3960c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek File(llvm::MemoryBuffer::getFile(file.c_str())); 3970c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek 3988a6aec620dbec1f292fe4116c0373ac81ab90234Ted Kremenek if (!File) { 3998a6aec620dbec1f292fe4116c0373ac81ab90234Ted Kremenek if (Diags) { 4008a6aec620dbec1f292fe4116c0373ac81ab90234Ted Kremenek unsigned DiagID = Diags->getCustomDiagID(Diagnostic::Note, 4018a6aec620dbec1f292fe4116c0373ac81ab90234Ted Kremenek "PTH file %0 could not be read"); 4028a6aec620dbec1f292fe4116c0373ac81ab90234Ted Kremenek Diags->Report(FullSourceLoc(), DiagID) << file; 4038a6aec620dbec1f292fe4116c0373ac81ab90234Ted Kremenek } 4048a6aec620dbec1f292fe4116c0373ac81ab90234Ted Kremenek 4050c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek return 0; 4068a6aec620dbec1f292fe4116c0373ac81ab90234Ted Kremenek } 4070c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek 4080c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek // Get the buffer ranges and check if there are at least three 32-bit 4090c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek // words at the end of the file. 410da9d61c96c412f6babc7f824152609562f302388Chris Lattner const unsigned char* BufBeg = (unsigned char*)File->getBufferStart(); 411da9d61c96c412f6babc7f824152609562f302388Chris Lattner const unsigned char* BufEnd = (unsigned char*)File->getBufferEnd(); 412e1b6498c41b94c3bc5cede17b0702282543385efTed Kremenek 413e1b6498c41b94c3bc5cede17b0702282543385efTed Kremenek // Check the prologue of the file. 4144adc71ae2cfc190f8d2cf58876e2a7893aa74ee0Ted Kremenek if ((BufEnd - BufBeg) < (signed) (sizeof("cfe-pth") + 3 + 4) || 415e1b6498c41b94c3bc5cede17b0702282543385efTed Kremenek memcmp(BufBeg, "cfe-pth", sizeof("cfe-pth") - 1) != 0) 416e1b6498c41b94c3bc5cede17b0702282543385efTed Kremenek return 0; 4170c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek 41867d15050bbea16ae256e204ecd464f2e454c3c99Ted Kremenek // Read the PTH version. 419e1b6498c41b94c3bc5cede17b0702282543385efTed Kremenek const unsigned char *p = BufBeg + (sizeof("cfe-pth") - 1); 42067d15050bbea16ae256e204ecd464f2e454c3c99Ted Kremenek unsigned Version = ReadLE32(p); 42167d15050bbea16ae256e204ecd464f2e454c3c99Ted Kremenek 42267d15050bbea16ae256e204ecd464f2e454c3c99Ted Kremenek if (Version != PTHManager::Version) 42367d15050bbea16ae256e204ecd464f2e454c3c99Ted Kremenek return 0; 42467d15050bbea16ae256e204ecd464f2e454c3c99Ted Kremenek 42567d15050bbea16ae256e204ecd464f2e454c3c99Ted Kremenek // Compute the address of the index table at the end of the PTH file. 426e1b6498c41b94c3bc5cede17b0702282543385efTed Kremenek const unsigned char *EndTable = BufBeg + ReadLE32(p); 427e1b6498c41b94c3bc5cede17b0702282543385efTed Kremenek 428e1b6498c41b94c3bc5cede17b0702282543385efTed Kremenek if (EndTable >= BufEnd) 429e1b6498c41b94c3bc5cede17b0702282543385efTed Kremenek return 0; 4300c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek 4310c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek // Construct the file lookup table. This will be used for mapping from 4320c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek // FileEntry*'s to cached tokens. 433da9d61c96c412f6babc7f824152609562f302388Chris Lattner const unsigned char* FileTableOffset = EndTable + sizeof(uint32_t)*3; 4345ff4317536dbd7f03332bb250c8b35ec04a6f5dbChris Lattner const unsigned char* FileTable = BufBeg + ReadLE32(FileTableOffset); 4350c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek 4360c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek if (!(FileTable > BufBeg && FileTable < BufEnd)) { 4370c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek assert(false && "Invalid PTH file."); 4380c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek return 0; // FIXME: Proper error diagnostic? 4390c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek } 4400c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek 4410c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek llvm::OwningPtr<PTHFileLookup> FL(new PTHFileLookup()); 4420c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek FL->ReadTable(FileTable); 443cdd8f2153e18796e9e2a126ebcbd4f3e1bd7135bTed Kremenek 444cdd8f2153e18796e9e2a126ebcbd4f3e1bd7135bTed Kremenek if (FL->isEmpty()) 445cdd8f2153e18796e9e2a126ebcbd4f3e1bd7135bTed Kremenek return 0; 4460c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek 4470c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek // Get the location of the table mapping from persistent ids to the 4480c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek // data needed to reconstruct identifiers. 449da9d61c96c412f6babc7f824152609562f302388Chris Lattner const unsigned char* IDTableOffset = EndTable + sizeof(uint32_t)*1; 4505ff4317536dbd7f03332bb250c8b35ec04a6f5dbChris Lattner const unsigned char* IData = BufBeg + ReadLE32(IDTableOffset); 451cdd8f2153e18796e9e2a126ebcbd4f3e1bd7135bTed Kremenek 452cdd8f2153e18796e9e2a126ebcbd4f3e1bd7135bTed Kremenek if (!(IData >= BufBeg && IData < BufEnd)) { 4530c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek assert(false && "Invalid PTH file."); 4540c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek return 0; // FIXME: Proper error diagnostic? 4550c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek } 4560c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek 45772b1b15ee88aac0a63e2c1dc53fe22f5ab297b20Ted Kremenek // Get the location of the lexigraphically-sorted table of persistent IDs. 458da9d61c96c412f6babc7f824152609562f302388Chris Lattner const unsigned char* SortedIdTableOffset = EndTable + sizeof(uint32_t)*2; 4595ff4317536dbd7f03332bb250c8b35ec04a6f5dbChris Lattner const unsigned char* SortedIdTable = BufBeg + ReadLE32(SortedIdTableOffset); 460cdd8f2153e18796e9e2a126ebcbd4f3e1bd7135bTed Kremenek if (!(SortedIdTable >= BufBeg && SortedIdTable < BufEnd)) { 46172b1b15ee88aac0a63e2c1dc53fe22f5ab297b20Ted Kremenek assert(false && "Invalid PTH file."); 46272b1b15ee88aac0a63e2c1dc53fe22f5ab297b20Ted Kremenek return 0; // FIXME: Proper error diagnostic? 46372b1b15ee88aac0a63e2c1dc53fe22f5ab297b20Ted Kremenek } 46472b1b15ee88aac0a63e2c1dc53fe22f5ab297b20Ted Kremenek 465277faca30c9f8f72b79f55695cbe3395ec246e7cTed Kremenek // Get the location of the spelling cache. 466277faca30c9f8f72b79f55695cbe3395ec246e7cTed Kremenek const unsigned char* spellingBaseOffset = EndTable + sizeof(uint32_t)*4; 467277faca30c9f8f72b79f55695cbe3395ec246e7cTed Kremenek const unsigned char* spellingBase = BufBeg + ReadLE32(spellingBaseOffset); 468277faca30c9f8f72b79f55695cbe3395ec246e7cTed Kremenek if (!(spellingBase >= BufBeg && spellingBase < BufEnd)) { 469277faca30c9f8f72b79f55695cbe3395ec246e7cTed Kremenek assert(false && "Invalid PTH file."); 470277faca30c9f8f72b79f55695cbe3395ec246e7cTed Kremenek return 0; 471277faca30c9f8f72b79f55695cbe3395ec246e7cTed Kremenek } 472277faca30c9f8f72b79f55695cbe3395ec246e7cTed Kremenek 4736183e4815a4019e97ad01bd880f12355599b75fdTed Kremenek // Get the number of IdentifierInfos and pre-allocate the identifier cache. 4745ff4317536dbd7f03332bb250c8b35ec04a6f5dbChris Lattner uint32_t NumIds = ReadLE32(IData); 475cdd8f2153e18796e9e2a126ebcbd4f3e1bd7135bTed Kremenek 4766183e4815a4019e97ad01bd880f12355599b75fdTed Kremenek // Pre-allocate the peristent ID -> IdentifierInfo* cache. We use calloc() 4776183e4815a4019e97ad01bd880f12355599b75fdTed Kremenek // so that we in the best case only zero out memory once when the OS returns 4786183e4815a4019e97ad01bd880f12355599b75fdTed Kremenek // us new pages. 479cdd8f2153e18796e9e2a126ebcbd4f3e1bd7135bTed Kremenek IdentifierInfo** PerIDCache = 0; 4806183e4815a4019e97ad01bd880f12355599b75fdTed Kremenek 481cdd8f2153e18796e9e2a126ebcbd4f3e1bd7135bTed Kremenek if (NumIds) { 482cdd8f2153e18796e9e2a126ebcbd4f3e1bd7135bTed Kremenek PerIDCache = (IdentifierInfo**)calloc(NumIds, sizeof(*PerIDCache)); 483cdd8f2153e18796e9e2a126ebcbd4f3e1bd7135bTed Kremenek if (!PerIDCache) { 484cdd8f2153e18796e9e2a126ebcbd4f3e1bd7135bTed Kremenek assert(false && "Could not allocate Persistent ID cache."); 485cdd8f2153e18796e9e2a126ebcbd4f3e1bd7135bTed Kremenek return 0; 486cdd8f2153e18796e9e2a126ebcbd4f3e1bd7135bTed Kremenek } 4876183e4815a4019e97ad01bd880f12355599b75fdTed Kremenek } 488cdd8f2153e18796e9e2a126ebcbd4f3e1bd7135bTed Kremenek 48972b1b15ee88aac0a63e2c1dc53fe22f5ab297b20Ted Kremenek // Create the new PTHManager. 49072b1b15ee88aac0a63e2c1dc53fe22f5ab297b20Ted Kremenek return new PTHManager(File.take(), FL.take(), IData, PerIDCache, 491277faca30c9f8f72b79f55695cbe3395ec246e7cTed Kremenek SortedIdTable, NumIds, spellingBase); 4920c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek} 49377ecb3a28f21496ecfdbb3d5f5b66b0d2abf48c9Chris LattnerIdentifierInfo* PTHManager::LazilyCreateIdentifierInfo(unsigned PersistentID) { 4940c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek // Look in the PTH file for the string data for the IdentifierInfo object. 49577ecb3a28f21496ecfdbb3d5f5b66b0d2abf48c9Chris Lattner const unsigned char* TableEntry = IdDataTable + sizeof(uint32_t)*PersistentID; 496da9d61c96c412f6babc7f824152609562f302388Chris Lattner const unsigned char* IDData = 4975ff4317536dbd7f03332bb250c8b35ec04a6f5dbChris Lattner (const unsigned char*)Buf->getBufferStart() + ReadLE32(TableEntry); 498da9d61c96c412f6babc7f824152609562f302388Chris Lattner assert(IDData < (const unsigned char*)Buf->getBufferEnd()); 4990c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek 50072b1b15ee88aac0a63e2c1dc53fe22f5ab297b20Ted Kremenek // Allocate the object. 501da9d61c96c412f6babc7f824152609562f302388Chris Lattner std::pair<IdentifierInfo,const unsigned char*> *Mem = 502da9d61c96c412f6babc7f824152609562f302388Chris Lattner Alloc.Allocate<std::pair<IdentifierInfo,const unsigned char*> >(); 50372b1b15ee88aac0a63e2c1dc53fe22f5ab297b20Ted Kremenek 50472b1b15ee88aac0a63e2c1dc53fe22f5ab297b20Ted Kremenek Mem->second = IDData; 505ea9c26b3dbd74a1497f5609ae6e19a85f42b6073Ted Kremenek IdentifierInfo *II = new ((void*) Mem) IdentifierInfo(); 5060c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek 50772b1b15ee88aac0a63e2c1dc53fe22f5ab297b20Ted Kremenek // Store the new IdentifierInfo in the cache. 50877ecb3a28f21496ecfdbb3d5f5b66b0d2abf48c9Chris Lattner PerIDCache[PersistentID] = II; 5090c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek return II; 5100c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek} 5110c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek 51272b1b15ee88aac0a63e2c1dc53fe22f5ab297b20Ted KremenekIdentifierInfo* PTHManager::get(const char *NameStart, const char *NameEnd) { 51372b1b15ee88aac0a63e2c1dc53fe22f5ab297b20Ted Kremenek unsigned min = 0; 51472b1b15ee88aac0a63e2c1dc53fe22f5ab297b20Ted Kremenek unsigned max = NumIds; 5151b5285e1ba31975864da356b2ed927e87670e654Chris Lattner unsigned Len = NameEnd - NameStart; 51672b1b15ee88aac0a63e2c1dc53fe22f5ab297b20Ted Kremenek 51772b1b15ee88aac0a63e2c1dc53fe22f5ab297b20Ted Kremenek do { 51872b1b15ee88aac0a63e2c1dc53fe22f5ab297b20Ted Kremenek unsigned i = (max - min) / 2 + min; 5191b5285e1ba31975864da356b2ed927e87670e654Chris Lattner const unsigned char *Ptr = SortedIdTable + (i * 4); 52072b1b15ee88aac0a63e2c1dc53fe22f5ab297b20Ted Kremenek 52172b1b15ee88aac0a63e2c1dc53fe22f5ab297b20Ted Kremenek // Read the persistentID. 5225ff4317536dbd7f03332bb250c8b35ec04a6f5dbChris Lattner unsigned perID = ReadLE32(Ptr); 52372b1b15ee88aac0a63e2c1dc53fe22f5ab297b20Ted Kremenek 52472b1b15ee88aac0a63e2c1dc53fe22f5ab297b20Ted Kremenek // Get the IdentifierInfo. 52572b1b15ee88aac0a63e2c1dc53fe22f5ab297b20Ted Kremenek IdentifierInfo* II = GetIdentifierInfo(perID); 52672b1b15ee88aac0a63e2c1dc53fe22f5ab297b20Ted Kremenek 52772b1b15ee88aac0a63e2c1dc53fe22f5ab297b20Ted Kremenek // First compare the lengths. 52872b1b15ee88aac0a63e2c1dc53fe22f5ab297b20Ted Kremenek unsigned IILen = II->getLength(); 5291b5285e1ba31975864da356b2ed927e87670e654Chris Lattner if (Len < IILen) goto IsLess; 5301b5285e1ba31975864da356b2ed927e87670e654Chris Lattner if (Len > IILen) goto IsGreater; 53172b1b15ee88aac0a63e2c1dc53fe22f5ab297b20Ted Kremenek 53272b1b15ee88aac0a63e2c1dc53fe22f5ab297b20Ted Kremenek // Now compare the strings! 53372b1b15ee88aac0a63e2c1dc53fe22f5ab297b20Ted Kremenek { 5341b5285e1ba31975864da356b2ed927e87670e654Chris Lattner signed comp = strncmp(NameStart, II->getName(), Len); 53572b1b15ee88aac0a63e2c1dc53fe22f5ab297b20Ted Kremenek if (comp < 0) goto IsLess; 53672b1b15ee88aac0a63e2c1dc53fe22f5ab297b20Ted Kremenek if (comp > 0) goto IsGreater; 53772b1b15ee88aac0a63e2c1dc53fe22f5ab297b20Ted Kremenek } 53872b1b15ee88aac0a63e2c1dc53fe22f5ab297b20Ted Kremenek // We found a match! 53972b1b15ee88aac0a63e2c1dc53fe22f5ab297b20Ted Kremenek return II; 54072b1b15ee88aac0a63e2c1dc53fe22f5ab297b20Ted Kremenek 54172b1b15ee88aac0a63e2c1dc53fe22f5ab297b20Ted Kremenek IsGreater: 54272b1b15ee88aac0a63e2c1dc53fe22f5ab297b20Ted Kremenek if (i == min) break; 54372b1b15ee88aac0a63e2c1dc53fe22f5ab297b20Ted Kremenek min = i; 54472b1b15ee88aac0a63e2c1dc53fe22f5ab297b20Ted Kremenek continue; 54572b1b15ee88aac0a63e2c1dc53fe22f5ab297b20Ted Kremenek 54672b1b15ee88aac0a63e2c1dc53fe22f5ab297b20Ted Kremenek IsLess: 54772b1b15ee88aac0a63e2c1dc53fe22f5ab297b20Ted Kremenek max = i; 54872b1b15ee88aac0a63e2c1dc53fe22f5ab297b20Ted Kremenek assert(!(max == min) || (min == i)); 54972b1b15ee88aac0a63e2c1dc53fe22f5ab297b20Ted Kremenek } 550e1deaac73379eeb9864215c7979f6005ebd74cefTed Kremenek while (min != max); 55172b1b15ee88aac0a63e2c1dc53fe22f5ab297b20Ted Kremenek 55272b1b15ee88aac0a63e2c1dc53fe22f5ab297b20Ted Kremenek return 0; 55372b1b15ee88aac0a63e2c1dc53fe22f5ab297b20Ted Kremenek} 55472b1b15ee88aac0a63e2c1dc53fe22f5ab297b20Ted Kremenek 55572b1b15ee88aac0a63e2c1dc53fe22f5ab297b20Ted Kremenek 556f056d92e182cbe4f62c8d14102544dc38066dabcChris LattnerPTHLexer *PTHManager::CreateLexer(FileID FID) { 557f056d92e182cbe4f62c8d14102544dc38066dabcChris Lattner const FileEntry *FE = PP->getSourceManager().getFileEntryForID(FID); 5580c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek if (!FE) 5590c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek return 0; 5600c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek 5610c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek // Lookup the FileEntry object in our file lookup data structure. It will 5620c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek // return a variant that indicates whether or not there is an offset within 5630c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek // the PTH file that contains cached tokens. 5642b2453a7d8fe732561795431f39ceb2b2a832d84Chris Lattner PTHFileLookup::Val FileData = ((PTHFileLookup*)FileLookup)->Lookup(FE); 5650c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek 566fb645b6547b75ddc2e3c7ab2126ad8beeefca62dTed Kremenek if (!FileData.isValid()) // No tokens available. 5670c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek return 0; 5680c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek 569da9d61c96c412f6babc7f824152609562f302388Chris Lattner const unsigned char *BufStart = (const unsigned char *)Buf->getBufferStart(); 5700c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek // Compute the offset of the token data within the buffer. 571da9d61c96c412f6babc7f824152609562f302388Chris Lattner const unsigned char* data = BufStart + FileData.getTokenOffset(); 572268ee7016a2811803989487c0ad3799486092c63Ted Kremenek 573268ee7016a2811803989487c0ad3799486092c63Ted Kremenek // Get the location of pp-conditional table. 574da9d61c96c412f6babc7f824152609562f302388Chris Lattner const unsigned char* ppcond = BufStart + FileData.getPPCondOffset(); 5755ff4317536dbd7f03332bb250c8b35ec04a6f5dbChris Lattner uint32_t Len = ReadLE32(ppcond); 5761b5285e1ba31975864da356b2ed927e87670e654Chris Lattner if (Len == 0) ppcond = 0; 57732a8ad526f9bc00539f000a2dd1ac3e167db61c1Ted Kremenek 57872b1b15ee88aac0a63e2c1dc53fe22f5ab297b20Ted Kremenek assert(PP && "No preprocessor set yet!"); 579277faca30c9f8f72b79f55695cbe3395ec246e7cTed Kremenek return new PTHLexer(*PP, FID, data, ppcond, *this); 5800c6a77bc1f52f282a969538f139ebde429076ed3Ted Kremenek} 581