PTHLexer.cpp revision 89d7ee9619d2dbdfa8d956a695c612a104a92cad
1274b20863a728cc6a31ee75c670e3733600c1531Ted Kremenek//===--- PTHLexer.cpp - Lex from a token stream ---------------------------===// 2274b20863a728cc6a31ee75c670e3733600c1531Ted Kremenek// 3274b20863a728cc6a31ee75c670e3733600c1531Ted Kremenek// The LLVM Compiler Infrastructure 4274b20863a728cc6a31ee75c670e3733600c1531Ted Kremenek// 5274b20863a728cc6a31ee75c670e3733600c1531Ted Kremenek// This file is distributed under the University of Illinois Open Source 6274b20863a728cc6a31ee75c670e3733600c1531Ted Kremenek// License. See LICENSE.TXT for details. 7274b20863a728cc6a31ee75c670e3733600c1531Ted Kremenek// 8274b20863a728cc6a31ee75c670e3733600c1531Ted Kremenek//===----------------------------------------------------------------------===// 9274b20863a728cc6a31ee75c670e3733600c1531Ted Kremenek// 10274b20863a728cc6a31ee75c670e3733600c1531Ted Kremenek// This file implements the PTHLexer interface. 11274b20863a728cc6a31ee75c670e3733600c1531Ted Kremenek// 12274b20863a728cc6a31ee75c670e3733600c1531Ted Kremenek//===----------------------------------------------------------------------===// 13274b20863a728cc6a31ee75c670e3733600c1531Ted Kremenek 14274b20863a728cc6a31ee75c670e3733600c1531Ted Kremenek#include "clang/Lex/PTHLexer.h" 15274b20863a728cc6a31ee75c670e3733600c1531Ted Kremenek#include "clang/Lex/Preprocessor.h" 16274b20863a728cc6a31ee75c670e3733600c1531Ted Kremenek#include "clang/Basic/TokenKinds.h" 17274b20863a728cc6a31ee75c670e3733600c1531Ted Kremenekusing namespace clang; 18274b20863a728cc6a31ee75c670e3733600c1531Ted Kremenek 19274b20863a728cc6a31ee75c670e3733600c1531Ted KremenekPTHLexer::PTHLexer(Preprocessor& pp, SourceLocation fileloc, 20452e37837a48b2f0ced144784277fd4d28cbede9Ted Kremenek const Token *TokArray, unsigned NumTokens) 21452e37837a48b2f0ced144784277fd4d28cbede9Ted Kremenek : PreprocessorLexer(&pp, fileloc), 22452e37837a48b2f0ced144784277fd4d28cbede9Ted Kremenek Tokens(TokArray), 2331aba425a01c8c957e662ccfaa71f923d0f0932aTed Kremenek LastTokenIdx(NumTokens - 1), 2431aba425a01c8c957e662ccfaa71f923d0f0932aTed Kremenek CurTokenIdx(0) { 25274b20863a728cc6a31ee75c670e3733600c1531Ted Kremenek 26d6f53dc4951aace69014619761760addac9e59ecTed Kremenek assert(NumTokens >= 1); 2731aba425a01c8c957e662ccfaa71f923d0f0932aTed Kremenek assert(Tokens[LastTokenIdx].is(tok::eof)); 28274b20863a728cc6a31ee75c670e3733600c1531Ted Kremenek} 29274b20863a728cc6a31ee75c670e3733600c1531Ted Kremenek 3089d7ee9619d2dbdfa8d956a695c612a104a92cadTed KremenekToken PTHLexer::GetToken() { 3189d7ee9619d2dbdfa8d956a695c612a104a92cadTed Kremenek Token Tok = Tokens[CurTokenIdx]; 3289d7ee9619d2dbdfa8d956a695c612a104a92cadTed Kremenek 3389d7ee9619d2dbdfa8d956a695c612a104a92cadTed Kremenek // If we are in raw mode, zero out identifier pointers. This is 3489d7ee9619d2dbdfa8d956a695c612a104a92cadTed Kremenek // needed for 'pragma poison'. Note that this requires that the Preprocessor 3589d7ee9619d2dbdfa8d956a695c612a104a92cadTed Kremenek // can go back to the original source when it calls getSpelling(). 3689d7ee9619d2dbdfa8d956a695c612a104a92cadTed Kremenek if (LexingRawMode && Tok.is(tok::identifier)) 3789d7ee9619d2dbdfa8d956a695c612a104a92cadTed Kremenek Tok.setIdentifierInfo(0); 3889d7ee9619d2dbdfa8d956a695c612a104a92cadTed Kremenek 3989d7ee9619d2dbdfa8d956a695c612a104a92cadTed Kremenek return Tok; 4089d7ee9619d2dbdfa8d956a695c612a104a92cadTed Kremenek} 4189d7ee9619d2dbdfa8d956a695c612a104a92cadTed Kremenek 42274b20863a728cc6a31ee75c670e3733600c1531Ted Kremenekvoid PTHLexer::Lex(Token& Tok) { 43d6f53dc4951aace69014619761760addac9e59ecTed KremenekLexNextToken: 4431aba425a01c8c957e662ccfaa71f923d0f0932aTed Kremenek if (AtLastToken()) { 45d6f53dc4951aace69014619761760addac9e59ecTed Kremenek if (ParsingPreprocessorDirective) { 46274b20863a728cc6a31ee75c670e3733600c1531Ted Kremenek ParsingPreprocessorDirective = false; 4731aba425a01c8c957e662ccfaa71f923d0f0932aTed Kremenek Tok = GetToken(); 48d6f53dc4951aace69014619761760addac9e59ecTed Kremenek Tok.setKind(tok::eom); 49d6f53dc4951aace69014619761760addac9e59ecTed Kremenek MIOpt.ReadToken(); 50d6f53dc4951aace69014619761760addac9e59ecTed Kremenek return; 51274b20863a728cc6a31ee75c670e3733600c1531Ted Kremenek } 52274b20863a728cc6a31ee75c670e3733600c1531Ted Kremenek 53d6f53dc4951aace69014619761760addac9e59ecTed Kremenek assert(!LexingRawMode && "PTHLexer cannot lex in raw mode."); 54d6f53dc4951aace69014619761760addac9e59ecTed Kremenek 55d6f53dc4951aace69014619761760addac9e59ecTed Kremenek // FIXME: Issue diagnostics similar to Lexer. 56d6f53dc4951aace69014619761760addac9e59ecTed Kremenek PP->HandleEndOfFile(Tok, false); 57274b20863a728cc6a31ee75c670e3733600c1531Ted Kremenek return; 58274b20863a728cc6a31ee75c670e3733600c1531Ted Kremenek } 59274b20863a728cc6a31ee75c670e3733600c1531Ted Kremenek 6031aba425a01c8c957e662ccfaa71f923d0f0932aTed Kremenek Tok = GetToken(); 61274b20863a728cc6a31ee75c670e3733600c1531Ted Kremenek 62d6f53dc4951aace69014619761760addac9e59ecTed Kremenek // Don't advance to the next token yet. Check if we are at the 63d6f53dc4951aace69014619761760addac9e59ecTed Kremenek // start of a new line and we're processing a directive. If so, we 64d6f53dc4951aace69014619761760addac9e59ecTed Kremenek // consume this token twice, once as an tok::eom. 65d6f53dc4951aace69014619761760addac9e59ecTed Kremenek if (Tok.isAtStartOfLine() && ParsingPreprocessorDirective) { 66d6f53dc4951aace69014619761760addac9e59ecTed Kremenek ParsingPreprocessorDirective = false; 67d6f53dc4951aace69014619761760addac9e59ecTed Kremenek Tok.setKind(tok::eom); 68274b20863a728cc6a31ee75c670e3733600c1531Ted Kremenek MIOpt.ReadToken(); 69274b20863a728cc6a31ee75c670e3733600c1531Ted Kremenek return; 70274b20863a728cc6a31ee75c670e3733600c1531Ted Kremenek } 71d6f53dc4951aace69014619761760addac9e59ecTed Kremenek 72d6f53dc4951aace69014619761760addac9e59ecTed Kremenek // Advance to the next token. 7331aba425a01c8c957e662ccfaa71f923d0f0932aTed Kremenek AdvanceToken(); 74d6f53dc4951aace69014619761760addac9e59ecTed Kremenek 75d6f53dc4951aace69014619761760addac9e59ecTed Kremenek if (Tok.is(tok::hash)) { 76d6f53dc4951aace69014619761760addac9e59ecTed Kremenek if (Tok.isAtStartOfLine() && !LexingRawMode) { 77d6f53dc4951aace69014619761760addac9e59ecTed Kremenek PP->HandleDirective(Tok); 78274b20863a728cc6a31ee75c670e3733600c1531Ted Kremenek 79d6f53dc4951aace69014619761760addac9e59ecTed Kremenek if (PP->isCurrentLexer(this)) 80d6f53dc4951aace69014619761760addac9e59ecTed Kremenek goto LexNextToken; 81d6f53dc4951aace69014619761760addac9e59ecTed Kremenek 82d6f53dc4951aace69014619761760addac9e59ecTed Kremenek return PP->Lex(Tok); 83d6f53dc4951aace69014619761760addac9e59ecTed Kremenek } 84274b20863a728cc6a31ee75c670e3733600c1531Ted Kremenek } 85d6f53dc4951aace69014619761760addac9e59ecTed Kremenek 86274b20863a728cc6a31ee75c670e3733600c1531Ted Kremenek MIOpt.ReadToken(); 87d6f53dc4951aace69014619761760addac9e59ecTed Kremenek 88d6f53dc4951aace69014619761760addac9e59ecTed Kremenek if (Tok.is(tok::identifier)) { 89d6f53dc4951aace69014619761760addac9e59ecTed Kremenek if (LexingRawMode) return; 90d6f53dc4951aace69014619761760addac9e59ecTed Kremenek return PP->HandleIdentifier(Tok); 91d6f53dc4951aace69014619761760addac9e59ecTed Kremenek } 92274b20863a728cc6a31ee75c670e3733600c1531Ted Kremenek} 93274b20863a728cc6a31ee75c670e3733600c1531Ted Kremenek 94274b20863a728cc6a31ee75c670e3733600c1531Ted Kremenekvoid PTHLexer::setEOF(Token& Tok) { 9531aba425a01c8c957e662ccfaa71f923d0f0932aTed Kremenek Tok = Tokens[LastTokenIdx]; 96274b20863a728cc6a31ee75c670e3733600c1531Ted Kremenek} 9717ff58a63197b398ae52697b088dc0fb8b255519Ted Kremenek 9817ff58a63197b398ae52697b088dc0fb8b255519Ted Kremenekvoid PTHLexer::DiscardToEndOfLine() { 9917ff58a63197b398ae52697b088dc0fb8b255519Ted Kremenek assert(ParsingPreprocessorDirective && ParsingFilename == false && 10017ff58a63197b398ae52697b088dc0fb8b255519Ted Kremenek "Must be in a preprocessing directive!"); 1014d35da2e41941965bbee8ed7e8c30e7c21000d71Ted Kremenek 1024d35da2e41941965bbee8ed7e8c30e7c21000d71Ted Kremenek // Already at end-of-file? 10331aba425a01c8c957e662ccfaa71f923d0f0932aTed Kremenek if (AtLastToken()) 1044d35da2e41941965bbee8ed7e8c30e7c21000d71Ted Kremenek return; 1054d35da2e41941965bbee8ed7e8c30e7c21000d71Ted Kremenek 1064d35da2e41941965bbee8ed7e8c30e7c21000d71Ted Kremenek // Find the first token that is not the start of the *current* line. 10731aba425a01c8c957e662ccfaa71f923d0f0932aTed Kremenek for (AdvanceToken(); !AtLastToken(); AdvanceToken()) 10831aba425a01c8c957e662ccfaa71f923d0f0932aTed Kremenek if (GetToken().isAtStartOfLine()) 1094d35da2e41941965bbee8ed7e8c30e7c21000d71Ted Kremenek return; 11017ff58a63197b398ae52697b088dc0fb8b255519Ted Kremenek} 111