PTHLexer.cpp revision d6f53dc4951aace69014619761760addac9e59ec
1274b20863a728cc6a31ee75c670e3733600c1531Ted Kremenek//===--- PTHLexer.cpp - Lex from a token stream ---------------------------===// 2274b20863a728cc6a31ee75c670e3733600c1531Ted Kremenek// 3274b20863a728cc6a31ee75c670e3733600c1531Ted Kremenek// The LLVM Compiler Infrastructure 4274b20863a728cc6a31ee75c670e3733600c1531Ted Kremenek// 5274b20863a728cc6a31ee75c670e3733600c1531Ted Kremenek// This file is distributed under the University of Illinois Open Source 6274b20863a728cc6a31ee75c670e3733600c1531Ted Kremenek// License. See LICENSE.TXT for details. 7274b20863a728cc6a31ee75c670e3733600c1531Ted Kremenek// 8274b20863a728cc6a31ee75c670e3733600c1531Ted Kremenek//===----------------------------------------------------------------------===// 9274b20863a728cc6a31ee75c670e3733600c1531Ted Kremenek// 10274b20863a728cc6a31ee75c670e3733600c1531Ted Kremenek// This file implements the PTHLexer interface. 11274b20863a728cc6a31ee75c670e3733600c1531Ted Kremenek// 12274b20863a728cc6a31ee75c670e3733600c1531Ted Kremenek//===----------------------------------------------------------------------===// 13274b20863a728cc6a31ee75c670e3733600c1531Ted Kremenek 14274b20863a728cc6a31ee75c670e3733600c1531Ted Kremenek#include "clang/Lex/PTHLexer.h" 15274b20863a728cc6a31ee75c670e3733600c1531Ted Kremenek#include "clang/Lex/Preprocessor.h" 16274b20863a728cc6a31ee75c670e3733600c1531Ted Kremenek#include "clang/Basic/TokenKinds.h" 17274b20863a728cc6a31ee75c670e3733600c1531Ted Kremenekusing namespace clang; 18274b20863a728cc6a31ee75c670e3733600c1531Ted Kremenek 19274b20863a728cc6a31ee75c670e3733600c1531Ted KremenekPTHLexer::PTHLexer(Preprocessor& pp, SourceLocation fileloc, 20452e37837a48b2f0ced144784277fd4d28cbede9Ted Kremenek const Token *TokArray, unsigned NumTokens) 21452e37837a48b2f0ced144784277fd4d28cbede9Ted Kremenek : PreprocessorLexer(&pp, fileloc), 22452e37837a48b2f0ced144784277fd4d28cbede9Ted Kremenek Tokens(TokArray), 23452e37837a48b2f0ced144784277fd4d28cbede9Ted Kremenek LastToken(NumTokens - 1), 24452e37837a48b2f0ced144784277fd4d28cbede9Ted Kremenek CurToken(0) { 25274b20863a728cc6a31ee75c670e3733600c1531Ted Kremenek 26d6f53dc4951aace69014619761760addac9e59ecTed Kremenek assert(NumTokens >= 1); 27d6f53dc4951aace69014619761760addac9e59ecTed Kremenek assert(Tokens[LastToken].is(tok::eof)); 28274b20863a728cc6a31ee75c670e3733600c1531Ted Kremenek} 29274b20863a728cc6a31ee75c670e3733600c1531Ted Kremenek 30274b20863a728cc6a31ee75c670e3733600c1531Ted Kremenekvoid PTHLexer::Lex(Token& Tok) { 31d6f53dc4951aace69014619761760addac9e59ecTed KremenekLexNextToken: 32d6f53dc4951aace69014619761760addac9e59ecTed Kremenek if (CurToken == LastToken) { 33d6f53dc4951aace69014619761760addac9e59ecTed Kremenek if (ParsingPreprocessorDirective) { 34274b20863a728cc6a31ee75c670e3733600c1531Ted Kremenek ParsingPreprocessorDirective = false; 35d6f53dc4951aace69014619761760addac9e59ecTed Kremenek Tok = Tokens[LastToken]; 36d6f53dc4951aace69014619761760addac9e59ecTed Kremenek Tok.setKind(tok::eom); 37d6f53dc4951aace69014619761760addac9e59ecTed Kremenek MIOpt.ReadToken(); 38d6f53dc4951aace69014619761760addac9e59ecTed Kremenek return; 39274b20863a728cc6a31ee75c670e3733600c1531Ted Kremenek } 40274b20863a728cc6a31ee75c670e3733600c1531Ted Kremenek 41d6f53dc4951aace69014619761760addac9e59ecTed Kremenek assert(!LexingRawMode && "PTHLexer cannot lex in raw mode."); 42d6f53dc4951aace69014619761760addac9e59ecTed Kremenek 43d6f53dc4951aace69014619761760addac9e59ecTed Kremenek // FIXME: Issue diagnostics similar to Lexer. 44d6f53dc4951aace69014619761760addac9e59ecTed Kremenek PP->HandleEndOfFile(Tok, false); 45274b20863a728cc6a31ee75c670e3733600c1531Ted Kremenek return; 46274b20863a728cc6a31ee75c670e3733600c1531Ted Kremenek } 47274b20863a728cc6a31ee75c670e3733600c1531Ted Kremenek 48274b20863a728cc6a31ee75c670e3733600c1531Ted Kremenek Tok = Tokens[CurToken]; 49274b20863a728cc6a31ee75c670e3733600c1531Ted Kremenek 50d6f53dc4951aace69014619761760addac9e59ecTed Kremenek // Don't advance to the next token yet. Check if we are at the 51d6f53dc4951aace69014619761760addac9e59ecTed Kremenek // start of a new line and we're processing a directive. If so, we 52d6f53dc4951aace69014619761760addac9e59ecTed Kremenek // consume this token twice, once as an tok::eom. 53d6f53dc4951aace69014619761760addac9e59ecTed Kremenek if (Tok.isAtStartOfLine() && ParsingPreprocessorDirective) { 54d6f53dc4951aace69014619761760addac9e59ecTed Kremenek ParsingPreprocessorDirective = false; 55d6f53dc4951aace69014619761760addac9e59ecTed Kremenek Tok.setKind(tok::eom); 56274b20863a728cc6a31ee75c670e3733600c1531Ted Kremenek MIOpt.ReadToken(); 57274b20863a728cc6a31ee75c670e3733600c1531Ted Kremenek return; 58274b20863a728cc6a31ee75c670e3733600c1531Ted Kremenek } 59d6f53dc4951aace69014619761760addac9e59ecTed Kremenek 60d6f53dc4951aace69014619761760addac9e59ecTed Kremenek // Advance to the next token. 61d6f53dc4951aace69014619761760addac9e59ecTed Kremenek ++CurToken; 62d6f53dc4951aace69014619761760addac9e59ecTed Kremenek 63d6f53dc4951aace69014619761760addac9e59ecTed Kremenek if (Tok.is(tok::hash)) { 64d6f53dc4951aace69014619761760addac9e59ecTed Kremenek if (Tok.isAtStartOfLine() && !LexingRawMode) { 65d6f53dc4951aace69014619761760addac9e59ecTed Kremenek PP->HandleDirective(Tok); 66274b20863a728cc6a31ee75c670e3733600c1531Ted Kremenek 67d6f53dc4951aace69014619761760addac9e59ecTed Kremenek if (PP->isCurrentLexer(this)) 68d6f53dc4951aace69014619761760addac9e59ecTed Kremenek goto LexNextToken; 69d6f53dc4951aace69014619761760addac9e59ecTed Kremenek 70d6f53dc4951aace69014619761760addac9e59ecTed Kremenek return PP->Lex(Tok); 71d6f53dc4951aace69014619761760addac9e59ecTed Kremenek } 72274b20863a728cc6a31ee75c670e3733600c1531Ted Kremenek } 73d6f53dc4951aace69014619761760addac9e59ecTed Kremenek 74274b20863a728cc6a31ee75c670e3733600c1531Ted Kremenek MIOpt.ReadToken(); 75d6f53dc4951aace69014619761760addac9e59ecTed Kremenek 76d6f53dc4951aace69014619761760addac9e59ecTed Kremenek if (Tok.is(tok::identifier)) { 77d6f53dc4951aace69014619761760addac9e59ecTed Kremenek if (LexingRawMode) return; 78d6f53dc4951aace69014619761760addac9e59ecTed Kremenek return PP->HandleIdentifier(Tok); 79d6f53dc4951aace69014619761760addac9e59ecTed Kremenek } 80274b20863a728cc6a31ee75c670e3733600c1531Ted Kremenek} 81274b20863a728cc6a31ee75c670e3733600c1531Ted Kremenek 82274b20863a728cc6a31ee75c670e3733600c1531Ted Kremenekvoid PTHLexer::setEOF(Token& Tok) { 83452e37837a48b2f0ced144784277fd4d28cbede9Ted Kremenek Tok = Tokens[LastToken]; 84274b20863a728cc6a31ee75c670e3733600c1531Ted Kremenek} 8517ff58a63197b398ae52697b088dc0fb8b255519Ted Kremenek 8617ff58a63197b398ae52697b088dc0fb8b255519Ted Kremenekvoid PTHLexer::DiscardToEndOfLine() { 8717ff58a63197b398ae52697b088dc0fb8b255519Ted Kremenek assert(ParsingPreprocessorDirective && ParsingFilename == false && 8817ff58a63197b398ae52697b088dc0fb8b255519Ted Kremenek "Must be in a preprocessing directive!"); 894d35da2e41941965bbee8ed7e8c30e7c21000d71Ted Kremenek 904d35da2e41941965bbee8ed7e8c30e7c21000d71Ted Kremenek // Already at end-of-file? 91452e37837a48b2f0ced144784277fd4d28cbede9Ted Kremenek if (CurToken == LastToken) 924d35da2e41941965bbee8ed7e8c30e7c21000d71Ted Kremenek return; 934d35da2e41941965bbee8ed7e8c30e7c21000d71Ted Kremenek 944d35da2e41941965bbee8ed7e8c30e7c21000d71Ted Kremenek // Find the first token that is not the start of the *current* line. 95452e37837a48b2f0ced144784277fd4d28cbede9Ted Kremenek for ( ++CurToken; CurToken != LastToken ; ++CurToken ) 964d35da2e41941965bbee8ed7e8c30e7c21000d71Ted Kremenek if (Tokens[CurToken].isAtStartOfLine()) 974d35da2e41941965bbee8ed7e8c30e7c21000d71Ted Kremenek return; 9817ff58a63197b398ae52697b088dc0fb8b255519Ted Kremenek} 992f1c0243f7fc5d12446efcb006c071854c345923Ted Kremenek 1004d35da2e41941965bbee8ed7e8c30e7c21000d71Ted Kremenekunsigned PTHLexer::isNextPPTokenLParen() { 101452e37837a48b2f0ced144784277fd4d28cbede9Ted Kremenek if (CurToken == LastToken) 1024d35da2e41941965bbee8ed7e8c30e7c21000d71Ted Kremenek return 2; 1034d35da2e41941965bbee8ed7e8c30e7c21000d71Ted Kremenek 1044d35da2e41941965bbee8ed7e8c30e7c21000d71Ted Kremenek return Tokens[CurToken].is(tok::l_paren); 1052f1c0243f7fc5d12446efcb006c071854c345923Ted Kremenek} 1062f1c0243f7fc5d12446efcb006c071854c345923Ted Kremenek 107