1// 2// Copyright (c) 2012 The ANGLE Project Authors. All rights reserved. 3// Use of this source code is governed by a BSD-style license that can be 4// found in the LICENSE file. 5// 6 7#ifndef COMPILER_PREPROCESSOR_TOKENIZER_H_ 8#define COMPILER_PREPROCESSOR_TOKENIZER_H_ 9 10#include "Input.h" 11#include "Lexer.h" 12#include "pp_utils.h" 13 14namespace pp 15{ 16 17class Diagnostics; 18 19class Tokenizer : public Lexer 20{ 21 public: 22 struct Context 23 { 24 Diagnostics* diagnostics; 25 26 Input input; 27 // The location where yytext points to. Token location should track 28 // scanLoc instead of Input::mReadLoc because they may not be the same 29 // if text is buffered up in the scanner input buffer. 30 Input::Location scanLoc; 31 32 bool leadingSpace; 33 bool lineStart; 34 }; 35 36 Tokenizer(Diagnostics* diagnostics); 37 ~Tokenizer(); 38 39 bool init(size_t count, const char* const string[], const int length[]); 40 41 void setMaxTokenLength(size_t maxLength) { mMaxTokenLength = maxLength; } 42 void setFileNumber(int file); 43 void setLineNumber(int line); 44 45 virtual void lex(Token* token); 46 47 private: 48 PP_DISALLOW_COPY_AND_ASSIGN(Tokenizer); 49 bool initScanner(); 50 void destroyScanner(); 51 52 void* mHandle; // Scanner handle. 53 Context mContext; // Scanner extra. 54 size_t mMaxTokenLength; 55}; 56 57} // namespace pp 58#endif // COMPILER_PREPROCESSOR_TOKENIZER_H_ 59 60