1// Copyright 2016 The SwiftShader Authors. All Rights Reserved. 2// 3// Licensed under the Apache License, Version 2.0 (the "License"); 4// you may not use this file except in compliance with the License. 5// You may obtain a copy of the License at 6// 7// http://www.apache.org/licenses/LICENSE-2.0 8// 9// Unless required by applicable law or agreed to in writing, software 10// distributed under the License is distributed on an "AS IS" BASIS, 11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12// See the License for the specific language governing permissions and 13// limitations under the License. 14 15#ifndef COMPILER_PREPROCESSOR_TOKENIZER_H_ 16#define COMPILER_PREPROCESSOR_TOKENIZER_H_ 17 18#include "Input.h" 19#include "Lexer.h" 20#include "pp_utils.h" 21 22namespace pp 23{ 24 25class Diagnostics; 26 27class Tokenizer : public Lexer 28{ 29public: 30 struct Context 31 { 32 Diagnostics *diagnostics; 33 34 Input input; 35 // The location where yytext points to. Token location should track 36 // scanLoc instead of Input::mReadLoc because they may not be the same 37 // if text is buffered up in the scanner input buffer. 38 Input::Location scanLoc; 39 40 bool leadingSpace; 41 bool lineStart; 42 }; 43 44 Tokenizer(Diagnostics *diagnostics); 45 ~Tokenizer() override; 46 47 bool init(size_t count, const char *const string[], const int length[]); 48 49 void setFileNumber(int file); 50 void setLineNumber(int line); 51 void setMaxTokenSize(size_t maxTokenSize); 52 53 void lex(Token *token) override; 54 55private: 56 PP_DISALLOW_COPY_AND_ASSIGN(Tokenizer); 57 bool initScanner(); 58 void destroyScanner(); 59 60 void *mHandle; // Scanner handle. 61 Context mContext; // Scanner extra. 62 size_t mMaxTokenSize; // Maximum token size 63}; 64 65} // namespace pp 66#endif // COMPILER_PREPROCESSOR_TOKENIZER_H_ 67 68