1/* 2// Copyright 2016 The SwiftShader Authors. All Rights Reserved. 3// 4// Licensed under the Apache License, Version 2.0 (the "License"); 5// you may not use this file except in compliance with the License. 6// You may obtain a copy of the License at 7// 8// http://www.apache.org/licenses/LICENSE-2.0 9// 10// Unless required by applicable law or agreed to in writing, software 11// distributed under the License is distributed on an "AS IS" BASIS, 12// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13// See the License for the specific language governing permissions and 14// limitations under the License. 15 16This file contains the Lex specification for GLSL ES preprocessor. 17Based on Microsoft Visual Studio 2010 Preprocessor Grammar: 18http://msdn.microsoft.com/en-us/library/2scxys89.aspx 19 20IF YOU MODIFY THIS FILE YOU ALSO NEED TO RUN generate_parser.sh. 21*/ 22 23%top{ 24// Copyright 2016 The SwiftShader Authors. All Rights Reserved. 25// 26// Licensed under the Apache License, Version 2.0 (the "License"); 27// you may not use this file except in compliance with the License. 28// You may obtain a copy of the License at 29// 30// http://www.apache.org/licenses/LICENSE-2.0 31// 32// Unless required by applicable law or agreed to in writing, software 33// distributed under the License is distributed on an "AS IS" BASIS, 34// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 35// See the License for the specific language governing permissions and 36// limitations under the License. 37 38// This file is auto-generated by generate_parser.sh. DO NOT EDIT! 39} 40 41%{ 42#include "Tokenizer.h" 43 44#include "Diagnostics.h" 45#include "Token.h" 46 47#if defined(__GNUC__) 48// Triggered by the auto-generated yy_fatal_error function. 49#pragma GCC diagnostic ignored "-Wmissing-noreturn" 50#endif 51 52typedef std::string YYSTYPE; 53typedef pp::SourceLocation YYLTYPE; 54 55// Use the unused yycolumn variable to track file (string) number. 56#define yyfileno yycolumn 57 58#define YY_USER_INIT \ 59 do { \ 60 yyfileno = 0; \ 61 yylineno = 1; \ 62 yyextra->leadingSpace = false; \ 63 yyextra->lineStart = true; \ 64 } while(0); 65 66#define YY_USER_ACTION \ 67 do \ 68 { \ 69 pp::Input* input = &yyextra->input; \ 70 pp::Input::Location* scanLoc = &yyextra->scanLoc; \ 71 while ((scanLoc->sIndex < input->count()) && \ 72 (scanLoc->cIndex >= input->length(scanLoc->sIndex))) \ 73 { \ 74 scanLoc->cIndex -= input->length(scanLoc->sIndex++); \ 75 ++yyfileno; yylineno = 1; \ 76 } \ 77 yylloc->file = yyfileno; \ 78 yylloc->line = yylineno; \ 79 scanLoc->cIndex += yyleng; \ 80 } while(0); 81 82#define YY_INPUT(buf, result, maxSize) \ 83 result = yyextra->input.read(buf, maxSize); 84 85%} 86 87%option noyywrap nounput never-interactive 88%option reentrant bison-bridge bison-locations 89%option prefix="pp" 90%option extra-type="pp::Tokenizer::Context*" 91%x COMMENT 92 93NEWLINE \n|\r|\r\n 94IDENTIFIER [_a-zA-Z][_a-zA-Z0-9]* 95PUNCTUATOR [][<>(){}.+-/*%^|&~=!:;,?] 96 97DECIMAL_CONSTANT [1-9][0-9]*[uU]? 98OCTAL_CONSTANT 0[0-7]*[uU]? 99HEXADECIMAL_CONSTANT 0[xX][0-9a-fA-F]+[uU]? 100 101DIGIT [0-9] 102EXPONENT_PART [eE][+-]?{DIGIT}+ 103FRACTIONAL_CONSTANT ({DIGIT}*"."{DIGIT}+)|({DIGIT}+".") 104 105%% 106 107 /* Line comment */ 108"//"[^\r\n]* 109 110 /* Block comment */ 111 /* Line breaks are just counted - not returned. */ 112 /* The comment is replaced by a single space. */ 113"/*" { BEGIN(COMMENT); } 114<COMMENT>[^*\r\n]+ 115<COMMENT>"*" 116<COMMENT>{NEWLINE} { ++yylineno; } 117<COMMENT>"*/" { 118 yyextra->leadingSpace = true; 119 BEGIN(INITIAL); 120} 121 122# { 123 // # is only valid at start of line for preprocessor directives. 124 yylval->assign(1, yytext[0]); 125 return yyextra->lineStart ? pp::Token::PP_HASH : pp::Token::PP_OTHER; 126} 127 128{IDENTIFIER} { 129 yylval->assign(yytext, yyleng); 130 return pp::Token::IDENTIFIER; 131} 132 133({DECIMAL_CONSTANT}[uU]?)|({OCTAL_CONSTANT}[uU]?)|({HEXADECIMAL_CONSTANT}[uU]?) { 134 yylval->assign(yytext, yyleng); 135 return pp::Token::CONST_INT; 136} 137 138({DIGIT}+{EXPONENT_PART}[fF]?)|({FRACTIONAL_CONSTANT}{EXPONENT_PART}?[fF]?) { 139 yylval->assign(yytext, yyleng); 140 return pp::Token::CONST_FLOAT; 141} 142 143 /* Anything that starts with a {DIGIT} or .{DIGIT} must be a number. */ 144 /* Rule to catch all invalid integers and floats. */ 145({DIGIT}+[_a-zA-Z0-9.]*)|("."{DIGIT}+[_a-zA-Z0-9.]*) { 146 yylval->assign(yytext, yyleng); 147 return pp::Token::PP_NUMBER; 148} 149 150"++" { 151 yylval->assign(yytext, yyleng); 152 return pp::Token::OP_INC; 153} 154"--" { 155 yylval->assign(yytext, yyleng); 156 return pp::Token::OP_DEC; 157} 158"<<" { 159 yylval->assign(yytext, yyleng); 160 return pp::Token::OP_LEFT; 161} 162">>" { 163 yylval->assign(yytext, yyleng); 164 return pp::Token::OP_RIGHT; 165} 166"<=" { 167 yylval->assign(yytext, yyleng); 168 return pp::Token::OP_LE; 169} 170">=" { 171 yylval->assign(yytext, yyleng); 172 return pp::Token::OP_GE; 173} 174"==" { 175 yylval->assign(yytext, yyleng); 176 return pp::Token::OP_EQ; 177} 178"!=" { 179 yylval->assign(yytext, yyleng); 180 return pp::Token::OP_NE; 181} 182"&&" { 183 yylval->assign(yytext, yyleng); 184 return pp::Token::OP_AND; 185} 186"^^" { 187 yylval->assign(yytext, yyleng); 188 return pp::Token::OP_XOR; 189} 190"||" { 191 yylval->assign(yytext, yyleng); 192 return pp::Token::OP_OR; 193} 194"+=" { 195 yylval->assign(yytext, yyleng); 196 return pp::Token::OP_ADD_ASSIGN; 197} 198"-=" { 199 yylval->assign(yytext, yyleng); 200 return pp::Token::OP_SUB_ASSIGN; 201} 202"*=" { 203 yylval->assign(yytext, yyleng); 204 return pp::Token::OP_MUL_ASSIGN; 205} 206"/=" { 207 yylval->assign(yytext, yyleng); 208 return pp::Token::OP_DIV_ASSIGN; 209} 210"%=" { 211 yylval->assign(yytext, yyleng); 212 return pp::Token::OP_MOD_ASSIGN; 213} 214"<<=" { 215 yylval->assign(yytext, yyleng); 216 return pp::Token::OP_LEFT_ASSIGN; 217} 218">>=" { 219 yylval->assign(yytext, yyleng); 220 return pp::Token::OP_RIGHT_ASSIGN; 221} 222"&=" { 223 yylval->assign(yytext, yyleng); 224 return pp::Token::OP_AND_ASSIGN; 225} 226"^=" { 227 yylval->assign(yytext, yyleng); 228 return pp::Token::OP_XOR_ASSIGN; 229} 230"|=" { 231 yylval->assign(yytext, yyleng); 232 return pp::Token::OP_OR_ASSIGN; 233} 234 235{PUNCTUATOR} { 236 yylval->assign(1, yytext[0]); 237 return yytext[0]; 238} 239 240[ \t\v\f]+ { yyextra->leadingSpace = true; } 241 242{NEWLINE} { 243 ++yylineno; 244 yylval->assign(1, '\n'); 245 return '\n'; 246} 247 248\\{NEWLINE} { ++yylineno; } 249 250. { 251 yylval->assign(1, yytext[0]); 252 return pp::Token::PP_OTHER; 253} 254 255<*><<EOF>> { 256 // YY_USER_ACTION is not invoked for handling EOF. 257 // Set the location for EOF token manually. 258 pp::Input* input = &yyextra->input; 259 pp::Input::Location* scanLoc = &yyextra->scanLoc; 260 int sIndexMax = std::max(0, input->count() - 1); 261 if (scanLoc->sIndex != sIndexMax) 262 { 263 // We can only reach here if there are empty strings at the 264 // end of the input. 265 scanLoc->sIndex = sIndexMax; scanLoc->cIndex = 0; 266 yyfileno = sIndexMax; yylineno = 1; 267 } 268 yylloc->file = yyfileno; 269 yylloc->line = yylineno; 270 yylval->clear(); 271 272 if (YY_START == COMMENT) 273 { 274 yyextra->diagnostics->report(pp::Diagnostics::EOF_IN_COMMENT, 275 pp::SourceLocation(yyfileno, yylineno), 276 ""); 277 } 278 yyterminate(); 279} 280 281%% 282 283namespace pp { 284 285// TODO(alokp): Maximum token length should ideally be specified by 286// the preprocessor client, i.e., the compiler. 287const size_t Tokenizer::kMaxTokenLength = 256; 288 289Tokenizer::Tokenizer(Diagnostics* diagnostics) : mHandle(0) 290{ 291 mContext.diagnostics = diagnostics; 292} 293 294Tokenizer::~Tokenizer() 295{ 296 destroyScanner(); 297} 298 299bool Tokenizer::init(int count, const char* const string[], const int length[]) 300{ 301 if (count < 0) return false; 302 if ((count > 0) && (string == 0)) return false; 303 304 mContext.input = Input(count, string, length); 305 return initScanner(); 306} 307 308void Tokenizer::setFileNumber(int file) 309{ 310 // We use column number as file number. 311 // See macro yyfileno. 312 yyset_column(file, mHandle); 313} 314 315void Tokenizer::setLineNumber(int line) 316{ 317 yyset_lineno(line, mHandle); 318} 319 320void Tokenizer::lex(Token* token) 321{ 322 token->type = yylex(&token->text, &token->location, mHandle); 323 if (token->text.size() > kMaxTokenLength) 324 { 325 mContext.diagnostics->report(Diagnostics::TOKEN_TOO_LONG, 326 token->location, token->text); 327 token->text.erase(kMaxTokenLength); 328 } 329 330 token->flags = 0; 331 332 token->setAtStartOfLine(mContext.lineStart); 333 mContext.lineStart = token->type == '\n'; 334 335 token->setHasLeadingSpace(mContext.leadingSpace); 336 mContext.leadingSpace = false; 337} 338 339bool Tokenizer::initScanner() 340{ 341 if ((mHandle == NULL) && yylex_init_extra(&mContext, &mHandle)) 342 return false; 343 344 yyrestart(0, mHandle); 345 return true; 346} 347 348void Tokenizer::destroyScanner() 349{ 350 if (mHandle == NULL) 351 return; 352 353 yylex_destroy(mHandle); 354 mHandle = NULL; 355} 356 357} // namespace pp 358 359