1/* 2// 3// Copyright (c) 2002-2014 The ANGLE Project Authors. All rights reserved. 4// Use of this source code is governed by a BSD-style license that can be 5// found in the LICENSE file. 6// 7 8This file contains the Lex specification for GLSL ES preprocessor. 9Based on Microsoft Visual Studio 2010 Preprocessor Grammar: 10http://msdn.microsoft.com/en-us/library/2scxys89.aspx 11 12IF YOU MODIFY THIS FILE YOU ALSO NEED TO RUN generate_parser.sh. 13*/ 14 15%top{ 16// 17// Copyright (c) 2011-2014 The ANGLE Project Authors. All rights reserved. 18// Use of this source code is governed by a BSD-style license that can be 19// found in the LICENSE file. 20// 21 22// This file is auto-generated by generate_parser.sh. DO NOT EDIT! 23} 24 25%{ 26#include "Tokenizer.h" 27 28#include "DiagnosticsBase.h" 29#include "Token.h" 30 31#if defined(__GNUC__) 32// Triggered by the auto-generated yy_fatal_error function. 33#pragma GCC diagnostic ignored "-Wmissing-noreturn" 34#endif 35 36typedef std::string YYSTYPE; 37typedef pp::SourceLocation YYLTYPE; 38 39// Use the unused yycolumn variable to track file (string) number. 40#define yyfileno yycolumn 41 42#define YY_USER_INIT \ 43 do { \ 44 yyfileno = 0; \ 45 yylineno = 1; \ 46 yyextra->leadingSpace = false; \ 47 yyextra->lineStart = true; \ 48 } while(0); 49 50#define YY_USER_ACTION \ 51 do \ 52 { \ 53 pp::Input* input = &yyextra->input; \ 54 pp::Input::Location* scanLoc = &yyextra->scanLoc; \ 55 while ((scanLoc->sIndex < input->count()) && \ 56 (scanLoc->cIndex >= input->length(scanLoc->sIndex))) \ 57 { \ 58 scanLoc->cIndex -= input->length(scanLoc->sIndex++); \ 59 ++yyfileno; yylineno = 1; \ 60 } \ 61 yylloc->file = yyfileno; \ 62 yylloc->line = yylineno; \ 63 scanLoc->cIndex += yyleng; \ 64 } while(0); 65 66#define YY_INPUT(buf, result, maxSize) \ 67 result = yyextra->input.read(buf, maxSize); 68 69%} 70 71%option noyywrap nounput never-interactive 72%option reentrant bison-bridge bison-locations 73%option prefix="pp" 74%option extra-type="pp::Tokenizer::Context*" 75%x COMMENT 76 77NEWLINE \n|\r|\r\n 78IDENTIFIER [_a-zA-Z][_a-zA-Z0-9]* 79PUNCTUATOR [][<>(){}.+-/*%^|&~=!:;,?] 80 81DECIMAL_CONSTANT [1-9][0-9]*[uU]? 82OCTAL_CONSTANT 0[0-7]*[uU]? 83HEXADECIMAL_CONSTANT 0[xX][0-9a-fA-F]+[uU]? 84 85DIGIT [0-9] 86EXPONENT_PART [eE][+-]?{DIGIT}+ 87FRACTIONAL_CONSTANT ({DIGIT}*"."{DIGIT}+)|({DIGIT}+".") 88 89%% 90 91 /* Line comment */ 92"//"[^\r\n]* 93 94 /* Block comment */ 95 /* Line breaks are just counted - not returned. */ 96 /* The comment is replaced by a single space. */ 97"/*" { BEGIN(COMMENT); } 98<COMMENT>[^*\r\n]+ 99<COMMENT>"*" 100<COMMENT>{NEWLINE} { ++yylineno; } 101<COMMENT>"*/" { 102 yyextra->leadingSpace = true; 103 BEGIN(INITIAL); 104} 105 106# { 107 // # is only valid at start of line for preprocessor directives. 108 yylval->assign(1, yytext[0]); 109 return yyextra->lineStart ? pp::Token::PP_HASH : pp::Token::PP_OTHER; 110} 111 112{IDENTIFIER} { 113 yylval->assign(yytext, yyleng); 114 return pp::Token::IDENTIFIER; 115} 116 117({DECIMAL_CONSTANT}[uU]?)|({OCTAL_CONSTANT}[uU]?)|({HEXADECIMAL_CONSTANT}[uU]?) { 118 yylval->assign(yytext, yyleng); 119 return pp::Token::CONST_INT; 120} 121 122({DIGIT}+{EXPONENT_PART}[fF]?)|({FRACTIONAL_CONSTANT}{EXPONENT_PART}?[fF]?) { 123 yylval->assign(yytext, yyleng); 124 return pp::Token::CONST_FLOAT; 125} 126 127 /* Anything that starts with a {DIGIT} or .{DIGIT} must be a number. */ 128 /* Rule to catch all invalid integers and floats. */ 129({DIGIT}+[_a-zA-Z0-9.]*)|("."{DIGIT}+[_a-zA-Z0-9.]*) { 130 yylval->assign(yytext, yyleng); 131 return pp::Token::PP_NUMBER; 132} 133 134"++" { 135 yylval->assign(yytext, yyleng); 136 return pp::Token::OP_INC; 137} 138"--" { 139 yylval->assign(yytext, yyleng); 140 return pp::Token::OP_DEC; 141} 142"<<" { 143 yylval->assign(yytext, yyleng); 144 return pp::Token::OP_LEFT; 145} 146">>" { 147 yylval->assign(yytext, yyleng); 148 return pp::Token::OP_RIGHT; 149} 150"<=" { 151 yylval->assign(yytext, yyleng); 152 return pp::Token::OP_LE; 153} 154">=" { 155 yylval->assign(yytext, yyleng); 156 return pp::Token::OP_GE; 157} 158"==" { 159 yylval->assign(yytext, yyleng); 160 return pp::Token::OP_EQ; 161} 162"!=" { 163 yylval->assign(yytext, yyleng); 164 return pp::Token::OP_NE; 165} 166"&&" { 167 yylval->assign(yytext, yyleng); 168 return pp::Token::OP_AND; 169} 170"^^" { 171 yylval->assign(yytext, yyleng); 172 return pp::Token::OP_XOR; 173} 174"||" { 175 yylval->assign(yytext, yyleng); 176 return pp::Token::OP_OR; 177} 178"+=" { 179 yylval->assign(yytext, yyleng); 180 return pp::Token::OP_ADD_ASSIGN; 181} 182"-=" { 183 yylval->assign(yytext, yyleng); 184 return pp::Token::OP_SUB_ASSIGN; 185} 186"*=" { 187 yylval->assign(yytext, yyleng); 188 return pp::Token::OP_MUL_ASSIGN; 189} 190"/=" { 191 yylval->assign(yytext, yyleng); 192 return pp::Token::OP_DIV_ASSIGN; 193} 194"%=" { 195 yylval->assign(yytext, yyleng); 196 return pp::Token::OP_MOD_ASSIGN; 197} 198"<<=" { 199 yylval->assign(yytext, yyleng); 200 return pp::Token::OP_LEFT_ASSIGN; 201} 202">>=" { 203 yylval->assign(yytext, yyleng); 204 return pp::Token::OP_RIGHT_ASSIGN; 205} 206"&=" { 207 yylval->assign(yytext, yyleng); 208 return pp::Token::OP_AND_ASSIGN; 209} 210"^=" { 211 yylval->assign(yytext, yyleng); 212 return pp::Token::OP_XOR_ASSIGN; 213} 214"|=" { 215 yylval->assign(yytext, yyleng); 216 return pp::Token::OP_OR_ASSIGN; 217} 218 219{PUNCTUATOR} { 220 yylval->assign(1, yytext[0]); 221 return yytext[0]; 222} 223 224[ \t\v\f]+ { yyextra->leadingSpace = true; } 225 226{NEWLINE} { 227 ++yylineno; 228 yylval->assign(1, '\n'); 229 return '\n'; 230} 231 232\\{NEWLINE} { ++yylineno; } 233 234. { 235 yylval->assign(1, yytext[0]); 236 return pp::Token::PP_OTHER; 237} 238 239<*><<EOF>> { 240 // YY_USER_ACTION is not invoked for handling EOF. 241 // Set the location for EOF token manually. 242 pp::Input* input = &yyextra->input; 243 pp::Input::Location* scanLoc = &yyextra->scanLoc; 244 yy_size_t sIndexMax = input->count() ? input->count() - 1 : 0; 245 if (scanLoc->sIndex != sIndexMax) 246 { 247 // We can only reach here if there are empty strings at the 248 // end of the input. 249 scanLoc->sIndex = sIndexMax; scanLoc->cIndex = 0; 250 // FIXME: this is not 64-bit clean. 251 yyfileno = static_cast<int>(sIndexMax); yylineno = 1; 252 } 253 yylloc->file = yyfileno; 254 yylloc->line = yylineno; 255 yylval->clear(); 256 257 if (YY_START == COMMENT) 258 { 259 yyextra->diagnostics->report(pp::Diagnostics::PP_EOF_IN_COMMENT, 260 pp::SourceLocation(yyfileno, yylineno), 261 ""); 262 } 263 yyterminate(); 264} 265 266%% 267 268namespace pp { 269 270Tokenizer::Tokenizer(Diagnostics *diagnostics) : mHandle(0) 271{ 272 mContext.diagnostics = diagnostics; 273} 274 275Tokenizer::~Tokenizer() 276{ 277 destroyScanner(); 278} 279 280bool Tokenizer::init(size_t count, const char * const string[], const int length[]) 281{ 282 if ((count > 0) && (string == 0)) 283 return false; 284 285 mContext.input = Input(count, string, length); 286 return initScanner(); 287} 288 289void Tokenizer::setFileNumber(int file) 290{ 291 // We use column number as file number. 292 // See macro yyfileno. 293 yyset_column(file, mHandle); 294} 295 296void Tokenizer::setLineNumber(int line) 297{ 298 yyset_lineno(line, mHandle); 299} 300 301void Tokenizer::setMaxTokenSize(size_t maxTokenSize) 302{ 303 mMaxTokenSize = maxTokenSize; 304} 305 306void Tokenizer::lex(Token *token) 307{ 308 token->type = yylex(&token->text, &token->location, mHandle); 309 if (token->text.size() > mMaxTokenSize) 310 { 311 mContext.diagnostics->report(Diagnostics::PP_TOKEN_TOO_LONG, 312 token->location, token->text); 313 token->text.erase(mMaxTokenSize); 314 } 315 316 token->flags = 0; 317 318 token->setAtStartOfLine(mContext.lineStart); 319 mContext.lineStart = token->type == '\n'; 320 321 token->setHasLeadingSpace(mContext.leadingSpace); 322 mContext.leadingSpace = false; 323} 324 325bool Tokenizer::initScanner() 326{ 327 if ((mHandle == NULL) && yylex_init_extra(&mContext, &mHandle)) 328 return false; 329 330 yyrestart(0, mHandle); 331 return true; 332} 333 334void Tokenizer::destroyScanner() 335{ 336 if (mHandle == NULL) 337 return; 338 339 yylex_destroy(mHandle); 340 mHandle = NULL; 341} 342 343} // namespace pp 344 345