1a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles)# Copyright 2014 The Chromium Authors. All rights reserved. 2a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles)# Use of this source code is governed by a BSD-style license that can be 3a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles)# found in the LICENSE file. 45d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 55c02ac1a9c1b504631c0a3d2b6e737b5d738bae1Bo Liuimport imp 60529e5d033099cbfc42635f6f6183833b09dff6eBen Murdochimport os.path 75d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles)import sys 85d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 95c02ac1a9c1b504631c0a3d2b6e737b5d738bae1Bo Liudef _GetDirAbove(dirname): 105c02ac1a9c1b504631c0a3d2b6e737b5d738bae1Bo Liu """Returns the directory "above" this file containing |dirname| (which must 115c02ac1a9c1b504631c0a3d2b6e737b5d738bae1Bo Liu also be "above" this file).""" 120529e5d033099cbfc42635f6f6183833b09dff6eBen Murdoch path = os.path.abspath(__file__) 130529e5d033099cbfc42635f6f6183833b09dff6eBen Murdoch while True: 140529e5d033099cbfc42635f6f6183833b09dff6eBen Murdoch path, tail = os.path.split(path) 150529e5d033099cbfc42635f6f6183833b09dff6eBen Murdoch assert tail 165c02ac1a9c1b504631c0a3d2b6e737b5d738bae1Bo Liu if tail == dirname: 175c02ac1a9c1b504631c0a3d2b6e737b5d738bae1Bo Liu return path 185d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 195c02ac1a9c1b504631c0a3d2b6e737b5d738bae1Bo Liutry: 205c02ac1a9c1b504631c0a3d2b6e737b5d738bae1Bo Liu imp.find_module("ply") 215c02ac1a9c1b504631c0a3d2b6e737b5d738bae1Bo Liuexcept ImportError: 225c02ac1a9c1b504631c0a3d2b6e737b5d738bae1Bo Liu sys.path.append(os.path.join(_GetDirAbove("mojo"), "third_party")) 235c02ac1a9c1b504631c0a3d2b6e737b5d738bae1Bo Liufrom ply.lex import TOKEN 245c02ac1a9c1b504631c0a3d2b6e737b5d738bae1Bo Liu 255c02ac1a9c1b504631c0a3d2b6e737b5d738bae1Bo Liufrom ..error import Error 265d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 27effb81e5f8246d0db0270817048dc992db66e9fbBen Murdoch 285c02ac1a9c1b504631c0a3d2b6e737b5d738bae1Bo Liuclass LexError(Error): 295c02ac1a9c1b504631c0a3d2b6e737b5d738bae1Bo Liu """Class for errors from the lexer.""" 30effb81e5f8246d0db0270817048dc992db66e9fbBen Murdoch 315c02ac1a9c1b504631c0a3d2b6e737b5d738bae1Bo Liu def __init__(self, filename, message, lineno): 325c02ac1a9c1b504631c0a3d2b6e737b5d738bae1Bo Liu Error.__init__(self, filename, message, lineno=lineno) 33effb81e5f8246d0db0270817048dc992db66e9fbBen Murdoch 34effb81e5f8246d0db0270817048dc992db66e9fbBen Murdoch 355c02ac1a9c1b504631c0a3d2b6e737b5d738bae1Bo Liu# We have methods which look like they could be functions: 365c02ac1a9c1b504631c0a3d2b6e737b5d738bae1Bo Liu# pylint: disable=R0201 375d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles)class Lexer(object): 38effb81e5f8246d0db0270817048dc992db66e9fbBen Murdoch 39effb81e5f8246d0db0270817048dc992db66e9fbBen Murdoch def __init__(self, filename): 40effb81e5f8246d0db0270817048dc992db66e9fbBen Murdoch self.filename = filename 41effb81e5f8246d0db0270817048dc992db66e9fbBen Murdoch 425d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) ######################-- PRIVATE --###################### 435d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 445d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) ## 455d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) ## Internal auxiliary methods 465d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) ## 475d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) def _error(self, msg, token): 485c02ac1a9c1b504631c0a3d2b6e737b5d738bae1Bo Liu raise LexError(self.filename, msg, token.lineno) 495d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 505d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) ## 515d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) ## Reserved keywords 525d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) ## 535d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) keywords = ( 545d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 'HANDLE', 555d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 565d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 'IMPORT', 575d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 'MODULE', 585d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 'STRUCT', 595d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 'INTERFACE', 605d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 'ENUM', 61cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) 'CONST', 6246d4c2bc3267f3f028f39e7e311b0f89aba2e4fdTorne (Richard Coles) 'TRUE', 6346d4c2bc3267f3f028f39e7e311b0f89aba2e4fdTorne (Richard Coles) 'FALSE', 6446d4c2bc3267f3f028f39e7e311b0f89aba2e4fdTorne (Richard Coles) 'DEFAULT', 655d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) ) 665d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 675d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) keyword_map = {} 685d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) for keyword in keywords: 695d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) keyword_map[keyword.lower()] = keyword 705d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 715d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) ## 725d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) ## All the tokens recognized by the lexer 735d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) ## 745d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) tokens = keywords + ( 755d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) # Identifiers 765d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 'NAME', 775d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 78effb81e5f8246d0db0270817048dc992db66e9fbBen Murdoch # Constants 795d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 'ORDINAL', 80cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) 'INT_CONST_DEC', 'INT_CONST_HEX', 81c5cede9ae108bb15f6b7a8aea21c7e1fefa2834cBen Murdoch 'FLOAT_CONST', 825d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 835d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) # String literals 845d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 'STRING_LITERAL', 855d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 865d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) # Operators 8746d4c2bc3267f3f028f39e7e311b0f89aba2e4fdTorne (Richard Coles) 'MINUS', 8846d4c2bc3267f3f028f39e7e311b0f89aba2e4fdTorne (Richard Coles) 'PLUS', 89cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) 'AMP', 905f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 'QSTN', 915d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 925d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) # Assignment 935d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 'EQUALS', 945d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 95a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) # Request / response 96a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) 'RESPONSE', 97a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) 98effb81e5f8246d0db0270817048dc992db66e9fbBen Murdoch # Delimiters 995d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 'LPAREN', 'RPAREN', # ( ) 1005d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 'LBRACKET', 'RBRACKET', # [ ] 1015d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 'LBRACE', 'RBRACE', # { } 102effb81e5f8246d0db0270817048dc992db66e9fbBen Murdoch 'LANGLE', 'RANGLE', # < > 103effb81e5f8246d0db0270817048dc992db66e9fbBen Murdoch 'SEMI', # ; 1045d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 'COMMA', 'DOT' # , . 1055d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) ) 1065d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 1075d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) ## 1085d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) ## Regexes for use in tokens 1095d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) ## 1105d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 111cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) # valid C identifiers (K&R2: A.2.3) 112cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) identifier = r'[a-zA-Z_][0-9a-zA-Z_]*' 1135d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 1145d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) hex_prefix = '0[xX]' 1155d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) hex_digits = '[0-9a-fA-F]+' 1165d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 1175d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) # integer constants (K&R2: A.2.5.1) 118cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) decimal_constant = '0|([1-9][0-9]*)' 119cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) hex_constant = hex_prefix+hex_digits 120cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) # Don't allow octal constants (even invalid octal). 121cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) octal_constant_disallowed = '0[0-9]+' 1225d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 1235d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) # character constants (K&R2: A.2.5.2) 1245d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) # Note: a-zA-Z and '.-~^_!=&;,' are allowed as escape chars to support #line 1255d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) # directives with Windows paths as filenames (..\..\dir\file) 1265d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) # For the same reason, decimal_escape allows all digit sequences. We want to 1275d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) # parse all correct code, even if it means to sometimes parse incorrect 1285d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) # code. 1295d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) # 1305d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) simple_escape = r"""([a-zA-Z._~!=&\^\-\\?'"])""" 1315d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) decimal_escape = r"""(\d+)""" 1325d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) hex_escape = r"""(x[0-9a-fA-F]+)""" 1335d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) bad_escape = r"""([\\][^a-zA-Z._~^!=&\^\-\\?'"x0-7])""" 1345d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 1355d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) escape_sequence = \ 1365d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) r"""(\\("""+simple_escape+'|'+decimal_escape+'|'+hex_escape+'))' 1375d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 1385d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) # string literals (K&R2: A.2.6) 1395d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) string_char = r"""([^"\\\n]|"""+escape_sequence+')' 1405d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) string_literal = '"'+string_char+'*"' 1415d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) bad_string_literal = '"'+string_char+'*'+bad_escape+string_char+'*"' 1425d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 1435d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) # floating constants (K&R2: A.2.5.3) 1445d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) exponent_part = r"""([eE][-+]?[0-9]+)""" 1455d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) fractional_constant = r"""([0-9]*\.[0-9]+)|([0-9]+\.)""" 1465d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) floating_constant = \ 1475d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) '(((('+fractional_constant+')'+ \ 148c5cede9ae108bb15f6b7a8aea21c7e1fefa2834cBen Murdoch exponent_part+'?)|([0-9]+'+exponent_part+')))' 1495d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 150effb81e5f8246d0db0270817048dc992db66e9fbBen Murdoch # Ordinals 151effb81e5f8246d0db0270817048dc992db66e9fbBen Murdoch ordinal = r'@[0-9]+' 152effb81e5f8246d0db0270817048dc992db66e9fbBen Murdoch missing_ordinal_value = r'@' 153effb81e5f8246d0db0270817048dc992db66e9fbBen Murdoch # Don't allow ordinal values in octal (even invalid octal, like 09) or 154effb81e5f8246d0db0270817048dc992db66e9fbBen Murdoch # hexadecimal. 155effb81e5f8246d0db0270817048dc992db66e9fbBen Murdoch octal_or_hex_ordinal_disallowed = r'@((0[0-9]+)|('+hex_prefix+hex_digits+'))' 156effb81e5f8246d0db0270817048dc992db66e9fbBen Murdoch 1575d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) ## 1585d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) ## Rules for the normal state 1595d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) ## 160effb81e5f8246d0db0270817048dc992db66e9fbBen Murdoch t_ignore = ' \t\r' 1615d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 1625d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) # Newlines 1635d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) def t_NEWLINE(self, t): 1645d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) r'\n+' 165c5cede9ae108bb15f6b7a8aea21c7e1fefa2834cBen Murdoch t.lexer.lineno += len(t.value) 1665d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 1675d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) # Operators 1685d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) t_MINUS = r'-' 16946d4c2bc3267f3f028f39e7e311b0f89aba2e4fdTorne (Richard Coles) t_PLUS = r'\+' 170cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) t_AMP = r'&' 1715f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) t_QSTN = r'\?' 1725d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 1735d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) # = 1745d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) t_EQUALS = r'=' 1755d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 176a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) # => 177a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) t_RESPONSE = r'=>' 178a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) 179effb81e5f8246d0db0270817048dc992db66e9fbBen Murdoch # Delimiters 1805d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) t_LPAREN = r'\(' 1815d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) t_RPAREN = r'\)' 1825d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) t_LBRACKET = r'\[' 1835d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) t_RBRACKET = r'\]' 1845d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) t_LBRACE = r'\{' 1855d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) t_RBRACE = r'\}' 186effb81e5f8246d0db0270817048dc992db66e9fbBen Murdoch t_LANGLE = r'<' 187effb81e5f8246d0db0270817048dc992db66e9fbBen Murdoch t_RANGLE = r'>' 1885d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) t_COMMA = r',' 189effb81e5f8246d0db0270817048dc992db66e9fbBen Murdoch t_DOT = r'\.' 1905d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) t_SEMI = r';' 1915d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 1925d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) t_STRING_LITERAL = string_literal 1935d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 1945d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) # The following floating and integer constants are defined as 1955d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) # functions to impose a strict order (otherwise, decimal 1965d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) # is placed before the others because its regex is longer, 1975d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) # and this is bad) 1985d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) # 1995d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) @TOKEN(floating_constant) 2005d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) def t_FLOAT_CONST(self, t): 2015d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) return t 2025d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 2035d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) @TOKEN(hex_constant) 2045d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) def t_INT_CONST_HEX(self, t): 2055d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) return t 2065d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 207cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) @TOKEN(octal_constant_disallowed) 208cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) def t_OCTAL_CONSTANT_DISALLOWED(self, t): 209cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) msg = "Octal values not allowed" 2105d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) self._error(msg, t) 2115d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 2125d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) @TOKEN(decimal_constant) 2135d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) def t_INT_CONST_DEC(self, t): 2145d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) return t 2155d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 2165d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) # unmatched string literals are caught by the preprocessor 2175d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 2185d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) @TOKEN(bad_string_literal) 2195d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) def t_BAD_STRING_LITERAL(self, t): 2205d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) msg = "String contains invalid escape code" 2215d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) self._error(msg, t) 2225d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 223effb81e5f8246d0db0270817048dc992db66e9fbBen Murdoch # Handle ordinal-related tokens in the right order: 224effb81e5f8246d0db0270817048dc992db66e9fbBen Murdoch @TOKEN(octal_or_hex_ordinal_disallowed) 225effb81e5f8246d0db0270817048dc992db66e9fbBen Murdoch def t_OCTAL_OR_HEX_ORDINAL_DISALLOWED(self, t): 226effb81e5f8246d0db0270817048dc992db66e9fbBen Murdoch msg = "Octal and hexadecimal ordinal values not allowed" 227effb81e5f8246d0db0270817048dc992db66e9fbBen Murdoch self._error(msg, t) 228effb81e5f8246d0db0270817048dc992db66e9fbBen Murdoch 229effb81e5f8246d0db0270817048dc992db66e9fbBen Murdoch @TOKEN(ordinal) 230effb81e5f8246d0db0270817048dc992db66e9fbBen Murdoch def t_ORDINAL(self, t): 231effb81e5f8246d0db0270817048dc992db66e9fbBen Murdoch return t 232effb81e5f8246d0db0270817048dc992db66e9fbBen Murdoch 233effb81e5f8246d0db0270817048dc992db66e9fbBen Murdoch @TOKEN(missing_ordinal_value) 234effb81e5f8246d0db0270817048dc992db66e9fbBen Murdoch def t_BAD_ORDINAL(self, t): 235effb81e5f8246d0db0270817048dc992db66e9fbBen Murdoch msg = "Missing ordinal value" 236effb81e5f8246d0db0270817048dc992db66e9fbBen Murdoch self._error(msg, t) 237effb81e5f8246d0db0270817048dc992db66e9fbBen Murdoch 2385d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) @TOKEN(identifier) 2395d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) def t_NAME(self, t): 2405d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) t.type = self.keyword_map.get(t.value, "NAME") 2415d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) return t 2425d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 2435d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) # Ignore C and C++ style comments 2445d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) def t_COMMENT(self, t): 2455d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) r'(/\*(.|\n)*?\*/)|(//.*(\n[ \t]*//.*)*)' 246cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) t.lexer.lineno += t.value.count("\n") 2475d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 2485d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) def t_error(self, t): 249cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) msg = "Illegal character %s" % repr(t.value[0]) 2505d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) self._error(msg, t) 251