1#!/usr/bin/env python 2# Copyright (c) 2013 The Chromium Authors. All rights reserved. 3# Use of this source code is governed by a BSD-style license that can be 4# found in the LICENSE file. 5 6""" Lexer for PPAPI IDL 7 8The lexer uses the PLY library to build a tokenizer which understands both 9WebIDL and Pepper tokens. 10 11WebIDL, and WebIDL regular expressions can be found at: 12 http://www.w3.org/TR/2012/CR-WebIDL-20120419/ 13PLY can be found at: 14 http://www.dabeaz.com/ply/ 15""" 16 17import os.path 18import sys 19 20# 21# Try to load the ply module, if not, then assume it is in the third_party 22# directory. 23# 24try: 25 # Disable lint check which fails to find the ply module. 26 # pylint: disable=F0401 27 from ply import lex 28except ImportError: 29 module_path, module_name = os.path.split(__file__) 30 third_party = os.path.join(module_path, '..', '..', 'third_party') 31 sys.path.append(third_party) 32 # pylint: disable=F0401 33 from ply import lex 34 35# 36# IDL Lexer 37# 38class IDLLexer(object): 39 # 'literals' is a value expected by lex which specifies a list of valid 40 # literal tokens, meaning the token type and token value are identical. 41 literals = r'"*.(){}[],;:=+-/~|&^?<>' 42 43 # 't_ignore' contains ignored characters (spaces and tabs) 44 t_ignore = ' \t' 45 46 # 'tokens' is a value required by lex which specifies the complete list 47 # of valid token types. 48 tokens = [ 49 # Data types 50 'float', 51 'integer', 52 'string', 53 54 # Symbol and keywords types 55 'COMMENT', 56 'identifier', 57 58 # MultiChar operators 59 'ELLIPSIS', 60 ] 61 62 # 'keywords' is a map of string to token type. All tokens matching 63 # KEYWORD_OR_SYMBOL are matched against keywords dictionary, to determine 64 # if the token is actually a keyword. 65 keywords = { 66 'any' : 'ANY', 67 'attribute' : 'ATTRIBUTE', 68 'boolean' : 'BOOLEAN', 69 'byte' : 'BYTE', 70 'ByteString' : 'BYTESTRING', 71 'callback' : 'CALLBACK', 72 'const' : 'CONST', 73 'creator' : 'CREATOR', 74 'Date' : 'DATE', 75 'deleter' : 'DELETER', 76 'dictionary' : 'DICTIONARY', 77 'DOMString' : 'DOMSTRING', 78 'double' : 'DOUBLE', 79 'enum' : 'ENUM', 80 'false' : 'FALSE', 81 'float' : 'FLOAT', 82 'exception' : 'EXCEPTION', 83 'getter': 'GETTER', 84 'implements' : 'IMPLEMENTS', 85 'Infinity' : 'INFINITY', 86 'inherit' : 'INHERIT', 87 'interface' : 'INTERFACE', 88 'legacycaller' : 'LEGACYCALLER', 89 'long' : 'LONG', 90 'Nan' : 'NAN', 91 'null' : 'NULL', 92 'object' : 'OBJECT', 93 'octet' : 'OCTET', 94 'optional' : 'OPTIONAL', 95 'or' : 'OR', 96 'partial' : 'PARTIAL', 97 'readonly' : 'READONLY', 98 'RegExp' : 'REGEXP', 99 'sequence' : 'SEQUENCE', 100 'serializer' : 'SERIALIZER', 101 'setter': 'SETTER', 102 'short' : 'SHORT', 103 'static' : 'STATIC', 104 'stringifier' : 'STRINGIFIER', 105 'typedef' : 'TYPEDEF', 106 'true' : 'TRUE', 107 'unsigned' : 'UNSIGNED', 108 'unrestricted' : 'UNRESTRICTED', 109 'void' : 'VOID' 110 } 111 112 # Token definitions 113 # 114 # Lex assumes any value or function in the form of 't_<TYPE>' represents a 115 # regular expression where a match will emit a token of type <TYPE>. In the 116 # case of a function, the function is called when a match is made. These 117 # definitions come from WebIDL. 118 # 119 # These need to be methods for lexer construction, despite not using self. 120 # pylint: disable=R0201 121 def t_ELLIPSIS(self, t): 122 r'\.\.\.' 123 return t 124 125 # Regex needs to be in the docstring 126 # pylint: disable=C0301 127 def t_float(self, t): 128 r'-?(([0-9]+\.[0-9]*|[0-9]*\.[0-9]+)([Ee][+-]?[0-9]+)?|[0-9]+[Ee][+-]?[0-9]+)' 129 return t 130 131 def t_integer(self, t): 132 r'-?([1-9][0-9]*|0[Xx][0-9A-Fa-f]+|0[0-7]*)' 133 return t 134 135 136 # A line ending '\n', we use this to increment the line number 137 def t_LINE_END(self, t): 138 r'\n+' 139 self.AddLines(len(t.value)) 140 141 # We do not process escapes in the IDL strings. Strings are exclusively 142 # used for attributes and enums, and not used as typical 'C' constants. 143 def t_string(self, t): 144 r'"[^"]*"' 145 t.value = t.value[1:-1] 146 self.AddLines(t.value.count('\n')) 147 return t 148 149 # A C or C++ style comment: /* xxx */ or // 150 def t_COMMENT(self, t): 151 r'(/\*(.|\n)*?\*/)|(//.*(\n[ \t]*//.*)*)' 152 self.AddLines(t.value.count('\n')) 153 return t 154 155 # A symbol or keyword. 156 def t_KEYWORD_OR_SYMBOL(self, t): 157 r'_?[A-Za-z][A-Za-z_0-9]*' 158 159 # All non-keywords are assumed to be symbols 160 t.type = self.keywords.get(t.value, 'identifier') 161 162 # We strip leading underscores so that you can specify symbols with the same 163 # value as a keywords (E.g. a dictionary named 'interface'). 164 if t.value[0] == '_': 165 t.value = t.value[1:] 166 return t 167 168 def t_ANY_error(self, t): 169 msg = 'Unrecognized input' 170 line = self.Lexer().lineno 171 172 # If that line has not been accounted for, then we must have hit 173 # EoF, so compute the beginning of the line that caused the problem. 174 if line >= len(self.index): 175 # Find the offset in the line of the first word causing the issue 176 word = t.value.split()[0] 177 offs = self.lines[line - 1].find(word) 178 # Add the computed line's starting position 179 self.index.append(self.Lexer().lexpos - offs) 180 msg = 'Unexpected EoF reached after' 181 182 pos = self.Lexer().lexpos - self.index[line] 183 out = self.ErrorMessage(line, pos, msg) 184 sys.stderr.write(out + '\n') 185 self._lex_errors += 1 186 187 188 def AddLines(self, count): 189 # Set the lexer position for the beginning of the next line. In the case 190 # of multiple lines, tokens can not exist on any of the lines except the 191 # last one, so the recorded value for previous lines are unused. We still 192 # fill the array however, to make sure the line count is correct. 193 self.Lexer().lineno += count 194 for _ in range(count): 195 self.index.append(self.Lexer().lexpos) 196 197 def FileLineMsg(self, line, msg): 198 # Generate a message containing the file and line number of a token. 199 filename = self.Lexer().filename 200 if filename: 201 return "%s(%d) : %s" % (filename, line + 1, msg) 202 return "<BuiltIn> : %s" % msg 203 204 def SourceLine(self, line, pos): 205 # Create a source line marker 206 caret = ' ' * pos + '^' 207 # We decrement the line number since the array is 0 based while the 208 # line numbers are 1 based. 209 return "%s\n%s" % (self.lines[line - 1], caret) 210 211 def ErrorMessage(self, line, pos, msg): 212 return "\n%s\n%s" % ( 213 self.FileLineMsg(line, msg), 214 self.SourceLine(line, pos)) 215 216# 217# Tokenizer 218# 219# The token function returns the next token provided by IDLLexer for matching 220# against the leaf paterns. 221# 222 def token(self): 223 tok = self.Lexer().token() 224 if tok: 225 self.last = tok 226 return tok 227 228 229 def GetTokens(self): 230 outlist = [] 231 while True: 232 t = self.Lexer().token() 233 if not t: 234 break 235 outlist.append(t) 236 return outlist 237 238 def Tokenize(self, data, filename='__no_file__'): 239 lexer = self.Lexer() 240 lexer.lineno = 1 241 lexer.filename = filename 242 lexer.input(data) 243 self.lines = data.split('\n') 244 245 def KnownTokens(self): 246 return self.tokens 247 248 def Lexer(self): 249 if not self._lexobj: 250 self._lexobj = lex.lex(object=self, lextab=None, optimize=0) 251 return self._lexobj 252 253 def _AddToken(self, token): 254 if token in self.tokens: 255 raise RuntimeError('Same token: ' + token) 256 self.tokens.append(token) 257 258 def _AddTokens(self, tokens): 259 for token in tokens: 260 self._AddToken(token) 261 262 def _AddKeywords(self, keywords): 263 for key in keywords: 264 value = key.upper() 265 self._AddToken(value) 266 self.keywords[key] = value 267 268 def _DelKeywords(self, keywords): 269 for key in keywords: 270 self.tokens.remove(key.upper()) 271 del self.keywords[key] 272 273 def __init__(self): 274 self.index = [0] 275 self._lex_errors = 0 276 self.linex = [] 277 self.filename = None 278 self.keywords = {} 279 self.tokens = [] 280 self._AddTokens(IDLLexer.tokens) 281 self._AddKeywords(IDLLexer.keywords) 282 self._lexobj = None 283 self.last = None 284 self.lines = None 285 286# If run by itself, attempt to build the lexer 287if __name__ == '__main__': 288 lexer_object = IDLLexer() 289