1#!/usr/bin/env python
2# Copyright (c) 2013 The Chromium Authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
6""" Lexer for PPAPI IDL
7
8The lexer uses the PLY library to build a tokenizer which understands both
9WebIDL and Pepper tokens.
10
11WebIDL, and WebIDL regular expressions can be found at:
12   http://www.w3.org/TR/2012/CR-WebIDL-20120419/
13PLY can be found at:
14   http://www.dabeaz.com/ply/
15"""
16
17import os.path
18import sys
19
20#
21# Try to load the ply module, if not, then assume it is in the third_party
22# directory.
23#
24try:
25  # Disable lint check which fails to find the ply module.
26  # pylint: disable=F0401
27  from ply import lex
28except ImportError:
29  module_path, module_name = os.path.split(__file__)
30  third_party = os.path.join(module_path, '..', '..', 'third_party')
31  sys.path.append(third_party)
32  # pylint: disable=F0401
33  from ply import lex
34
35#
36# IDL Lexer
37#
38class IDLLexer(object):
39  # 'literals' is a value expected by lex which specifies a list of valid
40  # literal tokens, meaning the token type and token value are identical.
41  literals = r'"*.(){}[],;:=+-/~|&^?<>'
42
43  # 't_ignore' contains ignored characters (spaces and tabs)
44  t_ignore = ' \t'
45
46  # 'tokens' is a value required by lex which specifies the complete list
47  # of valid token types.
48  tokens = [
49    # Data types
50      'float',
51      'integer',
52      'string',
53
54    # Symbol and keywords types
55      'COMMENT',
56      'identifier',
57
58    # MultiChar operators
59      'ELLIPSIS',
60  ]
61
62  # 'keywords' is a map of string to token type.  All tokens matching
63  # KEYWORD_OR_SYMBOL are matched against keywords dictionary, to determine
64  # if the token is actually a keyword.
65  keywords = {
66    'any' : 'ANY',
67    'attribute' : 'ATTRIBUTE',
68    'boolean' : 'BOOLEAN',
69    'byte' : 'BYTE',
70    'ByteString' : 'BYTESTRING',
71    'callback' : 'CALLBACK',
72    'const' : 'CONST',
73    'creator' : 'CREATOR',
74    'Date' : 'DATE',
75    'deleter' : 'DELETER',
76    'dictionary' : 'DICTIONARY',
77    'DOMString' : 'DOMSTRING',
78    'double' : 'DOUBLE',
79    'enum'  : 'ENUM',
80    'false' : 'FALSE',
81    'float' : 'FLOAT',
82    'exception' : 'EXCEPTION',
83    'getter': 'GETTER',
84    'implements' : 'IMPLEMENTS',
85    'Infinity' : 'INFINITY',
86    'inherit' : 'INHERIT',
87    'interface' : 'INTERFACE',
88    'legacycaller' : 'LEGACYCALLER',
89    'long' : 'LONG',
90    'Nan' : 'NAN',
91    'null' : 'NULL',
92    'object' : 'OBJECT',
93    'octet' : 'OCTET',
94    'optional' : 'OPTIONAL',
95    'or' : 'OR',
96    'partial'  : 'PARTIAL',
97    'readonly' : 'READONLY',
98    'RegExp' : 'REGEXP',
99    'sequence' : 'SEQUENCE',
100    'serializer' : 'SERIALIZER',
101    'setter': 'SETTER',
102    'short' : 'SHORT',
103    'static' : 'STATIC',
104    'stringifier' : 'STRINGIFIER',
105    'typedef' : 'TYPEDEF',
106    'true' : 'TRUE',
107    'unsigned' : 'UNSIGNED',
108    'unrestricted' : 'UNRESTRICTED',
109    'void' : 'VOID'
110  }
111
112  # Token definitions
113  #
114  # Lex assumes any value or function in the form of 't_<TYPE>' represents a
115  # regular expression where a match will emit a token of type <TYPE>.  In the
116  # case of a function, the function is called when a match is made. These
117  # definitions come from WebIDL.
118  #
119  # These need to be methods for lexer construction, despite not using self.
120  # pylint: disable=R0201
121  def t_ELLIPSIS(self, t):
122    r'\.\.\.'
123    return t
124
125  # Regex needs to be in the docstring
126  # pylint: disable=C0301
127  def t_float(self, t):
128    r'-?(([0-9]+\.[0-9]*|[0-9]*\.[0-9]+)([Ee][+-]?[0-9]+)?|[0-9]+[Ee][+-]?[0-9]+)'
129    return t
130
131  def t_integer(self, t):
132    r'-?([1-9][0-9]*|0[Xx][0-9A-Fa-f]+|0[0-7]*)'
133    return t
134
135
136  # A line ending '\n', we use this to increment the line number
137  def t_LINE_END(self, t):
138    r'\n+'
139    self.AddLines(len(t.value))
140
141  # We do not process escapes in the IDL strings.  Strings are exclusively
142  # used for attributes and enums, and not used as typical 'C' constants.
143  def t_string(self, t):
144    r'"[^"]*"'
145    t.value = t.value[1:-1]
146    self.AddLines(t.value.count('\n'))
147    return t
148
149  # A C or C++ style comment:  /* xxx */ or //
150  def t_COMMENT(self, t):
151    r'(/\*(.|\n)*?\*/)|(//.*(\n[ \t]*//.*)*)'
152    self.AddLines(t.value.count('\n'))
153    return t
154
155  # A symbol or keyword.
156  def t_KEYWORD_OR_SYMBOL(self, t):
157    r'_?[A-Za-z][A-Za-z_0-9]*'
158
159    # All non-keywords are assumed to be symbols
160    t.type = self.keywords.get(t.value, 'identifier')
161
162    # We strip leading underscores so that you can specify symbols with the same
163    # value as a keywords (E.g. a dictionary named 'interface').
164    if t.value[0] == '_':
165      t.value = t.value[1:]
166    return t
167
168  def t_ANY_error(self, t):
169    msg = 'Unrecognized input'
170    line = self.Lexer().lineno
171
172    # If that line has not been accounted for, then we must have hit
173    # EoF, so compute the beginning of the line that caused the problem.
174    if line >= len(self.index):
175      # Find the offset in the line of the first word causing the issue
176      word = t.value.split()[0]
177      offs = self.lines[line - 1].find(word)
178      # Add the computed line's starting position
179      self.index.append(self.Lexer().lexpos - offs)
180      msg = 'Unexpected EoF reached after'
181
182    pos = self.Lexer().lexpos - self.index[line]
183    out = self.ErrorMessage(line, pos, msg)
184    sys.stderr.write(out + '\n')
185    self._lex_errors += 1
186
187
188  def AddLines(self, count):
189    # Set the lexer position for the beginning of the next line.  In the case
190    # of multiple lines, tokens can not exist on any of the lines except the
191    # last one, so the recorded value for previous lines are unused.  We still
192    # fill the array however, to make sure the line count is correct.
193    self.Lexer().lineno += count
194    for _ in range(count):
195      self.index.append(self.Lexer().lexpos)
196
197  def FileLineMsg(self, line, msg):
198    # Generate a message containing the file and line number of a token.
199    filename = self.Lexer().filename
200    if filename:
201      return "%s(%d) : %s" % (filename, line + 1, msg)
202    return "<BuiltIn> : %s" % msg
203
204  def SourceLine(self, line, pos):
205    # Create a source line marker
206    caret = ' ' * pos + '^'
207    # We decrement the line number since the array is 0 based while the
208    # line numbers are 1 based.
209    return "%s\n%s" % (self.lines[line - 1], caret)
210
211  def ErrorMessage(self, line, pos, msg):
212    return "\n%s\n%s" % (
213        self.FileLineMsg(line, msg),
214        self.SourceLine(line, pos))
215
216#
217# Tokenizer
218#
219# The token function returns the next token provided by IDLLexer for matching
220# against the leaf paterns.
221#
222  def token(self):
223    tok = self.Lexer().token()
224    if tok:
225      self.last = tok
226    return tok
227
228
229  def GetTokens(self):
230    outlist = []
231    while True:
232      t = self.Lexer().token()
233      if not t:
234        break
235      outlist.append(t)
236    return outlist
237
238  def Tokenize(self, data, filename='__no_file__'):
239    lexer = self.Lexer()
240    lexer.lineno = 1
241    lexer.filename = filename
242    lexer.input(data)
243    self.lines = data.split('\n')
244
245  def KnownTokens(self):
246    return self.tokens
247
248  def Lexer(self):
249    if not self._lexobj:
250      self._lexobj = lex.lex(object=self, lextab=None, optimize=0)
251    return self._lexobj
252
253  def _AddToken(self, token):
254    if token in self.tokens:
255      raise RuntimeError('Same token: ' + token)
256    self.tokens.append(token)
257
258  def _AddTokens(self, tokens):
259    for token in tokens:
260      self._AddToken(token)
261
262  def _AddKeywords(self, keywords):
263    for key in keywords:
264      value = key.upper()
265      self._AddToken(value)
266      self.keywords[key] = value
267
268  def _DelKeywords(self, keywords):
269    for key in keywords:
270      self.tokens.remove(key.upper())
271      del self.keywords[key]
272
273  def __init__(self):
274    self.index = [0]
275    self._lex_errors = 0
276    self.linex = []
277    self.filename = None
278    self.keywords = {}
279    self.tokens = []
280    self._AddTokens(IDLLexer.tokens)
281    self._AddKeywords(IDLLexer.keywords)
282    self._lexobj = None
283    self.last = None
284    self.lines = None
285
286# If run by itself, attempt to build the lexer
287if __name__ == '__main__':
288  lexer_object = IDLLexer()
289