idl_parser.py revision 2a99a7e74a7f215066514fe81d2bfa6639d9eddd
1#!/usr/bin/env python 2# Copyright (c) 2012 The Chromium Authors. All rights reserved. 3# Use of this source code is governed by a BSD-style license that can be 4# found in the LICENSE file. 5 6""" Parser for PPAPI IDL """ 7 8# 9# IDL Parser 10# 11# The parser is uses the PLY yacc library to build a set of parsing rules based 12# on WebIDL. 13# 14# WebIDL, and WebIDL regular expressions can be found at: 15# http://dev.w3.org/2006/webapi/WebIDL/ 16# PLY can be found at: 17# http://www.dabeaz.com/ply/ 18# 19# The parser generates a tree by recursively matching sets of items against 20# defined patterns. When a match is made, that set of items is reduced 21# to a new item. The new item can provide a match for parent patterns. 22# In this way an AST is built (reduced) depth first. 23 24 25import getopt 26import glob 27import os.path 28import re 29import sys 30import time 31 32from idl_ast import IDLAst 33from idl_log import ErrOut, InfoOut, WarnOut 34from idl_lexer import IDLLexer 35from idl_node import IDLAttribute, IDLFile, IDLNode 36from idl_option import GetOption, Option, ParseOptions 37from idl_lint import Lint 38from idl_visitor import IDLVisitor 39 40from ply import lex 41from ply import yacc 42 43Option('build_debug', 'Debug tree building.') 44Option('parse_debug', 'Debug parse reduction steps.') 45Option('token_debug', 'Debug token generation.') 46Option('dump_tree', 'Dump the tree.') 47Option('srcroot', 'Working directory.', default=os.path.join('..', 'api')) 48Option('include_private', 'Include private IDL directory in default API paths.') 49 50# 51# ERROR_REMAP 52# 53# Maps the standard error formula into a more friendly error message. 54# 55ERROR_REMAP = { 56 'Unexpected ")" after "(".' : 'Empty argument list.', 57 'Unexpected ")" after ",".' : 'Missing argument.', 58 'Unexpected "}" after ",".' : 'Trailing comma in block.', 59 'Unexpected "}" after "{".' : 'Unexpected empty block.', 60 'Unexpected comment after "}".' : 'Unexpected trailing comment.', 61 'Unexpected "{" after keyword "enum".' : 'Enum missing name.', 62 'Unexpected "{" after keyword "struct".' : 'Struct missing name.', 63 'Unexpected "{" after keyword "interface".' : 'Interface missing name.', 64} 65 66# DumpReduction 67# 68# Prints out the set of items which matched a particular pattern and the 69# new item or set it was reduced to. 70def DumpReduction(cls, p): 71 if p[0] is None: 72 InfoOut.Log("OBJ: %s(%d) - None\n" % (cls, len(p))) 73 InfoOut.Log(" [%s]\n" % [str(x) for x in p[1:]]) 74 else: 75 out = "" 76 for index in range(len(p) - 1): 77 out += " >%s< " % str(p[index + 1]) 78 InfoOut.Log("OBJ: %s(%d) - %s : %s\n" % (cls, len(p), str(p[0]), out)) 79 80 81# CopyToList 82# 83# Takes an input item, list, or None, and returns a new list of that set. 84def CopyToList(item): 85 # If the item is 'Empty' make it an empty list 86 if not item: item = [] 87 88 # If the item is not a list 89 if type(item) is not type([]): item = [item] 90 91 # Make a copy we can modify 92 return list(item) 93 94 95 96# ListFromConcat 97# 98# Generate a new List by joining of two sets of inputs which can be an 99# individual item, a list of items, or None. 100def ListFromConcat(*items): 101 itemsout = [] 102 for item in items: 103 itemlist = CopyToList(item) 104 itemsout.extend(itemlist) 105 106 return itemsout 107 108 109# TokenTypeName 110# 111# Generate a string which has the type and value of the token. 112def TokenTypeName(t): 113 if t.type == 'SYMBOL': return 'symbol %s' % t.value 114 if t.type in ['HEX', 'INT', 'OCT', 'FLOAT']: 115 return 'value %s' % t.value 116 if t.type == 'STRING' : return 'string "%s"' % t.value 117 if t.type == 'COMMENT' : return 'comment' 118 if t.type == t.value: return '"%s"' % t.value 119 return 'keyword "%s"' % t.value 120 121 122# 123# IDL Parser 124# 125# The Parser inherits the from the Lexer to provide PLY with the tokenizing 126# definitions. Parsing patterns are encoded as function where p_<name> is 127# is called any time a patern matching the function documentation is found. 128# Paterns are expressed in the form of: 129# """ <new item> : <item> .... 130# | <item> ....""" 131# 132# Where new item is the result of a match against one or more sets of items 133# separated by the "|". 134# 135# The function is called with an object 'p' where p[0] is the output object 136# and p[n] is the set of inputs for positive values of 'n'. Len(p) can be 137# used to distinguish between multiple item sets in the pattern. 138# 139# For more details on parsing refer to the PLY documentation at 140# http://www.dabeaz.com/ply/ 141# 142# 143# The parser uses the following conventions: 144# a <type>_block defines a block of <type> definitions in the form of: 145# [comment] [ext_attr_block] <type> <name> '{' <type>_list '}' ';' 146# A block is reduced by returning an object of <type> with a name of <name> 147# which in turn has <type>_list as children. 148# 149# A [comment] is a optional C style comment block enclosed in /* ... */ which 150# is appended to the adjacent node as a child. 151# 152# A [ext_attr_block] is an optional list of Extended Attributes which is 153# appended to the adjacent node as a child. 154# 155# a <type>_list defines a list of <type> items which will be passed as a 156# list of children to the parent pattern. A list is in the form of: 157# [comment] [ext_attr_block] <...DEF...> ';' <type>_list | (empty) 158# or 159# [comment] [ext_attr_block] <...DEF...> <type>_cont 160# 161# In the first form, the list is reduced recursively, where the right side 162# <type>_list is first reduced then joined with pattern currently being 163# matched. The list is terminated with the (empty) pattern is matched. 164# 165# In the second form the list is reduced recursively, where the right side 166# <type>_cont is first reduced then joined with the pattern currently being 167# matched. The type_<cont> is in the form of: 168# ',' <type>_list | (empty) 169# The <type>_cont form is used to consume the ',' which only occurs when 170# there is more than one object in the list. The <type>_cont also provides 171# the terminating (empty) definition. 172# 173 174 175class IDLParser(IDLLexer): 176# TOP 177# 178# This pattern defines the top of the parse tree. The parse tree is in the 179# the form of: 180# 181# top 182# *modifiers 183# *comments 184# *ext_attr_block 185# ext_attr_list 186# attr_arg_list 187# *integer, value 188# *param_list 189# *typeref 190# 191# top_list 192# describe_block 193# describe_list 194# enum_block 195# enum_item 196# interface_block 197# member 198# label_block 199# label_item 200# struct_block 201# member 202# typedef_decl 203# typedef_data 204# typedef_func 205# 206# (* sub matches found at multiple levels and are not truly children of top) 207# 208# We force all input files to start with two comments. The first comment is a 209# Copyright notice followed by a set of file wide Extended Attributes, followed 210# by the file comment and finally by file level patterns. 211# 212 # Find the Copyright, File comment, and optional file wide attributes. We 213 # use a match with COMMENT instead of comments to force the token to be 214 # present. The extended attributes and the top_list become siblings which 215 # in turn are children of the file object created from the results of top. 216 def p_top(self, p): 217 """top : COMMENT COMMENT ext_attr_block top_list""" 218 219 Copyright = self.BuildComment('Copyright', p, 1) 220 Filedoc = self.BuildComment('Comment', p, 2) 221 222 p[0] = ListFromConcat(Copyright, Filedoc, p[3], p[4]) 223 if self.parse_debug: DumpReduction('top', p) 224 225 def p_top_short(self, p): 226 """top : COMMENT ext_attr_block top_list""" 227 Copyright = self.BuildComment('Copyright', p, 1) 228 Filedoc = IDLNode('Comment', self.lexobj.filename, p.lineno(2)-1, 229 p.lexpos(2)-1, [self.BuildAttribute('NAME', ''), 230 self.BuildAttribute('FORM', 'cc')]) 231 p[0] = ListFromConcat(Copyright, Filedoc, p[2], p[3]) 232 if self.parse_debug: DumpReduction('top', p) 233 234 # Build a list of top level items. 235 def p_top_list(self, p): 236 """top_list : callback_decl top_list 237 | describe_block top_list 238 | dictionary_block top_list 239 | enum_block top_list 240 | inline top_list 241 | interface_block top_list 242 | label_block top_list 243 | namespace top_list 244 | struct_block top_list 245 | typedef_decl top_list 246 | bad_decl top_list 247 | """ 248 if len(p) > 2: 249 p[0] = ListFromConcat(p[1], p[2]) 250 if self.parse_debug: DumpReduction('top_list', p) 251 252 # Recover from error and continue parsing at the next top match. 253 def p_top_error(self, p): 254 """top_list : error top_list""" 255 p[0] = p[2] 256 257 # Recover from error and continue parsing at the next top match. 258 def p_bad_decl(self, p): 259 """bad_decl : modifiers SYMBOL error '}' ';'""" 260 p[0] = [] 261 262# 263# Modifier List 264# 265# 266 def p_modifiers(self, p): 267 """modifiers : comments ext_attr_block""" 268 p[0] = ListFromConcat(p[1], p[2]) 269 if self.parse_debug: DumpReduction('modifiers', p) 270 271# 272# Comments 273# 274# Comments are optional list of C style comment objects. Comments are returned 275# as a list or None. 276# 277 def p_comments(self, p): 278 """comments : COMMENT comments 279 | """ 280 if len(p) > 1: 281 child = self.BuildComment('Comment', p, 1) 282 p[0] = ListFromConcat(child, p[2]) 283 if self.parse_debug: DumpReduction('comments', p) 284 else: 285 if self.parse_debug: DumpReduction('no comments', p) 286 287 288# 289# Namespace 290# 291# A namespace provides a named scope to an enclosed top_list. 292# 293 def p_namespace(self, p): 294 """namespace : modifiers NAMESPACE namespace_name '{' top_list '}' ';'""" 295 children = ListFromConcat(p[1], p[5]) 296 p[0] = self.BuildNamed('Namespace', p, 3, children) 297 298 # We allow namespace names of the form foo.bar.baz. 299 def p_namespace_name(self, p): 300 """namespace_name : SYMBOL 301 | SYMBOL '.' namespace_name""" 302 p[0] = "".join(p[1:]) 303 304 305# 306# Dictionary 307# 308# A dictionary is a named list of optional and required members. 309# 310 def p_dictionary_block(self, p): 311 """dictionary_block : modifiers DICTIONARY SYMBOL '{' struct_list '}' ';'""" 312 p[0] = self.BuildNamed('Dictionary', p, 3, ListFromConcat(p[1], p[5])) 313 314# 315# Callback 316# 317# A callback is essentially a single function declaration (outside of an 318# Interface). 319# 320 def p_callback_decl(self, p): 321 """callback_decl : modifiers CALLBACK SYMBOL '=' SYMBOL param_list ';'""" 322 children = ListFromConcat(p[1], p[6]) 323 p[0] = self.BuildNamed('Callback', p, 3, children) 324 325 326# 327# Inline 328# 329# Inline blocks define option code to be emitted based on language tag, 330# in the form of: 331# #inline <LANGUAGE> 332# <CODE> 333# #endinl 334# 335 def p_inline(self, p): 336 """inline : modifiers INLINE""" 337 words = p[2].split() 338 name = self.BuildAttribute('NAME', words[1]) 339 lines = p[2].split('\n') 340 value = self.BuildAttribute('VALUE', '\n'.join(lines[1:-1]) + '\n') 341 children = ListFromConcat(name, value, p[1]) 342 p[0] = self.BuildProduction('Inline', p, 2, children) 343 if self.parse_debug: DumpReduction('inline', p) 344 345# Extended Attributes 346# 347# Extended Attributes denote properties which will be applied to a node in the 348# AST. A list of extended attributes are denoted by a brackets '[' ... ']' 349# enclosing a comma separated list of extended attributes in the form of: 350# 351# Name 352# Name=HEX | INT | OCT | FLOAT 353# Name="STRING" 354# Name=Function(arg ...) 355# TODO(noelallen) -Not currently supported: 356# ** Name(arg ...) ... 357# ** Name=Scope::Value 358# 359# Extended Attributes are returned as a list or None. 360 361 def p_ext_attr_block(self, p): 362 """ext_attr_block : '[' ext_attr_list ']' 363 | """ 364 if len(p) > 1: 365 p[0] = p[2] 366 if self.parse_debug: DumpReduction('ext_attr_block', p) 367 else: 368 if self.parse_debug: DumpReduction('no ext_attr_block', p) 369 370 def p_ext_attr_list(self, p): 371 """ext_attr_list : SYMBOL '=' SYMBOL ext_attr_cont 372 | SYMBOL '=' value ext_attr_cont 373 | SYMBOL '=' SYMBOL param_list ext_attr_cont 374 | SYMBOL ext_attr_cont""" 375 # If there are 4 tokens plus a return slot, this must be in the form 376 # SYMBOL = SYMBOL|value ext_attr_cont 377 if len(p) == 5: 378 p[0] = ListFromConcat(self.BuildAttribute(p[1], p[3]), p[4]) 379 # If there are 5 tokens plus a return slot, this must be in the form 380 # SYMBOL = SYMBOL (param_list) ext_attr_cont 381 elif len(p) == 6: 382 member = self.BuildNamed('Member', p, 3, [p[4]]) 383 p[0] = ListFromConcat(self.BuildAttribute(p[1], member), p[5]) 384 # Otherwise, this must be: SYMBOL ext_attr_cont 385 else: 386 p[0] = ListFromConcat(self.BuildAttribute(p[1], 'True'), p[2]) 387 if self.parse_debug: DumpReduction('ext_attribute_list', p) 388 389 def p_ext_attr_list_values(self, p): 390 """ext_attr_list : SYMBOL '=' '(' values ')' ext_attr_cont 391 | SYMBOL '=' '(' symbols ')' ext_attr_cont""" 392 p[0] = ListFromConcat(self.BuildAttribute(p[1], p[4]), p[6]) 393 394 def p_values(self, p): 395 """values : value values_cont""" 396 p[0] = ListFromConcat(p[1], p[2]) 397 398 def p_symbols(self, p): 399 """symbols : SYMBOL symbols_cont""" 400 p[0] = ListFromConcat(p[1], p[2]) 401 402 def p_symbols_cont(self, p): 403 """symbols_cont : ',' SYMBOL symbols_cont 404 | """ 405 if len(p) > 1: p[0] = ListFromConcat(p[2], p[3]) 406 407 def p_values_cont(self, p): 408 """values_cont : ',' value values_cont 409 | """ 410 if len(p) > 1: p[0] = ListFromConcat(p[2], p[3]) 411 412 def p_ext_attr_cont(self, p): 413 """ext_attr_cont : ',' ext_attr_list 414 |""" 415 if len(p) > 1: p[0] = p[2] 416 if self.parse_debug: DumpReduction('ext_attribute_cont', p) 417 418 def p_ext_attr_func(self, p): 419 """ext_attr_list : SYMBOL '(' attr_arg_list ')' ext_attr_cont""" 420 p[0] = ListFromConcat(self.BuildAttribute(p[1] + '()', p[3]), p[5]) 421 if self.parse_debug: DumpReduction('attr_arg_func', p) 422 423 def p_ext_attr_arg_list(self, p): 424 """attr_arg_list : SYMBOL attr_arg_cont 425 | value attr_arg_cont""" 426 p[0] = ListFromConcat(p[1], p[2]) 427 428 def p_attr_arg_cont(self, p): 429 """attr_arg_cont : ',' attr_arg_list 430 | """ 431 if self.parse_debug: DumpReduction('attr_arg_cont', p) 432 if len(p) > 1: p[0] = p[2] 433 434 def p_attr_arg_error(self, p): 435 """attr_arg_cont : error attr_arg_cont""" 436 p[0] = p[2] 437 if self.parse_debug: DumpReduction('attr_arg_error', p) 438 439 440# 441# Describe 442# 443# A describe block is defined at the top level. It provides a mechanism for 444# attributing a group of ext_attr to a describe_list. Members of the 445# describe list are language specific 'Type' declarations 446# 447 def p_describe_block(self, p): 448 """describe_block : modifiers DESCRIBE '{' describe_list '}' ';'""" 449 children = ListFromConcat(p[1], p[4]) 450 p[0] = self.BuildProduction('Describe', p, 2, children) 451 if self.parse_debug: DumpReduction('describe_block', p) 452 453 # Recover from describe error and continue parsing at the next top match. 454 def p_describe_error(self, p): 455 """describe_list : error describe_list""" 456 p[0] = [] 457 458 def p_describe_list(self, p): 459 """describe_list : modifiers SYMBOL ';' describe_list 460 | modifiers ENUM ';' describe_list 461 | modifiers STRUCT ';' describe_list 462 | modifiers TYPEDEF ';' describe_list 463 | """ 464 if len(p) > 1: 465 Type = self.BuildNamed('Type', p, 2, p[1]) 466 p[0] = ListFromConcat(Type, p[4]) 467 468# 469# Constant Values (integer, value) 470# 471# Constant values can be found at various levels. A Constant value is returns 472# as the string value after validated against a FLOAT, HEX, INT, OCT or 473# STRING pattern as appropriate. 474# 475 def p_value(self, p): 476 """value : FLOAT 477 | HEX 478 | INT 479 | OCT 480 | STRING""" 481 p[0] = p[1] 482 if self.parse_debug: DumpReduction('value', p) 483 484 def p_value_lshift(self, p): 485 """value : integer LSHIFT INT""" 486 p[0] = "%s << %s" % (p[1], p[3]) 487 if self.parse_debug: DumpReduction('value', p) 488 489# Integers are numbers which may not be floats used in cases like array sizes. 490 def p_integer(self, p): 491 """integer : HEX 492 | INT 493 | OCT""" 494 p[0] = p[1] 495 if self.parse_debug: DumpReduction('integer', p) 496 497# 498# Expression 499# 500# A simple arithmetic expression. 501# 502 precedence = ( 503 ('left','|','&','^'), 504 ('left','LSHIFT','RSHIFT'), 505 ('left','+','-'), 506 ('left','*','/'), 507 ('right','UMINUS','~'), 508 ) 509 510 def p_expression_binop(self, p): 511 """expression : expression LSHIFT expression 512 | expression RSHIFT expression 513 | expression '|' expression 514 | expression '&' expression 515 | expression '^' expression 516 | expression '+' expression 517 | expression '-' expression 518 | expression '*' expression 519 | expression '/' expression""" 520 p[0] = "%s %s %s" % (str(p[1]), str(p[2]), str(p[3])) 521 if self.parse_debug: DumpReduction('expression_binop', p) 522 523 def p_expression_unop(self, p): 524 """expression : '-' expression %prec UMINUS 525 | '~' expression %prec '~'""" 526 p[0] = "%s%s" % (str(p[1]), str(p[2])) 527 if self.parse_debug: DumpReduction('expression_unop', p) 528 529 def p_expression_term(self, p): 530 "expression : '(' expression ')'" 531 p[0] = "%s%s%s" % (str(p[1]), str(p[2]), str(p[3])) 532 if self.parse_debug: DumpReduction('expression_term', p) 533 534 def p_expression_symbol(self, p): 535 "expression : SYMBOL" 536 p[0] = p[1] 537 if self.parse_debug: DumpReduction('expression_symbol', p) 538 539 def p_expression_integer(self, p): 540 "expression : integer" 541 p[0] = p[1] 542 if self.parse_debug: DumpReduction('expression_integer', p) 543 544# 545# Array List 546# 547# Defined a list of array sizes (if any). 548# 549 def p_arrays(self, p): 550 """arrays : '[' ']' arrays 551 | '[' integer ']' arrays 552 | """ 553 # If there are 3 tokens plus a return slot it is an unsized array 554 if len(p) == 4: 555 array = self.BuildProduction('Array', p, 1) 556 p[0] = ListFromConcat(array, p[3]) 557 # If there are 4 tokens plus a return slot it is a fixed array 558 elif len(p) == 5: 559 count = self.BuildAttribute('FIXED', p[2]) 560 array = self.BuildProduction('Array', p, 2, [count]) 561 p[0] = ListFromConcat(array, p[4]) 562 # If there is only a return slot, do not fill it for this terminator. 563 elif len(p) == 1: return 564 if self.parse_debug: DumpReduction('arrays', p) 565 566 567# An identifier is a legal value for a parameter or attribute name. Lots of 568# existing IDL files use "callback" as a parameter/attribute name, so we allow 569# a SYMBOL or the CALLBACK keyword. 570 def p_identifier(self, p): 571 """identifier : SYMBOL 572 | CALLBACK""" 573 p[0] = p[1] 574 # Save the line number of the underlying token (otherwise it gets 575 # discarded), since we use it in the productions with an identifier in 576 # them. 577 p.set_lineno(0, p.lineno(1)) 578 579# 580# Parameter List 581# 582# A parameter list is a collection of arguments which are passed to a 583# function. 584# 585 def p_param_list(self, p): 586 """param_list : '(' param_item param_cont ')' 587 | '(' ')' """ 588 if len(p) > 3: 589 args = ListFromConcat(p[2], p[3]) 590 else: 591 args = [] 592 p[0] = self.BuildProduction('Callspec', p, 1, args) 593 if self.parse_debug: DumpReduction('param_list', p) 594 595 def p_param_item(self, p): 596 """param_item : modifiers optional SYMBOL arrays identifier""" 597 typeref = self.BuildAttribute('TYPEREF', p[3]) 598 children = ListFromConcat(p[1], p[2], typeref, p[4]) 599 p[0] = self.BuildNamed('Param', p, 5, children) 600 if self.parse_debug: DumpReduction('param_item', p) 601 602 def p_optional(self, p): 603 """optional : OPTIONAL 604 | """ 605 if len(p) == 2: 606 p[0] = self.BuildAttribute('OPTIONAL', True) 607 608 609 def p_param_cont(self, p): 610 """param_cont : ',' param_item param_cont 611 | """ 612 if len(p) > 1: 613 p[0] = ListFromConcat(p[2], p[3]) 614 if self.parse_debug: DumpReduction('param_cont', p) 615 616 def p_param_error(self, p): 617 """param_cont : error param_cont""" 618 p[0] = p[2] 619 620 621# 622# Typedef 623# 624# A typedef creates a new referencable type. The typedef can specify an array 625# definition as well as a function declaration. 626# 627 def p_typedef_data(self, p): 628 """typedef_decl : modifiers TYPEDEF SYMBOL SYMBOL ';' """ 629 typeref = self.BuildAttribute('TYPEREF', p[3]) 630 children = ListFromConcat(p[1], typeref) 631 p[0] = self.BuildNamed('Typedef', p, 4, children) 632 if self.parse_debug: DumpReduction('typedef_data', p) 633 634 def p_typedef_array(self, p): 635 """typedef_decl : modifiers TYPEDEF SYMBOL arrays SYMBOL ';' """ 636 typeref = self.BuildAttribute('TYPEREF', p[3]) 637 children = ListFromConcat(p[1], typeref, p[4]) 638 p[0] = self.BuildNamed('Typedef', p, 5, children) 639 if self.parse_debug: DumpReduction('typedef_array', p) 640 641 def p_typedef_func(self, p): 642 """typedef_decl : modifiers TYPEDEF SYMBOL SYMBOL param_list ';' """ 643 typeref = self.BuildAttribute('TYPEREF', p[3]) 644 children = ListFromConcat(p[1], typeref, p[5]) 645 p[0] = self.BuildNamed('Typedef', p, 4, children) 646 if self.parse_debug: DumpReduction('typedef_func', p) 647 648# 649# Enumeration 650# 651# An enumeration is a set of named integer constants. An enumeration 652# is valid type which can be referenced in other definitions. 653# 654 def p_enum_block(self, p): 655 """enum_block : modifiers ENUM SYMBOL '{' enum_list '}' ';'""" 656 p[0] = self.BuildNamed('Enum', p, 3, ListFromConcat(p[1], p[5])) 657 if self.parse_debug: DumpReduction('enum_block', p) 658 659 # Recover from enum error and continue parsing at the next top match. 660 def p_enum_errorA(self, p): 661 """enum_block : modifiers ENUM error '{' enum_list '}' ';'""" 662 p[0] = [] 663 664 def p_enum_errorB(self, p): 665 """enum_block : modifiers ENUM error ';'""" 666 p[0] = [] 667 668 def p_enum_list(self, p): 669 """enum_list : modifiers SYMBOL '=' expression enum_cont 670 | modifiers SYMBOL enum_cont""" 671 if len(p) > 4: 672 val = self.BuildAttribute('VALUE', p[4]) 673 enum = self.BuildNamed('EnumItem', p, 2, ListFromConcat(val, p[1])) 674 p[0] = ListFromConcat(enum, p[5]) 675 else: 676 enum = self.BuildNamed('EnumItem', p, 2, p[1]) 677 p[0] = ListFromConcat(enum, p[3]) 678 if self.parse_debug: DumpReduction('enum_list', p) 679 680 def p_enum_cont(self, p): 681 """enum_cont : ',' enum_list 682 |""" 683 if len(p) > 1: p[0] = p[2] 684 if self.parse_debug: DumpReduction('enum_cont', p) 685 686 def p_enum_cont_error(self, p): 687 """enum_cont : error enum_cont""" 688 p[0] = p[2] 689 if self.parse_debug: DumpReduction('enum_error', p) 690 691 692# 693# Label 694# 695# A label is a special kind of enumeration which allows us to go from a 696# set of labels 697# 698 def p_label_block(self, p): 699 """label_block : modifiers LABEL SYMBOL '{' label_list '}' ';'""" 700 p[0] = self.BuildNamed('Label', p, 3, ListFromConcat(p[1], p[5])) 701 if self.parse_debug: DumpReduction('label_block', p) 702 703 def p_label_list(self, p): 704 """label_list : modifiers SYMBOL '=' FLOAT label_cont""" 705 val = self.BuildAttribute('VALUE', p[4]) 706 label = self.BuildNamed('LabelItem', p, 2, ListFromConcat(val, p[1])) 707 p[0] = ListFromConcat(label, p[5]) 708 if self.parse_debug: DumpReduction('label_list', p) 709 710 def p_label_cont(self, p): 711 """label_cont : ',' label_list 712 |""" 713 if len(p) > 1: p[0] = p[2] 714 if self.parse_debug: DumpReduction('label_cont', p) 715 716 def p_label_cont_error(self, p): 717 """label_cont : error label_cont""" 718 p[0] = p[2] 719 if self.parse_debug: DumpReduction('label_error', p) 720 721 722# 723# Members 724# 725# A member attribute or function of a struct or interface. 726# 727 def p_member_attribute(self, p): 728 """member_attribute : modifiers SYMBOL arrays questionmark identifier""" 729 typeref = self.BuildAttribute('TYPEREF', p[2]) 730 children = ListFromConcat(p[1], typeref, p[3], p[4]) 731 p[0] = self.BuildNamed('Member', p, 5, children) 732 if self.parse_debug: DumpReduction('attribute', p) 733 734 def p_member_function(self, p): 735 """member_function : modifiers static SYMBOL SYMBOL param_list""" 736 typeref = self.BuildAttribute('TYPEREF', p[3]) 737 children = ListFromConcat(p[1], p[2], typeref, p[5]) 738 p[0] = self.BuildNamed('Member', p, 4, children) 739 if self.parse_debug: DumpReduction('function', p) 740 741 def p_static(self, p): 742 """static : STATIC 743 | """ 744 if len(p) == 2: 745 p[0] = self.BuildAttribute('STATIC', True) 746 747 def p_questionmark(self, p): 748 """questionmark : '?' 749 | """ 750 if len(p) == 2: 751 p[0] = self.BuildAttribute('OPTIONAL', True) 752 753# 754# Interface 755# 756# An interface is a named collection of functions. 757# 758 def p_interface_block(self, p): 759 """interface_block : modifiers INTERFACE SYMBOL '{' interface_list '}' ';'""" 760 p[0] = self.BuildNamed('Interface', p, 3, ListFromConcat(p[1], p[5])) 761 if self.parse_debug: DumpReduction('interface_block', p) 762 763 def p_interface_error(self, p): 764 """interface_block : modifiers INTERFACE error '{' interface_list '}' ';'""" 765 p[0] = [] 766 767 def p_interface_list(self, p): 768 """interface_list : member_function ';' interface_list 769 | """ 770 if len(p) > 1 : 771 p[0] = ListFromConcat(p[1], p[3]) 772 if self.parse_debug: DumpReduction('interface_list', p) 773 774 775# 776# Struct 777# 778# A struct is a named collection of members which in turn reference other 779# types. The struct is a referencable type. 780# 781 def p_struct_block(self, p): 782 """struct_block : modifiers STRUCT SYMBOL '{' struct_list '}' ';'""" 783 children = ListFromConcat(p[1], p[5]) 784 p[0] = self.BuildNamed('Struct', p, 3, children) 785 if self.parse_debug: DumpReduction('struct_block', p) 786 787 # Recover from struct error and continue parsing at the next top match. 788 def p_struct_error(self, p): 789 """enum_block : modifiers STRUCT error '{' struct_list '}' ';'""" 790 p[0] = [] 791 792 def p_struct_list(self, p): 793 """struct_list : member_attribute ';' struct_list 794 | member_function ';' struct_list 795 |""" 796 if len(p) > 1: p[0] = ListFromConcat(p[1], p[3]) 797 798 799# 800# Parser Errors 801# 802# p_error is called whenever the parser can not find a pattern match for 803# a set of items from the current state. The p_error function defined here 804# is triggered logging an error, and parsing recover happens as the 805# p_<type>_error functions defined above are called. This allows the parser 806# to continue so as to capture more than one error per file. 807# 808 def p_error(self, t): 809 filename = self.lexobj.filename 810 self.parse_errors += 1 811 if t: 812 lineno = t.lineno 813 pos = t.lexpos 814 prev = self.yaccobj.symstack[-1] 815 if type(prev) == lex.LexToken: 816 msg = "Unexpected %s after %s." % ( 817 TokenTypeName(t), TokenTypeName(prev)) 818 else: 819 msg = "Unexpected %s." % (t.value) 820 else: 821 lineno = self.last.lineno 822 pos = self.last.lexpos 823 msg = "Unexpected end of file after %s." % TokenTypeName(self.last) 824 self.yaccobj.restart() 825 826 # Attempt to remap the error to a friendlier form 827 if msg in ERROR_REMAP: 828 msg = ERROR_REMAP[msg] 829 830 # Log the error 831 ErrOut.LogLine(filename, lineno, pos, msg) 832 833 def Warn(self, node, msg): 834 WarnOut.LogLine(node.filename, node.lineno, node.pos, msg) 835 self.parse_warnings += 1 836 837 def __init__(self): 838 IDLLexer.__init__(self) 839 self.yaccobj = yacc.yacc(module=self, tabmodule=None, debug=False, 840 optimize=0, write_tables=0) 841 842 self.build_debug = GetOption('build_debug') 843 self.parse_debug = GetOption('parse_debug') 844 self.token_debug = GetOption('token_debug') 845 self.verbose = GetOption('verbose') 846 self.parse_errors = 0 847 848# 849# Tokenizer 850# 851# The token function returns the next token provided by IDLLexer for matching 852# against the leaf paterns. 853# 854 def token(self): 855 tok = self.lexobj.token() 856 if tok: 857 self.last = tok 858 if self.token_debug: 859 InfoOut.Log("TOKEN %s(%s)" % (tok.type, tok.value)) 860 return tok 861 862# 863# BuildProduction 864# 865# Production is the set of items sent to a grammar rule resulting in a new 866# item being returned. 867# 868# p - Is the Yacc production object containing the stack of items 869# index - Index into the production of the name for the item being produced. 870# cls - The type of item being producted 871# childlist - The children of the new item 872 def BuildProduction(self, cls, p, index, childlist=None): 873 if not childlist: childlist = [] 874 filename = self.lexobj.filename 875 lineno = p.lineno(index) 876 pos = p.lexpos(index) 877 out = IDLNode(cls, filename, lineno, pos, childlist) 878 if self.build_debug: 879 InfoOut.Log("Building %s" % out) 880 return out 881 882 def BuildNamed(self, cls, p, index, childlist=None): 883 if not childlist: childlist = [] 884 childlist.append(self.BuildAttribute('NAME', p[index])) 885 return self.BuildProduction(cls, p, index, childlist) 886 887 def BuildComment(self, cls, p, index): 888 name = p[index] 889 890 # Remove comment markers 891 lines = [] 892 if name[:2] == '//': 893 # For C++ style, remove any leading whitespace and the '//' marker from 894 # each line. 895 form = 'cc' 896 for line in name.split('\n'): 897 start = line.find('//') 898 lines.append(line[start+2:]) 899 else: 900 # For C style, remove ending '*/'' 901 form = 'c' 902 for line in name[:-2].split('\n'): 903 # Remove characters until start marker for this line '*' if found 904 # otherwise it should be blank. 905 offs = line.find('*') 906 if offs >= 0: 907 line = line[offs + 1:].rstrip() 908 else: 909 line = '' 910 lines.append(line) 911 name = '\n'.join(lines) 912 913 childlist = [self.BuildAttribute('NAME', name), 914 self.BuildAttribute('FORM', form)] 915 return self.BuildProduction(cls, p, index, childlist) 916 917# 918# BuildAttribute 919# 920# An ExtendedAttribute is a special production that results in a property 921# which is applied to the adjacent item. Attributes have no children and 922# instead represent key/value pairs. 923# 924 def BuildAttribute(self, key, val): 925 return IDLAttribute(key, val) 926 927 928# 929# ParseData 930# 931# Attempts to parse the current data loaded in the lexer. 932# 933 def ParseData(self, data, filename='<Internal>'): 934 self.SetData(filename, data) 935 try: 936 self.parse_errors = 0 937 self.parse_warnings = 0 938 return self.yaccobj.parse(lexer=self) 939 940 except lex.LexError as le: 941 ErrOut.Log(str(le)) 942 return [] 943 944# 945# ParseFile 946# 947# Loads a new file into the lexer and attemps to parse it. 948# 949 def ParseFile(self, filename): 950 date = time.ctime(os.path.getmtime(filename)) 951 data = open(filename).read() 952 if self.verbose: 953 InfoOut.Log("Parsing %s" % filename) 954 try: 955 out = self.ParseData(data, filename) 956 957 # If we have a src root specified, remove it from the path 958 srcroot = GetOption('srcroot') 959 if srcroot and filename.find(srcroot) == 0: 960 filename = filename[len(srcroot) + 1:] 961 filenode = IDLFile(filename, out, self.parse_errors + self.lex_errors) 962 filenode.SetProperty('DATETIME', date) 963 return filenode 964 965 except Exception as e: 966 ErrOut.LogLine(filename, self.last.lineno, self.last.lexpos, 967 'Internal parsing error - %s.' % str(e)) 968 raise 969 970 971 972# 973# Flatten Tree 974# 975# Flattens the tree of IDLNodes for use in testing. 976# 977def FlattenTree(node): 978 add_self = False 979 out = [] 980 for child in node.children: 981 if child.IsA('Comment'): 982 add_self = True 983 else: 984 out.extend(FlattenTree(child)) 985 986 if add_self: 987 out = [str(node)] + out 988 return out 989 990 991def TestErrors(filename, filenode): 992 nodelist = filenode.GetChildren() 993 994 lexer = IDLLexer() 995 data = open(filename).read() 996 lexer.SetData(filename, data) 997 998 pass_comments = [] 999 fail_comments = [] 1000 while True: 1001 tok = lexer.lexobj.token() 1002 if tok == None: break 1003 if tok.type == 'COMMENT': 1004 args = tok.value[3:-3].split() 1005 if args[0] == 'OK': 1006 pass_comments.append((tok.lineno, ' '.join(args[1:]))) 1007 else: 1008 if args[0] == 'FAIL': 1009 fail_comments.append((tok.lineno, ' '.join(args[1:]))) 1010 obj_list = [] 1011 for node in nodelist: 1012 obj_list.extend(FlattenTree(node)) 1013 1014 errors = 0 1015 1016 # 1017 # Check for expected successes 1018 # 1019 obj_cnt = len(obj_list) 1020 pass_cnt = len(pass_comments) 1021 if obj_cnt != pass_cnt: 1022 InfoOut.Log("Mismatched pass (%d) vs. nodes built (%d)." 1023 % (pass_cnt, obj_cnt)) 1024 InfoOut.Log("PASS: %s" % [x[1] for x in pass_comments]) 1025 InfoOut.Log("OBJS: %s" % obj_list) 1026 errors += 1 1027 if pass_cnt > obj_cnt: pass_cnt = obj_cnt 1028 1029 for i in range(pass_cnt): 1030 line, comment = pass_comments[i] 1031 if obj_list[i] != comment: 1032 ErrOut.LogLine(filename, line, None, "OBJ %s : EXPECTED %s\n" % 1033 (obj_list[i], comment)) 1034 errors += 1 1035 1036 # 1037 # Check for expected errors 1038 # 1039 err_list = ErrOut.DrainLog() 1040 err_cnt = len(err_list) 1041 fail_cnt = len(fail_comments) 1042 if err_cnt != fail_cnt: 1043 InfoOut.Log("Mismatched fail (%d) vs. errors seen (%d)." 1044 % (fail_cnt, err_cnt)) 1045 InfoOut.Log("FAIL: %s" % [x[1] for x in fail_comments]) 1046 InfoOut.Log("ERRS: %s" % err_list) 1047 errors += 1 1048 if fail_cnt > err_cnt: fail_cnt = err_cnt 1049 1050 for i in range(fail_cnt): 1051 line, comment = fail_comments[i] 1052 err = err_list[i].strip() 1053 1054 if err_list[i] != comment: 1055 ErrOut.Log("%s(%d) Error\n\tERROR : %s\n\tEXPECT: %s" % ( 1056 filename, line, err_list[i], comment)) 1057 errors += 1 1058 1059 # Clear the error list for the next run 1060 err_list = [] 1061 return errors 1062 1063 1064def TestFile(parser, filename): 1065 # Capture errors instead of reporting them so we can compare them 1066 # with the expected errors. 1067 ErrOut.SetConsole(False) 1068 ErrOut.SetCapture(True) 1069 1070 filenode = parser.ParseFile(filename) 1071 1072 # Renable output 1073 ErrOut.SetConsole(True) 1074 ErrOut.SetCapture(False) 1075 1076 # Compare captured errors 1077 return TestErrors(filename, filenode) 1078 1079 1080def TestErrorFiles(filter): 1081 idldir = os.path.split(sys.argv[0])[0] 1082 idldir = os.path.join(idldir, 'test_parser', '*.idl') 1083 filenames = glob.glob(idldir) 1084 parser = IDLParser() 1085 total_errs = 0 1086 for filename in filenames: 1087 if filter and filename not in filter: continue 1088 errs = TestFile(parser, filename) 1089 if errs: 1090 ErrOut.Log("%s test failed with %d error(s)." % (filename, errs)) 1091 total_errs += errs 1092 1093 if total_errs: 1094 ErrOut.Log("Failed parsing test.") 1095 else: 1096 InfoOut.Log("Passed parsing test.") 1097 return total_errs 1098 1099 1100def TestNamespaceFiles(filter): 1101 idldir = os.path.split(sys.argv[0])[0] 1102 idldir = os.path.join(idldir, 'test_namespace', '*.idl') 1103 filenames = glob.glob(idldir) 1104 testnames = [] 1105 1106 for filename in filenames: 1107 if filter and filename not in filter: continue 1108 testnames.append(filename) 1109 1110 # If we have no files to test, then skip this test 1111 if not testnames: 1112 InfoOut.Log('No files to test for namespace.') 1113 return 0 1114 1115 InfoOut.SetConsole(False) 1116 ast = ParseFiles(testnames) 1117 InfoOut.SetConsole(True) 1118 1119 errs = ast.GetProperty('ERRORS') 1120 if errs: 1121 ErrOut.Log("Failed namespace test.") 1122 else: 1123 InfoOut.Log("Passed namespace test.") 1124 return errs 1125 1126 1127 1128def FindVersionError(releases, node): 1129 err_cnt = 0 1130 if node.IsA('Interface', 'Struct'): 1131 comment_list = [] 1132 comment = node.GetOneOf('Comment') 1133 if comment and comment.GetName()[:4] == 'REL:': 1134 comment_list = comment.GetName()[5:].strip().split(' ') 1135 1136 first_list = [node.first_release[rel] for rel in releases] 1137 first_list = sorted(set(first_list)) 1138 if first_list != comment_list: 1139 node.Error("Mismatch in releases: %s vs %s." % ( 1140 comment_list, first_list)) 1141 err_cnt += 1 1142 1143 for child in node.GetChildren(): 1144 err_cnt += FindVersionError(releases, child) 1145 return err_cnt 1146 1147 1148def TestVersionFiles(filter): 1149 idldir = os.path.split(sys.argv[0])[0] 1150 idldir = os.path.join(idldir, 'test_version', '*.idl') 1151 filenames = glob.glob(idldir) 1152 testnames = [] 1153 1154 for filename in filenames: 1155 if filter and filename not in filter: continue 1156 testnames.append(filename) 1157 1158 # If we have no files to test, then skip this test 1159 if not testnames: 1160 InfoOut.Log('No files to test for version.') 1161 return 0 1162 1163 ast = ParseFiles(testnames) 1164 errs = FindVersionError(ast.releases, ast) 1165 1166 if errs: 1167 ErrOut.Log("Failed version test.") 1168 else: 1169 InfoOut.Log("Passed version test.") 1170 return errs 1171 1172 1173default_dirs = ['.', 'trusted', 'dev', 'private', 'extensions', 1174 'extensions/dev'] 1175def ParseFiles(filenames): 1176 parser = IDLParser() 1177 filenodes = [] 1178 1179 if not filenames: 1180 filenames = [] 1181 srcroot = GetOption('srcroot') 1182 dirs = default_dirs 1183 if GetOption('include_private'): 1184 dirs += ['private'] 1185 for dirname in dirs: 1186 srcdir = os.path.join(srcroot, dirname, '*.idl') 1187 srcdir = os.path.normpath(srcdir) 1188 filenames += sorted(glob.glob(srcdir)) 1189 1190 if not filenames: 1191 ErrOut.Log('No sources provided.') 1192 1193 for filename in filenames: 1194 filenode = parser.ParseFile(filename) 1195 filenodes.append(filenode) 1196 1197 ast = IDLAst(filenodes) 1198 if GetOption('dump_tree'): ast.Dump(0) 1199 1200 Lint(ast) 1201 return ast 1202 1203 1204def Main(args): 1205 filenames = ParseOptions(args) 1206 1207 # If testing... 1208 if GetOption('test'): 1209 errs = TestErrorFiles(filenames) 1210 errs = TestNamespaceFiles(filenames) 1211 errs = TestVersionFiles(filenames) 1212 if errs: 1213 ErrOut.Log("Parser failed with %d errors." % errs) 1214 return -1 1215 return 0 1216 1217 # Otherwise, build the AST 1218 ast = ParseFiles(filenames) 1219 errs = ast.GetProperty('ERRORS') 1220 if errs: 1221 ErrOut.Log('Found %d error(s).' % errs); 1222 InfoOut.Log("%d files processed." % len(filenames)) 1223 return errs 1224 1225 1226if __name__ == '__main__': 1227 sys.exit(Main(sys.argv[1:])) 1228 1229