11dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project# a glorified C pre-processor parser 21dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 31dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Projectimport sys, re, string 41dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Projectfrom utils import * 51dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Projectfrom defaults import * 61dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 71dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source ProjectdebugTokens = False 81dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source ProjectdebugDirectiveTokenizer = False 91dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source ProjectdebugLineParsing = False 101dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source ProjectdebugCppExpr = False 111dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source ProjectdebugOptimIf01 = False 121dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 131dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project##################################################################################### 141dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project##################################################################################### 151dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project##### ##### 161dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project##### C P P T O K E N S ##### 171dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project##### ##### 181dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project##################################################################################### 191dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project##################################################################################### 201dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 211dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project# the list of supported C-preprocessor tokens 221dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project# plus a couple of C tokens as well 231dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source ProjecttokEOF = "\0" 241dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source ProjecttokLN = "\n" 251dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source ProjecttokSTRINGIFY = "#" 261dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source ProjecttokCONCAT = "##" 271dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source ProjecttokLOGICAND = "&&" 281dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source ProjecttokLOGICOR = "||" 291dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source ProjecttokSHL = "<<" 301dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source ProjecttokSHR = ">>" 311dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source ProjecttokEQUAL = "==" 321dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source ProjecttokNEQUAL = "!=" 331dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source ProjecttokLT = "<" 341dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source ProjecttokLTE = "<=" 351dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source ProjecttokGT = ">" 361dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source ProjecttokGTE = ">=" 371dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source ProjecttokELLIPSIS = "..." 381dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source ProjecttokSPACE = " " 391dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source ProjecttokDEFINED = "defined" 401dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source ProjecttokLPAREN = "(" 411dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source ProjecttokRPAREN = ")" 421dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source ProjecttokNOT = "!" 431dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source ProjecttokPLUS = "+" 441dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source ProjecttokMINUS = "-" 451dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source ProjecttokMULTIPLY = "*" 461dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source ProjecttokDIVIDE = "/" 471dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source ProjecttokMODULUS = "%" 481dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source ProjecttokBINAND = "&" 491dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source ProjecttokBINOR = "|" 501dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source ProjecttokBINXOR = "^" 511dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source ProjecttokCOMMA = "," 521dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source ProjecttokLBRACE = "{" 531dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source ProjecttokRBRACE = "}" 541dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source ProjecttokARROW = "->" 551dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source ProjecttokINCREMENT = "++" 561dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source ProjecttokDECREMENT = "--" 571dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source ProjecttokNUMBER = "<number>" 581dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source ProjecttokIDENT = "<ident>" 591dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source ProjecttokSTRING = "<string>" 601dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 611dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Projectclass Token: 621dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project """a simple class to hold information about a given token. 631dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project each token has a position in the source code, as well as 641dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project an 'id' and a 'value'. the id is a string that identifies 651dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project the token's class, while the value is the string of the 661dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project original token itself. 671dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 681dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project for example, the tokenizer concatenates a series of spaces 691dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project and tabs as a single tokSPACE id, whose value if the original 701dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project spaces+tabs sequence.""" 711dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 721dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project def __init__(self): 731dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project self.id = None 741dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project self.value = None 751dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project self.lineno = 0 761dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project self.colno = 0 771dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 781dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project def set(self,id,val=None): 791dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project self.id = id 801dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project if val: 811dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project self.value = val 821dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project else: 831dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project self.value = id 841dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project return None 851dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 861dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project def copyFrom(self,src): 871dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project self.id = src.id 881dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project self.value = src.value 891dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project self.lineno = src.lineno 901dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project self.colno = src.colno 911dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 921dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project def __repr__(self): 931dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project if self.id == tokIDENT: 941dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project return "(ident %s)" % self.value 951dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project if self.id == tokNUMBER: 961dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project return "(number %s)" % self.value 971dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project if self.id == tokSTRING: 981dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project return "(string '%s')" % self.value 991dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project if self.id == tokLN: 1001dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project return "<LN>" 1011dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project if self.id == tokEOF: 1021dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project return "<EOF>" 1031dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project if self.id == tokSPACE and self.value == "\\": 1041dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project # this corresponds to a trailing \ that was transformed into a tokSPACE 1051dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project return "<\\>" 1061dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 1071dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project return self.id 1081dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 1091dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project def __str__(self): 1101dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project if self.id == tokIDENT: 1111dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project return self.value 1121dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project if self.id == tokNUMBER: 1131dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project return self.value 1141dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project if self.id == tokSTRING: 1151dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project return self.value 1161dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project if self.id == tokEOF: 1171dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project return "<EOF>" 1181dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project if self.id == tokSPACE: 1191dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project if self.value == "\\": # trailing \ 1201dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project return "\\\n" 1211dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project else: 1221dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project return self.value 1231dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 1241dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project return self.id 1251dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 1261dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Projectclass BadExpectedToken(Exception): 1271dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project def __init__(self,msg): 1281dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project print msg 1291dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 1301dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 1311dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project##################################################################################### 1321dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project##################################################################################### 1331dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project##### ##### 1341dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project##### C P P T O K E N I Z E R ##### 1351dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project##### ##### 1361dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project##################################################################################### 1371dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project##################################################################################### 1381dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 1391dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project# list of long symbols, i.e. those that take more than one characters 1401dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source ProjectcppLongSymbols = [ tokCONCAT, tokLOGICAND, tokLOGICOR, tokSHL, tokSHR, tokELLIPSIS, tokEQUAL,\ 1411dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project tokNEQUAL, tokLTE, tokGTE, tokARROW, tokINCREMENT, tokDECREMENT ] 1421dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 1431dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Projectclass CppTokenizer: 1441dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project """an abstract class used to convert some input text into a list 1451dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project of tokens. real implementations follow and differ in the format 1461dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project of the input text only""" 1471dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 1481dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project def __init__(self): 1491dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project """initialize a new CppTokenizer object""" 1501dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project self.eof = False # end of file reached ? 1511dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project self.text = None # content of current line, with final \n stripped 1521dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project self.line = 0 # number of current line 1531dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project self.pos = 0 # current character position in current line 1541dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project self.len = 0 # length of current line text 1551dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project self.held = Token() 1561dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 1571dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project def setLineText(self,line): 1581dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project """set the content of the (next) current line. should be called 1591dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project by fillLineText() in derived classes""" 1601dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project self.text = line 1611dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project self.len = len(line) 1621dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project self.pos = 0 1631dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 1641dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project def fillLineText(self): 1651dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project """refresh the content of 'line' with a new line of input""" 1661dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project # to be overriden 1671dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project self.eof = True 1681dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 1691dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project def markPos(self,tok): 1701dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project """mark the position of the current token in the source file""" 1711dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project if self.eof or self.pos > self.len: 1721dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project tok.lineno = self.line + 1 1731dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project tok.colno = 0 1741dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project else: 1751dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project tok.lineno = self.line 1761dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project tok.colno = self.pos 1771dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 1781dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project def peekChar(self): 1791dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project """return the current token under the cursor without moving it""" 1801dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project if self.eof: 1811dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project return tokEOF 1821dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 1831dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project if self.pos > self.len: 1841dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project self.pos = 0 1851dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project self.line += 1 1861dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project self.fillLineText() 1871dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project if self.eof: 1881dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project return tokEOF 1891dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 1901dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project if self.pos == self.len: 1911dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project return tokLN 1921dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project else: 1931dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project return self.text[self.pos] 1941dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 1951dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project def peekNChar(self,n): 1961dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project """try to peek the next n chars on the same line""" 1971dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project if self.pos + n > self.len: 1981dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project return None 1991dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project return self.text[self.pos:self.pos+n] 2001dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 2011dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project def skipChar(self): 2021dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project """increment the token cursor position""" 2031dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project if not self.eof: 2041dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project self.pos += 1 2051dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 2061dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project def skipNChars(self,n): 2071dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project if self.pos + n <= self.len: 2081dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project self.pos += n 2091dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project else: 2101dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project while n > 0: 2111dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project self.skipChar() 2121dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project n -= 1 2131dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 2141dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project def nextChar(self): 2151dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project """retrieve the token at the current cursor position, then skip it""" 2161dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project result = self.peekChar() 2171dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project self.skipChar() 2181dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project return result 2191dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 2201dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project def getEscape(self): 2211dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project # try to get all characters after a backslash (\) 2221dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project result = self.nextChar() 2231dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project if result == "0": 2241dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project # octal number ? 2251dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project num = self.peekNChar(3) 2261dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project if num != None: 2271dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project isOctal = True 2281dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project for d in num: 2291dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project if not d in "01234567": 2301dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project isOctal = False 2311dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project break 2321dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project if isOctal: 2331dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project result += num 2341dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project self.skipNChars(3) 2351dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project elif result == "x" or result == "X": 2361dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project # hex number ? 2371dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project num = self.peekNChar(2) 2381dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project if num != None: 2391dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project isHex = True 2401dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project for d in num: 2411dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project if not d in "012345678abcdefABCDEF": 2421dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project isHex = False 2431dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project break 2441dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project if isHex: 2451dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project result += num 2461dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project self.skipNChars(2) 2471dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project elif result == "u" or result == "U": 2481dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project # unicode char ? 2491dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project num = self.peekNChar(4) 2501dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project if num != None: 2511dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project isHex = True 2521dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project for d in num: 2531dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project if not d in "012345678abcdefABCDEF": 2541dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project isHex = False 2551dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project break 2561dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project if isHex: 2571dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project result += num 2581dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project self.skipNChars(4) 2591dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 2601dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project return result 2611dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 2621dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project def nextRealToken(self,tok): 2631dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project """return next CPP token, used internally by nextToken()""" 2641dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project c = self.nextChar() 2651dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project if c == tokEOF or c == tokLN: 2661dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project return tok.set(c) 2671dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 2681dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project if c == '/': 2691dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project c = self.peekChar() 2701dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project if c == '/': # C++ comment line 2711dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project self.skipChar() 2721dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project while 1: 2731dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project c = self.nextChar() 2741dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project if c == tokEOF or c == tokLN: 2751dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project break 2761dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project return tok.set(tokLN) 2771dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project if c == '*': # C comment start 2781dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project self.skipChar() 2791dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project value = "/*" 2801dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project prev_c = None 2811dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project while 1: 2821dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project c = self.nextChar() 2831dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project if c == tokEOF: 2841dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project return tok.set(tokEOF,value) 2851dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project if c == '/' and prev_c == '*': 2861dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project break 2871dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project prev_c = c 2881dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project value += c 2891dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 2901dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project value += "/" 2911dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project return tok.set(tokSPACE,value) 2921dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project c = '/' 2931dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 2941dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project if c.isspace(): 2951dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project while 1: 2961dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project c2 = self.peekChar() 2971dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project if c2 == tokLN or not c2.isspace(): 2981dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project break 2991dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project c += c2 3001dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project self.skipChar() 3011dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project return tok.set(tokSPACE,c) 3021dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 3031dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project if c == '\\': 3041dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project if debugTokens: 3051dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project print "nextRealToken: \\ found, next token is '%s'" % repr(self.peekChar()) 3061dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project if self.peekChar() == tokLN: # trailing \ 3071dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project # eat the tokLN 3081dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project self.skipChar() 3091dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project # we replace a trailing \ by a tokSPACE whose value is 3101dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project # simply "\\". this allows us to detect them later when 3111dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project # needed. 3121dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project return tok.set(tokSPACE,"\\") 3131dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project else: 3141dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project # treat as a single token here ? 3151dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project c +=self.getEscape() 3161dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project return tok.set(c) 3171dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 3181dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project if c == "'": # chars 3191dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project c2 = self.nextChar() 3201dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project c += c2 3211dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project if c2 == '\\': 3221dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project c += self.getEscape() 3231dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 3241dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project while 1: 3251dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project c2 = self.nextChar() 3261dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project if c2 == tokEOF: 3271dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project break 3281dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project c += c2 3291dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project if c2 == "'": 3301dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project break 3311dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 3321dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project return tok.set(tokSTRING, c) 3331dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 3341dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project if c == '"': # strings 3351dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project quote = 0 3361dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project while 1: 3371dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project c2 = self.nextChar() 3381dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project if c2 == tokEOF: 3391dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project return tok.set(tokSTRING,c) 3401dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 3411dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project c += c2 3421dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project if not quote: 3431dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project if c2 == '"': 3441dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project return tok.set(tokSTRING,c) 3451dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project if c2 == "\\": 3461dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project quote = 1 3471dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project else: 3481dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project quote = 0 3491dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 3501dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project if c >= "0" and c <= "9": # integers ? 3511dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project while 1: 3521dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project c2 = self.peekChar() 3531dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project if c2 == tokLN or (not c2.isalnum() and c2 != "_"): 3541dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project break 3551dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project c += c2 3561dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project self.skipChar() 3571dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project return tok.set(tokNUMBER,c) 3581dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 3591dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project if c.isalnum() or c == "_": # identifiers ? 3601dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project while 1: 3611dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project c2 = self.peekChar() 3621dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project if c2 == tokLN or (not c2.isalnum() and c2 != "_"): 3631dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project break 3641dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project c += c2 3651dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project self.skipChar() 3661dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project if c == tokDEFINED: 3671dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project return tok.set(tokDEFINED) 3681dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project else: 3691dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project return tok.set(tokIDENT,c) 3701dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 3711dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project # check special symbols 3721dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project for sk in cppLongSymbols: 3731dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project if c == sk[0]: 3741dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project sklen = len(sk[1:]) 3751dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project if self.pos + sklen <= self.len and \ 3761dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project self.text[self.pos:self.pos+sklen] == sk[1:]: 3771dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project self.pos += sklen 3781dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project return tok.set(sk) 3791dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 3801dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project return tok.set(c) 3811dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 3821dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project def nextToken(self,tok): 3831dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project """return the next token from the input text. this function 3841dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project really updates 'tok', and does not return a new one""" 3851dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project self.markPos(tok) 3861dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project self.nextRealToken(tok) 3871dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 3881dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project def getToken(self): 3891dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project tok = Token() 3901dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project self.nextToken(tok) 3911dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project if debugTokens: 3921dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project print "getTokens: %s" % repr(tok) 3931dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project return tok 3941dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 3951dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project def toTokenList(self): 3961dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project """convert the input text of a CppTokenizer into a direct 3971dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project list of token objects. tokEOF is stripped from the result""" 3981dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project result = [] 3991dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project while 1: 4001dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project tok = Token() 4011dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project self.nextToken(tok) 4021dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project if tok.id == tokEOF: 4031dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project break 4041dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project result.append(tok) 4051dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project return result 4061dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 4071dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Projectclass CppLineTokenizer(CppTokenizer): 4081dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project """a CppTokenizer derived class that accepts a single line of text as input""" 4091dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project def __init__(self,line,lineno=1): 4101dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project CppTokenizer.__init__(self) 4111dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project self.line = lineno 4121dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project self.setLineText(line) 4131dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 4141dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 4151dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Projectclass CppLinesTokenizer(CppTokenizer): 4161dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project """a CppTokenizer derived class that accepts a list of texdt lines as input. 4171dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project the lines must not have a trailing \n""" 4181dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project def __init__(self,lines=[],lineno=1): 4191dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project """initialize a CppLinesTokenizer. you can later add lines using addLines()""" 4201dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project CppTokenizer.__init__(self) 4211dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project self.line = lineno 4221dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project self.lines = lines 4231dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project self.index = 0 4241dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project self.count = len(lines) 4251dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 4261dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project if self.count > 0: 4271dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project self.fillLineText() 4281dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project else: 4291dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project self.eof = True 4301dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 4311dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project def addLine(self,line): 4321dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project """add a line to a CppLinesTokenizer. this can be done after tokenization 4331dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project happens""" 4341dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project if self.count == 0: 4351dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project self.setLineText(line) 4361dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project self.index = 1 4371dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project self.lines.append(line) 4381dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project self.count += 1 4391dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project self.eof = False 4401dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 4411dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project def fillLineText(self): 4421dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project if self.index < self.count: 4431dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project self.setLineText(self.lines[self.index]) 4441dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project self.index += 1 4451dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project else: 4461dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project self.eof = True 4471dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 4481dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 4491dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Projectclass CppFileTokenizer(CppTokenizer): 4501dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project def __init__(self,file,lineno=1): 4511dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project CppTokenizer.__init__(self) 4521dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project self.file = file 4531dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project self.line = lineno 4541dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 4551dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project def fillLineText(self): 4561dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project line = self.file.readline() 4571dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project if len(line) > 0: 4581dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project if line[-1] == '\n': 4591dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project line = line[:-1] 4601dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project if len(line) > 0 and line[-1] == "\r": 4611dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project line = line[:-1] 4621dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project self.setLineText(line) 4631dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project else: 4641dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project self.eof = True 4651dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 4661dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project# Unit testing 4671dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project# 4681dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Projectclass CppTokenizerTester: 4691dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project """a class used to test CppTokenizer classes""" 4701dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project def __init__(self,tokenizer=None): 4711dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project self.tokenizer = tokenizer 4721dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project self.token = Token() 4731dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 4741dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project def setTokenizer(self,tokenizer): 4751dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project self.tokenizer = tokenizer 4761dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 4771dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project def expect(self,id): 4781dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project self.tokenizer.nextToken(self.token) 4791dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project tokid = self.token.id 4801dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project if tokid == id: 4811dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project return 4821dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project if self.token.value == id and (tokid == tokIDENT or tokid == tokNUMBER): 4831dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project return 4841dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project raise BadExpectedToken, "### BAD TOKEN: '%s' expecting '%s'" % (self.token.id,id) 4851dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 4861dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project def expectToken(self,id,line,col): 4871dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project self.expect(id) 4881dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project if self.token.lineno != line: 4891dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project raise BadExpectedToken, "### BAD LINENO: token '%s' got '%d' expecting '%d'" % (id,self.token.lineno,line) 4901dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project if self.token.colno != col: 4911dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project raise BadExpectedToken, "### BAD COLNO: '%d' expecting '%d'" % (self.token.colno,col) 4921dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 4931dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project def expectTokenVal(self,id,value,line,col): 4941dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project self.expectToken(id,line,col) 4951dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project if self.token.value != value: 4961dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project raise BadExpectedToken, "### BAD VALUE: '%s' expecting '%s'" % (self.token.value,value) 4971dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 4981dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project def expectList(self,list): 4991dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project for item in list: 5001dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project self.expect(item) 5011dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 5021dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Projectdef test_CppTokenizer(): 5031dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project tester = CppTokenizerTester() 5041dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 5051dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project tester.setTokenizer( CppLineTokenizer("#an/example && (01923_xy)") ) 5061dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project tester.expectList( ["#", "an", "/", "example", tokSPACE, tokLOGICAND, tokSPACE, tokLPAREN, "01923_xy", \ 5071dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project tokRPAREN, tokLN, tokEOF] ) 5081dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 5091dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project tester.setTokenizer( CppLineTokenizer("FOO(BAR) && defined(BAZ)") ) 5101dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project tester.expectList( ["FOO", tokLPAREN, "BAR", tokRPAREN, tokSPACE, tokLOGICAND, tokSPACE, 5111dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project tokDEFINED, tokLPAREN, "BAZ", tokRPAREN, tokLN, tokEOF] ) 5121dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 5131dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project tester.setTokenizer( CppLinesTokenizer( ["/*", "#", "*/"] ) ) 5141dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project tester.expectList( [ tokSPACE, tokLN, tokEOF ] ) 5151dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 5161dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project tester.setTokenizer( CppLinesTokenizer( ["first", "second"] ) ) 5171dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project tester.expectList( [ "first", tokLN, "second", tokLN, tokEOF ] ) 5181dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 5191dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project tester.setTokenizer( CppLinesTokenizer( ["first second", " third"] ) ) 5201dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project tester.expectToken( "first", 1, 0 ) 5211dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project tester.expectToken( tokSPACE, 1, 5 ) 5221dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project tester.expectToken( "second", 1, 6 ) 5231dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project tester.expectToken( tokLN, 1, 12 ) 5241dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project tester.expectToken( tokSPACE, 2, 0 ) 5251dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project tester.expectToken( "third", 2, 2 ) 5261dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 5271dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project tester.setTokenizer( CppLinesTokenizer( [ "boo /* what the", "hell */" ] ) ) 5281dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project tester.expectList( [ "boo", tokSPACE ] ) 5291dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project tester.expectTokenVal( tokSPACE, "/* what the\nhell */", 1, 4 ) 5301dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project tester.expectList( [ tokLN, tokEOF ] ) 5311dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 5321dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project tester.setTokenizer( CppLinesTokenizer( [ "an \\", " example" ] ) ) 5331dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project tester.expectToken( "an", 1, 0 ) 5341dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project tester.expectToken( tokSPACE, 1, 2 ) 5351dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project tester.expectTokenVal( tokSPACE, "\\", 1, 3 ) 5361dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project tester.expectToken( tokSPACE, 2, 0 ) 5371dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project tester.expectToken( "example", 2, 1 ) 5381dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project tester.expectToken( tokLN, 2, 8 ) 5391dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 5401dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project return True 5411dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 5421dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 5431dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project##################################################################################### 5441dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project##################################################################################### 5451dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project##### ##### 5461dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project##### C P P E X P R E S S I O N S ##### 5471dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project##### ##### 5481dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project##################################################################################### 5491dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project##################################################################################### 5501dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 5511dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Projectclass CppExpr: 5521dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project """a class that models the condition of #if directives into 5531dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project an expression tree. each node in the tree is of the form (op,arg) or (op,arg1,arg2) 5541dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project where "op" is a string describing the operation""" 5551dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 5561dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project unaries = [ "!", "~" ] 5571198fd38645ff94bf48daae10f8b74903444badcElliott Hughes binaries = [ "+", "-", "<", "<=", ">=", ">", "&&", "||", "*", "/", "%", "&", "|", "^", "<<", ">>", "==", "!=", "?", ":" ] 5581198fd38645ff94bf48daae10f8b74903444badcElliott Hughes precedences = { 5591198fd38645ff94bf48daae10f8b74903444badcElliott Hughes "?": 1, ":": 1, 5601198fd38645ff94bf48daae10f8b74903444badcElliott Hughes "||": 2, 5611198fd38645ff94bf48daae10f8b74903444badcElliott Hughes "&&": 3, 5621198fd38645ff94bf48daae10f8b74903444badcElliott Hughes "|": 4, 5631198fd38645ff94bf48daae10f8b74903444badcElliott Hughes "^": 5, 5641198fd38645ff94bf48daae10f8b74903444badcElliott Hughes "&": 6, 5651198fd38645ff94bf48daae10f8b74903444badcElliott Hughes "==": 7, "!=": 7, 5661198fd38645ff94bf48daae10f8b74903444badcElliott Hughes "<": 8, "<=": 8, ">": 8, ">=": 8, 5671198fd38645ff94bf48daae10f8b74903444badcElliott Hughes "<<": 9, ">>": 9, 5681198fd38645ff94bf48daae10f8b74903444badcElliott Hughes "+": 10, "-": 10, 5691198fd38645ff94bf48daae10f8b74903444badcElliott Hughes "*": 11, "/": 11, "%": 11, 5701198fd38645ff94bf48daae10f8b74903444badcElliott Hughes "!": 12, "~": 12 5711198fd38645ff94bf48daae10f8b74903444badcElliott Hughes } 5721dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 57340596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes re_cpp_constant = re.compile(r"((\d|\w|_)+)") 57440596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes 5751dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project def __init__(self, tokens): 5761dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project """initialize a CppExpr. 'tokens' must be a CppToken list""" 5771dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project self.tok = tokens 5781dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project self.n = len(tokens) 57940596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes self.i = 0 5801dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project if debugCppExpr: 5811dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project print "CppExpr: trying to parse %s" % repr(tokens) 58240596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes self.expr = self.parseExpression(0) 5831dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project if debugCppExpr: 58440596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes print "CppExpr: got " + repr(self.expr) 58540596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes if self.i != self.n: 5861198fd38645ff94bf48daae10f8b74903444badcElliott Hughes print 'crap at end of input (%d != %d): %s' % (self.i, self.n, repr(tokens)) 58740596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes raise 5881dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 5891dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 59040596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes def throw(self, exception, msg): 59140596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes if self.i < self.n: 59240596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes tok = self.tok[self.i] 5931dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project print "%d:%d: %s" % (tok.lineno,tok.colno,msg) 5941dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project else: 5951dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project print "EOF: %s" % msg 59640596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes raise exception(msg) 5971dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 59840596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes 59940596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes def skip_spaces(self): 6001dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project """skip spaces in input token list""" 60140596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes while self.i < self.n: 60240596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes t = self.tok[self.i] 6031dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project if t.id != tokSPACE and t.id != tokLN: 6041dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project break 60540596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes self.i += 1 6061dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 60740596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes 60840596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes def expectId(self, id): 6091dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project """check that a given token id is at the current position, then skip over it""" 61040596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes self.skip_spaces() 61140596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes if self.i >= self.n or self.tok[self.i].id != id: 61240596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes self.throw(BadExpectedToken,self.i,"### expecting '%s' in expression, got '%s'" % (id, self.tok[self.i].id)) 61340596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes self.i += 1 61440596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes 61540596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes 61640596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes def expectIdent(self): 61740596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes self.skip_spaces() 61840596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes if self.i >= self.n or self.tok[self.i].id != tokIDENT: 61940596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes self.throw(BadExpectedToken, self.i,"### expecting identifier in expression, got '%s'" % (id, self.tok[self.i].id)) 62040596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes self.i += 1 62140596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes 62240596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes 62340596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes def is_decimal(self): 62440596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes v = self.tok[self.i].value[:] 6251dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project while len(v) > 0 and v[-1] in "ULul": 6261dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project v = v[:-1] 6271dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project for digit in v: 6281dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project if not digit.isdigit(): 6291dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project return None 6301dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 63140596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes self.i += 1 63240596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes return ("int", string.atoi(v)) 6331dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 63440596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes 63540596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes def is_hexadecimal(self): 63640596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes v = self.tok[self.i].value[:] 6371dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project while len(v) > 0 and v[-1] in "ULul": 6381dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project v = v[:-1] 6391dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project if len(v) > 2 and (v[0:2] == "0x" or v[0:2] == "0X"): 6401dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project for digit in v[2:]: 6411dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project if not digit in "0123456789abcdefABCDEF": 6421dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project return None 6431dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 64440596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes # for a hex expression tuple, the argument 6451dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project # is the value as an integer 64640596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes self.i += 1 64740596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes return ("hex", int(v[2:], 16)) 6481dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 6491dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project return None 6501dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 65140596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes 65240596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes def is_integer(self): 65340596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes if self.tok[self.i].id != tokNUMBER: 6541dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project return None 6551dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 65640596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes c = self.is_decimal() 6571dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project if c: return c 6581dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 65940596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes c = self.is_hexadecimal() 6601dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project if c: return c 6611dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 6621dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project return None 6631dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 66440596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes 66540596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes def is_number(self): 66640596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes t = self.tok[self.i] 66740596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes if t.id == tokMINUS and self.i+1 < self.n: 66840596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes self.i += 1 66940596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes c = self.is_integer() 6701dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project if c: 67140596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes op, val = c 67240596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes return (op, -val) 67340596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes if t.id == tokPLUS and self.i+1 < self.n: 67440596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes c = self.is_integer() 6751dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project if c: return c 6761dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 67740596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes return self.is_integer() 6781dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 6791dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 68040596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes def is_defined(self): 68140596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes t = self.tok[self.i] 6821dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project if t.id != tokDEFINED: 6831dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project return None 6841dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 6851dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project # we have the defined keyword, check the rest 68640596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes self.i += 1 68740596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes self.skip_spaces() 688fddbafdc0a63a1b4c5e9252719a913197a4eecfaElliott Hughes used_parens = 0 68940596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes if self.i < self.n and self.tok[self.i].id == tokLPAREN: 690fddbafdc0a63a1b4c5e9252719a913197a4eecfaElliott Hughes used_parens = 1 69140596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes self.i += 1 69240596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes self.skip_spaces() 6931dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 69440596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes if self.i >= self.n: 6951dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project self.throw(CppConstantExpected,i,"### 'defined' must be followed by macro name or left paren") 6961dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 69740596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes t = self.tok[self.i] 6981dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project if t.id != tokIDENT: 6991dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project self.throw(CppConstantExpected,i,"### 'defined' must be followed by macro name") 7001dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 70140596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes self.i += 1 702fddbafdc0a63a1b4c5e9252719a913197a4eecfaElliott Hughes if used_parens: 70340596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes self.expectId(tokRPAREN) 7041dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 70540596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes return ("defined", t.value) 7061dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 7071dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 70840596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes def is_call_or_ident(self): 70940596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes self.skip_spaces() 71040596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes if self.i >= self.n: 7111dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project return None 7121dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 71340596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes t = self.tok[self.i] 7141dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project if t.id != tokIDENT: 7151dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project return None 7161dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 7171dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project name = t.value 7181dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 71940596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes self.i += 1 72040596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes self.skip_spaces() 72140596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes if self.i >= self.n or self.tok[self.i].id != tokLPAREN: 72240596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes return ("ident", name) 7231dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 7241dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project params = [] 7251dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project depth = 1 72640596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes self.i += 1 72740596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes j = self.i 72840596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes while self.i < self.n: 72940596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes id = self.tok[self.i].id 7301dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project if id == tokLPAREN: 7311dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project depth += 1 7321dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project elif depth == 1 and (id == tokCOMMA or id == tokRPAREN): 73340596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes while j < self.i and self.tok[j].id == tokSPACE: 7341dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project j += 1 73540596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes k = self.i 7361dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project while k > j and self.tok[k-1].id == tokSPACE: 7371dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project k -= 1 7381dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project param = self.tok[j:k] 73940596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes params.append(param) 7401dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project if id == tokRPAREN: 7411dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project break 74240596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes j = self.i+1 7431dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project elif id == tokRPAREN: 7441dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project depth -= 1 74540596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes self.i += 1 7461dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 74740596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes if self.i >= self.n: 7481dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project return None 7491dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 75040596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes self.i += 1 75140596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes return ("call", (name, params)) 7521dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 7531dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 75440596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes # Implements the "precedence climbing" algorithm from http://www.engr.mun.ca/~theo/Misc/exp_parsing.htm. 75540596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes # The "classic" algorithm would be fine if we were using a tool to generate the parser, but we're not. 75640596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes # Dijkstra's "shunting yard" algorithm hasn't been necessary yet. 75740596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes def parseExpression(self, minPrecedence): 75840596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes self.skip_spaces() 75940596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes if self.i >= self.n: 76040596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes return None 7611dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 76240596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes node = self.parsePrimary() 76340596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes while self.token() != None and self.isBinary(self.token()) and self.precedence(self.token()) >= minPrecedence: 76440596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes op = self.token() 76540596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes self.nextToken() 76640596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes rhs = self.parseExpression(self.precedence(op) + 1) 76740596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes node = (op.id, node, rhs) 76840596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes 76940596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes return node 77040596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes 77140596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes 77240596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes def parsePrimary(self): 77340596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes op = self.token() 77440596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes if self.isUnary(op): 77540596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes self.nextToken() 77640596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes return (op.id, self.parseExpression(self.precedence(op))) 77740596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes 77840596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes primary = None 77940596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes if op.id == tokLPAREN: 78040596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes self.nextToken() 78140596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes primary = self.parseExpression(0) 78240596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes self.expectId(tokRPAREN) 7831198fd38645ff94bf48daae10f8b74903444badcElliott Hughes elif op.id == "?": 7841198fd38645ff94bf48daae10f8b74903444badcElliott Hughes self.nextToken() 7851198fd38645ff94bf48daae10f8b74903444badcElliott Hughes primary = self.parseExpression(0) 7861198fd38645ff94bf48daae10f8b74903444badcElliott Hughes self.expectId(":") 78740596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes elif op.id == tokNUMBER: 78840596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes primary = self.is_number() 78940596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes elif op.id == tokIDENT: 79040596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes primary = self.is_call_or_ident() 79140596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes elif op.id == tokDEFINED: 79240596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes primary = self.is_defined() 79340596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes else: 79440596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes self.throw(BadExpectedToken, "didn't expect to see a %s in factor" % (self.tok[self.i].id)) 7951dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 79640596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes self.skip_spaces() 7971dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 79840596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes return primary; 7991dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 8001dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 80140596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes def isBinary(self, token): 80240596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes return token.id in self.binaries 8031dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 8041dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 80540596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes def isUnary(self, token): 80640596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes return token.id in self.unaries 8071dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 8081dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 80940596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes def precedence(self, token): 81040596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes return self.precedences.get(token.id) 8111dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 8121dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 81340596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes def token(self): 81440596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes if self.i >= self.n: 8151dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project return None 81640596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes return self.tok[self.i] 8171dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 8181dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 81940596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes def nextToken(self): 82040596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes self.i += 1 82140596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes self.skip_spaces() 82240596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes if self.i >= self.n: 82340596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes return None 82440596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes return self.tok[self.i] 8251dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 8261dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 82740596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes def dump_node(self, e): 8281dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project op = e[0] 8291dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project line = "(" + op 8301dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project if op == "int": 8311dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project line += " %d)" % e[1] 8321dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project elif op == "hex": 8331dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project line += " 0x%x)" % e[1] 8341dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project elif op == "ident": 8351dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project line += " %s)" % e[1] 8361dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project elif op == "defined": 8371dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project line += " %s)" % e[1] 8381dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project elif op == "call": 8391dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project arg = e[1] 8401dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project line += " %s [" % arg[0] 8411dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project prefix = "" 8421dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project for param in arg[1]: 8431dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project par = "" 8441dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project for tok in param: 8451dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project par += str(tok) 8461dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project line += "%s%s" % (prefix, par) 8471dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project prefix = "," 8481dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project line += "])" 8491dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project elif op in CppExpr.unaries: 8501dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project line += " %s)" % self.dump_node(e[1]) 8511dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project elif op in CppExpr.binaries: 8521dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project line += " %s %s)" % (self.dump_node(e[1]), self.dump_node(e[2])) 8531dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project else: 8541dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project line += " ?%s)" % repr(e[1]) 8551dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 8561dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project return line 8571dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 8581dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project def __repr__(self): 8591dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project return self.dump_node(self.expr) 8601dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 86140596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes def source_node(self, e): 8621dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project op = e[0] 8631dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project if op == "int": 8641dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project return "%d" % e[1] 8651dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project if op == "hex": 8661dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project return "0x%x" % e[1] 8671dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project if op == "ident": 8681dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project # XXX: should try to expand 8691dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project return e[1] 8701dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project if op == "defined": 8711dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project return "defined(%s)" % e[1] 8721dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 8731dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project prec = CppExpr.precedences.get(op,1000) 8741dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project arg = e[1] 8751dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project if op in CppExpr.unaries: 8761dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project arg_src = self.source_node(arg) 8771dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project arg_op = arg[0] 8781dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project arg_prec = CppExpr.precedences.get(arg[0],1000) 8791dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project if arg_prec < prec: 8801dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project return "!(" + arg_src + ")" 8811dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project else: 8821dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project return "!" + arg_src 8831dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project if op in CppExpr.binaries: 8841dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project arg2 = e[2] 8851dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project arg1_op = arg[0] 8861dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project arg2_op = arg2[0] 8871dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project arg1_src = self.source_node(arg) 8881dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project arg2_src = self.source_node(arg2) 8891dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project if CppExpr.precedences.get(arg1_op,1000) < prec: 8901dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project arg1_src = "(%s)" % arg1_src 8911dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project if CppExpr.precedences.get(arg2_op,1000) < prec: 8921dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project arg2_src = "(%s)" % arg2_src 8931dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 8941dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project return "%s %s %s" % (arg1_src, op, arg2_src) 8951dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project return "???" 8961dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 8971dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project def __str__(self): 8981dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project return self.source_node(self.expr) 8991dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 9001dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project def int_node(self,e): 9011dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project if e[0] == "int": 9021dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project return e[1] 9031dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project elif e[1] == "hex": 9041dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project return int(e[1],16) 9051dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project else: 9061dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project return None 9071dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 9081dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project def toInt(self): 9091dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project return self.int_node(self.expr) 9101dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 91140596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes def optimize_node(self, e, macros={}): 9121dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project op = e[0] 9131dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project if op == "defined": 91440596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes op, name = e 9151dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project if macros.has_key(name): 9161dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project if macros[name] == kCppUndefinedMacro: 9171dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project return ("int", 0) 9181dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project else: 919d3e64a3f403918abf2cc3f9f0f9a2204a420ec96Elliott Hughes try: 920d3e64a3f403918abf2cc3f9f0f9a2204a420ec96Elliott Hughes value = int(macros[name]) 921d3e64a3f403918abf2cc3f9f0f9a2204a420ec96Elliott Hughes return ("int", value) 922d3e64a3f403918abf2cc3f9f0f9a2204a420ec96Elliott Hughes except: 923d3e64a3f403918abf2cc3f9f0f9a2204a420ec96Elliott Hughes return ("defined", macros[name]) 9241dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 9251dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project if kernel_remove_config_macros and name.startswith("CONFIG_"): 9261dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project return ("int", 0) 9271dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 92840596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes return e 92940596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes 93040596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes elif op == "ident": 93140596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes op, name = e 93240596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes if macros.has_key(name): 93340596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes try: 93440596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes value = int(macros[name]) 93540596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes expanded = ("int", value) 93640596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes except: 93740596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes expanded = ("ident", macros[name]) 93840596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes return self.optimize_node(expanded, macros) 93940596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes return e 94040596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes 9411dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project elif op == "!": 9421dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project op, v = e 9431dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project v = self.optimize_node(v, macros) 9441dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project if v[0] == "int": 9451dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project if v[1] == 0: 9461dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project return ("int", 1) 9471dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project else: 9481dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project return ("int", 0) 94940596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes return ('!', v) 9501dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 9511dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project elif op == "&&": 9521dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project op, l, r = e 9531dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project l = self.optimize_node(l, macros) 9541dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project r = self.optimize_node(r, macros) 9551dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project li = self.int_node(l) 9561dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project ri = self.int_node(r) 9571dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project if li != None: 9581dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project if li == 0: 9591dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project return ("int", 0) 9601dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project else: 9611dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project return r 96240596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes elif ri != None: 96340596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes if ri == 0: 96440596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes return ("int", 0) 96540596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes else: 96640596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes return l 96740596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes return (op, l, r) 9681dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 9691dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project elif op == "||": 9701dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project op, l, r = e 9711dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project l = self.optimize_node(l, macros) 9721dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project r = self.optimize_node(r, macros) 9731dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project li = self.int_node(l) 9741dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project ri = self.int_node(r) 9751dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project if li != None: 9761dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project if li == 0: 9771dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project return r 9781dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project else: 9791dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project return ("int", 1) 9801dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project elif ri != None: 9811dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project if ri == 0: 9821dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project return l 9831dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project else: 9841dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project return ("int", 1) 98540596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes return (op, l, r) 9861dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 98740596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes else: 98840596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes return e 9891dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 99040596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes def optimize(self,macros={}): 99140596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes self.expr = self.optimize_node(self.expr, macros) 9921dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 9931dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project def is_equal_node(self,e1,e2): 9941dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project if e1[0] != e2[0] or len(e1) != len(e2): 9951dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project return False 9961dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 9971dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project op = e1[0] 9981dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project if op == "int" or op == "hex" or op == "!" or op == "defined": 9991dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project return e1[0] == e2[0] 10001dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 10011dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project return self.is_equal_node(e1[1],e2[1]) and self.is_equal_node(e1[2],e2[2]) 10021dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 10031dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project def is_equal(self,other): 10041dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project return self.is_equal_node(self.expr,other.expr) 10051dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 10061dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Projectdef test_cpp_expr(expr, expected): 10071dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project e = CppExpr( CppLineTokenizer( expr ).toTokenList() ) 10081dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project s1 = repr(e) 10091dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project if s1 != expected: 101040596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes print "[FAIL]: expression '%s' generates '%s', should be '%s'" % (expr, s1, expected) 101140596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes global failure_count 101240596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes failure_count += 1 10131dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 10141dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Projectdef test_cpp_expr_optim(expr, expected, macros={}): 10151dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project e = CppExpr( CppLineTokenizer( expr ).toTokenList() ) 10161dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project e.optimize(macros) 10171dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project s1 = repr(e) 10181dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project if s1 != expected: 101940596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes print "[FAIL]: optimized expression '%s' generates '%s' with macros %s, should be '%s'" % (expr, s1, macros, expected) 102040596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes global failure_count 102140596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes failure_count += 1 10221dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 10231dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Projectdef test_cpp_expr_source(expr, expected): 10241dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project e = CppExpr( CppLineTokenizer( expr ).toTokenList() ) 10251dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project s1 = str(e) 10261dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project if s1 != expected: 102740596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes print "[FAIL]: source expression '%s' generates '%s', should be '%s'" % (expr, s1, expected) 102840596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes global failure_count 102940596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes failure_count += 1 10301dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 10311dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Projectdef test_CppExpr(): 103240596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes test_cpp_expr("0", "(int 0)") 103340596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes test_cpp_expr("1", "(int 1)") 103440596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes test_cpp_expr("(0)", "(int 0)") 103540596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes test_cpp_expr("1 && 1", "(&& (int 1) (int 1))") 103640596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes test_cpp_expr("1 && 0", "(&& (int 1) (int 0))") 103740596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes test_cpp_expr("EXAMPLE", "(ident EXAMPLE)") 103840596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes test_cpp_expr("EXAMPLE - 3", "(- (ident EXAMPLE) (int 3))") 103940596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes test_cpp_expr("defined(EXAMPLE)", "(defined EXAMPLE)") 104040596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes test_cpp_expr("defined ( EXAMPLE ) ", "(defined EXAMPLE)") 104140596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes test_cpp_expr("!defined(EXAMPLE)", "(! (defined EXAMPLE))") 104240596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes test_cpp_expr("defined(ABC) || defined(BINGO)", "(|| (defined ABC) (defined BINGO))") 104340596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes test_cpp_expr("FOO(BAR)", "(call FOO [BAR])") 104440596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes test_cpp_expr("A == 1 || defined(B)", "(|| (== (ident A) (int 1)) (defined B))") 104540596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes 104640596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes test_cpp_expr_optim("0", "(int 0)") 104740596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes test_cpp_expr_optim("1", "(int 1)") 104840596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes test_cpp_expr_optim("1 && 1", "(int 1)") 104940596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes test_cpp_expr_optim("1 && 0", "(int 0)") 105040596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes test_cpp_expr_optim("0 && 1", "(int 0)") 105140596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes test_cpp_expr_optim("0 && 0", "(int 0)") 105240596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes test_cpp_expr_optim("1 || 1", "(int 1)") 105340596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes test_cpp_expr_optim("1 || 0", "(int 1)") 105440596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes test_cpp_expr_optim("0 || 1", "(int 1)") 105540596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes test_cpp_expr_optim("0 || 0", "(int 0)") 105640596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes test_cpp_expr_optim("A", "(ident A)") 105740596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes test_cpp_expr_optim("A", "(int 1)", { "A": 1 }) 105840596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes test_cpp_expr_optim("A || B", "(int 1)", { "A": 1 }) 105940596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes test_cpp_expr_optim("A || B", "(int 1)", { "B": 1 }) 106040596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes test_cpp_expr_optim("A && B", "(ident B)", { "A": 1 }) 106140596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes test_cpp_expr_optim("A && B", "(ident A)", { "B": 1 }) 106240596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes test_cpp_expr_optim("A && B", "(&& (ident A) (ident B))") 106340596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes test_cpp_expr_optim("EXAMPLE", "(ident EXAMPLE)") 106440596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes test_cpp_expr_optim("EXAMPLE - 3", "(- (ident EXAMPLE) (int 3))") 106540596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes test_cpp_expr_optim("defined(EXAMPLE)", "(defined EXAMPLE)") 106640596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes test_cpp_expr_optim("defined(EXAMPLE)", "(defined XOWOE)", { "EXAMPLE": "XOWOE" }) 106740596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes test_cpp_expr_optim("defined(EXAMPLE)", "(int 0)", { "EXAMPLE": kCppUndefinedMacro}) 106840596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes test_cpp_expr_optim("!defined(EXAMPLE)", "(! (defined EXAMPLE))") 106940596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes test_cpp_expr_optim("!defined(EXAMPLE)", "(! (defined XOWOE))", { "EXAMPLE" : "XOWOE" }) 107040596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes test_cpp_expr_optim("!defined(EXAMPLE)", "(int 1)", { "EXAMPLE" : kCppUndefinedMacro }) 107140596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes test_cpp_expr_optim("defined(A) || defined(B)", "(|| (defined A) (defined B))") 107240596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes test_cpp_expr_optim("defined(A) || defined(B)", "(int 1)", { "A" : "1" }) 107340596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes test_cpp_expr_optim("defined(A) || defined(B)", "(int 1)", { "B" : "1" }) 107440596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes test_cpp_expr_optim("defined(A) || defined(B)", "(defined A)", { "B" : kCppUndefinedMacro }) 107540596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes test_cpp_expr_optim("defined(A) || defined(B)", "(int 0)", { "A" : kCppUndefinedMacro, "B" : kCppUndefinedMacro }) 107640596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes test_cpp_expr_optim("defined(A) && defined(B)", "(&& (defined A) (defined B))") 107740596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes test_cpp_expr_optim("defined(A) && defined(B)", "(defined B)", { "A" : "1" }) 107840596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes test_cpp_expr_optim("defined(A) && defined(B)", "(defined A)", { "B" : "1" }) 107940596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes test_cpp_expr_optim("defined(A) && defined(B)", "(int 0)", { "B" : kCppUndefinedMacro }) 108040596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes test_cpp_expr_optim("defined(A) && defined(B)", "(int 0)", { "A" : kCppUndefinedMacro }) 108140596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes test_cpp_expr_optim("A == 1 || defined(B)", "(|| (== (ident A) (int 1)) (defined B))" ) 108240596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes test_cpp_expr_optim("defined(__KERNEL__) || !defined(__GLIBC__) || (__GLIBC__ < 2)", "(|| (! (defined __GLIBC__)) (< (ident __GLIBC__) (int 2)))", { "__KERNEL__": kCppUndefinedMacro }) 108340596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes 108440596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes test_cpp_expr_source("0", "0") 108540596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes test_cpp_expr_source("1", "1") 108640596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes test_cpp_expr_source("1 && 1", "1 && 1") 108740596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes test_cpp_expr_source("1 && 0", "1 && 0") 108840596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes test_cpp_expr_source("0 && 1", "0 && 1") 108940596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes test_cpp_expr_source("0 && 0", "0 && 0") 109040596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes test_cpp_expr_source("1 || 1", "1 || 1") 109140596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes test_cpp_expr_source("1 || 0", "1 || 0") 109240596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes test_cpp_expr_source("0 || 1", "0 || 1") 109340596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes test_cpp_expr_source("0 || 0", "0 || 0") 109440596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes test_cpp_expr_source("EXAMPLE", "EXAMPLE") 109540596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes test_cpp_expr_source("EXAMPLE - 3", "EXAMPLE - 3") 109640596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes test_cpp_expr_source("defined(EXAMPLE)", "defined(EXAMPLE)") 109740596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes test_cpp_expr_source("defined EXAMPLE", "defined(EXAMPLE)") 109840596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes test_cpp_expr_source("A == 1 || defined(B)", "A == 1 || defined(B)") 10991dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 11001dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 11011dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project##################################################################################### 11021dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project##################################################################################### 11031dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project##### ##### 11041dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project##### C P P B L O C K ##### 11051dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project##### ##### 11061dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project##################################################################################### 11071dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project##################################################################################### 11081dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 11091dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Projectclass Block: 11101dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project """a class used to model a block of input source text. there are two block types: 1111fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner - directive blocks: contain the tokens of a single pre-processor directive (e.g. #if) 11121dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project - text blocks, contain the tokens of non-directive blocks 11131dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 11141dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project the cpp parser class below will transform an input source file into a list of Block 11151dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project objects (grouped in a BlockList object for convenience)""" 11161dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 11171dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project def __init__(self,tokens,directive=None,lineno=0): 11181dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project """initialize a new block, if 'directive' is None, this is a text block 11191dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project NOTE: this automatically converts '#ifdef MACRO' into '#if defined(MACRO)' 11201dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project and '#ifndef MACRO' into '#if !defined(MACRO)'""" 11211dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project if directive == "ifdef": 11221dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project tok = Token() 11231dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project tok.set(tokDEFINED) 11241dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project tokens = [ tok ] + tokens 11251dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project directive = "if" 11261dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 11271dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project elif directive == "ifndef": 11281dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project tok1 = Token() 11291dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project tok2 = Token() 11301dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project tok1.set(tokNOT) 11311dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project tok2.set(tokDEFINED) 11321dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project tokens = [ tok1, tok2 ] + tokens 11331dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project directive = "if" 11341dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 11351dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project self.tokens = tokens 11361dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project self.directive = directive 11371dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project if lineno > 0: 11381dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project self.lineno = lineno 11391dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project else: 11401dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project self.lineno = self.tokens[0].lineno 11411dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 11421dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project if self.isIf(): 11431dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project self.expr = CppExpr( self.tokens ) 11441dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 11451dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project def isDirective(self): 11461dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project """returns True iff this is a directive block""" 11471dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project return self.directive != None 11481dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 11491dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project def isConditional(self): 11501dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project """returns True iff this is a conditional directive block""" 11511dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project return self.directive in ["if","ifdef","ifndef","else","elif","endif"] 11521dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 11531dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project def isDefine(self): 11541dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project """returns the macro name in a #define directive, or None otherwise""" 11551dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project if self.directive != "define": 11561dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project return None 11571dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 11581dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project return self.tokens[0].value 11591dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 11601dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project def isIf(self): 11611dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project """returns True iff this is an #if-like directive block""" 11621dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project return self.directive in ["if","ifdef","ifndef","elif"] 11631dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 11641dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project def isInclude(self): 1165fddbafdc0a63a1b4c5e9252719a913197a4eecfaElliott Hughes """checks whether this is a #include directive. if true, then returns the 11661dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project corresponding file name (with brackets or double-qoutes). None otherwise""" 11671dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project if self.directive != "include": 11681dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project return None 11691dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 11701dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project if self.tokens[0].id == tokSTRING: 11711dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project # a double-quote include, that's easy 11721dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project return self.tokens[0].value 11731dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 11741dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project # we only want the bracket part, not any comments or junk after it 11751dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project if self.tokens[0].id == "<": 11761dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project i = 0 11771dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project tok = self.tokens 11781dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project n = len(tok) 11791dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project while i < n and tok[i].id != ">": 11801dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project i += 1 11811dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 11821dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project if i >= n: 11831dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project return None 11841dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 11851dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project return string.join([ str(x) for x in tok[:i+1] ],"") 11861dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 11871dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project else: 11881dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project return None 11891dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 1190fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner def removeWhiteSpace(self): 1191fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner # Remove trailing whitespace and empty lines 1192fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner # All whitespace is also contracted to a single space 1193fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner if self.directive != None: 1194fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner return 1195fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner 1196fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner tokens = [] 1197fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner line = 0 # index of line start 1198fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner space = -1 # index of first space, or -1 1199fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner ii = 0 1200fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner nn = len(self.tokens) 1201fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner while ii < nn: 1202fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner tok = self.tokens[ii] 1203fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner 1204fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner # If we find a space, record its position if this is the first 1205fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner # one the line start or the previous character. Don't append 1206fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner # anything to tokens array yet though. 1207fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner if tok.id == tokSPACE: 1208fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner if space < 0: 1209fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner space = ii 1210fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner ii += 1 1211fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner continue 1212fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner 1213fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner # If this is a line space, ignore the spaces we found previously 1214fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner # on the line, and remove empty lines. 1215fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner if tok.id == tokLN: 1216fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner old_line = line 1217fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner old_space = space 1218fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner ii += 1 1219fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner line = ii 1220fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner space = -1 1221fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner if old_space == old_line: # line only contains spaces 1222fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner continue 1223fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner if ii-1 == old_line: # line is empty 1224fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner continue 1225fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner tokens.append(tok) 1226fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner continue 1227fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner 1228fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner # Other token, append any space range if any, converting each 1229fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner # one to a single space character, then append the token. 1230fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner if space >= 0: 1231fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner jj = space 1232fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner space = -1 1233fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner while jj < ii: 1234fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner tok2 = self.tokens[jj] 1235fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner tok2.value = " " 1236fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner tokens.append(tok2) 1237fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner jj += 1 1238fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner 1239fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner tokens.append(tok) 1240fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner ii += 1 1241fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner 1242fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner self.tokens = tokens 1243fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner 1244fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner def writeWithWarning(self,out,warning,left_count,repeat_count): 1245fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner # removeWhiteSpace() will sometimes creates non-directive blocks 1246fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner # without any tokens. These come from blocks that only contained 1247fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner # empty lines and spaces. They should not be printed in the final 1248fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner # output, and then should not be counted for this operation. 1249fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner # 1250fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner if not self.directive and self.tokens == []: 1251fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner return left_count 1252fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner 1253fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner if self.directive: 1254c95eb57405d3d2f0e6cfab313aa74b1bad280452Elliott Hughes out.write(str(self).rstrip() + "\n") 1255fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner left_count -= 1 1256fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner if left_count == 0: 1257fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner out.write(warning) 1258fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner left_count = repeat_count 1259fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner 1260fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner else: 1261fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner for tok in self.tokens: 1262fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner out.write(str(tok)) 1263fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner if tok.id == tokLN: 1264fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner left_count -= 1 1265fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner if left_count == 0: 1266fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner out.write(warning) 1267fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner left_count = repeat_count 1268fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner 1269fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner return left_count 1270fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner 1271fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner 12721dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project def __repr__(self): 12731dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project """generate the representation of a given block""" 12741dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project if self.directive: 12751dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project result = "#%s " % self.directive 12761dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project if self.isIf(): 12771dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project result += repr(self.expr) 12781dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project else: 12791dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project for tok in self.tokens: 12801dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project result += repr(tok) 12811dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project else: 12821dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project result = "" 12831dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project for tok in self.tokens: 12841dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project result += repr(tok) 12851dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 12861dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project return result 12871dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 12881dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project def __str__(self): 12891dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project """generate the string representation of a given block""" 12901dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project if self.directive: 12911dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project if self.directive == "if": 12921dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project # small optimization to re-generate #ifdef and #ifndef 12931dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project e = self.expr.expr 12941dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project op = e[0] 12951dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project if op == "defined": 12961dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project result = "#ifdef %s" % e[1] 12971dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project elif op == "!" and e[1][0] == "defined": 12981dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project result = "#ifndef %s" % e[1][1] 12991dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project else: 13001dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project result = "#if " + str(self.expr) 13011dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project else: 13021dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project result = "#%s" % self.directive 13031dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project if len(self.tokens): 13041dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project result += " " 13051dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project for tok in self.tokens: 13061dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project result += str(tok) 13071dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project else: 13081dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project result = "" 13091dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project for tok in self.tokens: 13101dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project result += str(tok) 13111dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 13121dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project return result 13131dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 13141dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Projectclass BlockList: 13151dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project """a convenience class used to hold and process a list of blocks returned by 13161dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project the cpp parser""" 13171dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project def __init__(self,blocks): 13181dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project self.blocks = blocks 13191dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 13201dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project def __len__(self): 13211dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project return len(self.blocks) 13221dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 13231dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project def __getitem__(self,n): 13241dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project return self.blocks[n] 13251dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 13261dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project def __repr__(self): 13271dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project return repr(self.blocks) 13281dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 13291dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project def __str__(self): 13301dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project result = "" 13311dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project for b in self.blocks: 13321dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project result += str(b) 13331dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project if b.isDirective(): 1334c95eb57405d3d2f0e6cfab313aa74b1bad280452Elliott Hughes result = result.rstrip() + '\n' 13351dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project return result 13361dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 13371dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project def optimizeIf01(self): 13381dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project """remove the code between #if 0 .. #endif in a BlockList""" 13391dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project self.blocks = optimize_if01(self.blocks) 13401dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 13411dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project def optimizeMacros(self, macros): 13421dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project """remove known defined and undefined macros from a BlockList""" 13431dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project for b in self.blocks: 13441dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project if b.isIf(): 13451dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project b.expr.optimize(macros) 13461dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 13471dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project def removeMacroDefines(self,macros): 13481dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project """remove known macro definitions from a BlockList""" 13491dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project self.blocks = remove_macro_defines(self.blocks,macros) 13501dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 1351fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner def removeWhiteSpace(self): 1352fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner for b in self.blocks: 1353fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner b.removeWhiteSpace() 1354fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner 13551dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project def optimizeAll(self,macros): 13561dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project self.optimizeMacros(macros) 13571dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project self.optimizeIf01() 13581dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project return 13591dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 13601dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project def findIncludes(self): 13611dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project """return the list of included files in a BlockList""" 13621dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project result = [] 13631dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project for b in self.blocks: 13641dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project i = b.isInclude() 13651dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project if i: 13661dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project result.append(i) 13671dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 13681dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project return result 13691dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 13701dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 13711dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project def write(self,out): 13721dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project out.write(str(self)) 13731dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 1374fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner def writeWithWarning(self,out,warning,repeat_count): 1375fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner left_count = repeat_count 1376fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner for b in self.blocks: 1377fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner left_count = b.writeWithWarning(out,warning,left_count,repeat_count) 1378fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner 13791dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project def removeComments(self): 13801dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project for b in self.blocks: 13811dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project for tok in b.tokens: 13821dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project if tok.id == tokSPACE: 13831dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project tok.value = " " 13841dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 13851dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project def removeVarsAndFuncs(self,knownStatics=set()): 13861dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project """remove all extern and static declarations corresponding 13871dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project to variable and function declarations. we only accept typedefs 13881dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project and enum/structs/union declarations. 13891dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 13901dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project however, we keep the definitions corresponding to the set 13911dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project of known static inline functions in the set 'knownStatics', 13921dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project which is useful for optimized byteorder swap functions and 13931dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project stuff like that. 13941dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project """ 13951dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project # state = 0 => normal (i.e. LN + spaces) 1396fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner # state = 1 => typedef/struct encountered, ends with ";" 1397fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner # state = 2 => var declaration encountered, ends with ";" 1398fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner # state = 3 => func declaration encountered, ends with "}" 13991dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project state = 0 14001dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project depth = 0 14011dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project blocks2 = [] 1402fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner skipTokens = False 14031dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project for b in self.blocks: 14041dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project if b.isDirective(): 14051dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project blocks2.append(b) 14061dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project else: 14071dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project n = len(b.tokens) 14081dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project i = 0 1409fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner if skipTokens: 14101dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project first = n 1411fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner else: 1412fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner first = 0 14131dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project while i < n: 14141dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project tok = b.tokens[i] 1415fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner tokid = tok.id 1416fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner # If we are not looking for the start of a new 1417fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner # type/var/func, then skip over tokens until 1418fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner # we find our terminator, managing the depth of 1419fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner # accolades as we go. 1420fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner if state > 0: 1421fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner terminator = False 1422fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner if tokid == '{': 14231dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project depth += 1 1424fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner elif tokid == '}': 14251dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project if depth > 0: 14261dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project depth -= 1 1427fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner if (depth == 0) and (state == 3): 1428fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner terminator = True 1429fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner elif tokid == ';' and depth == 0: 1430fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner terminator = True 14311dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 1432fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner if terminator: 1433fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner # we found the terminator 14341dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project state = 0 1435fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner if skipTokens: 1436fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner skipTokens = False 1437fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner first = i+1 1438fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner 1439fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner i = i+1 1440fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner continue 1441fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner 1442fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner # We are looking for the start of a new type/func/var 1443fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner # ignore whitespace 1444fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner if tokid in [tokLN, tokSPACE]: 1445fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner i = i+1 1446fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner continue 1447fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner 1448fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner # Is it a new type definition, then start recording it 1449fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner if tok.value in [ 'struct', 'typedef', 'enum', 'union', '__extension__' ]: 1450fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner state = 1 1451fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner i = i+1 1452fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner continue 1453fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner 1454fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner # Is it a variable or function definition. If so, first 1455fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner # try to determine which type it is, and also extract 1456fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner # its name. 1457fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner # 1458fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner # We're going to parse the next tokens of the same block 1459fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner # until we find a semi-column or a left parenthesis. 1460fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner # 1461fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner # The semi-column corresponds to a variable definition, 1462fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner # the left-parenthesis to a function definition. 1463fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner # 1464fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner # We also assume that the var/func name is the last 1465fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner # identifier before the terminator. 1466fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner # 1467fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner j = i+1 1468fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner ident = "" 1469fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner while j < n: 1470fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner tokid = b.tokens[j].id 1471fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner if tokid == '(': # a function declaration 1472fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner state = 3 1473fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner break 1474fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner elif tokid == ';': # a variable declaration 1475fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner state = 2 1476fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner break 1477fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner if tokid == tokIDENT: 1478fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner ident = b.tokens[j].value 1479fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner j += 1 14801dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 1481fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner if j >= n: 1482fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner # This can only happen when the declaration 1483fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner # does not end on the current block (e.g. with 1484fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner # a directive mixed inside it. 1485fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner # 1486fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner # We will treat it as malformed because 1487fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner # it's very hard to recover from this case 1488fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner # without making our parser much more 1489fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner # complex. 1490fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner # 1491fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner #print "### skip unterminated static '%s'" % ident 1492fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner break 1493fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner 1494fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner if ident in knownStatics: 1495fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner #print "### keep var/func '%s': %s" % (ident,repr(b.tokens[i:j])) 1496fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner pass 1497fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner else: 1498fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner # We're going to skip the tokens for this declaration 1499fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner #print "### skip variable /func'%s': %s" % (ident,repr(b.tokens[i:j])) 1500fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner if i > first: 1501fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner blocks2.append( Block(b.tokens[first:i])) 1502fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner skipTokens = True 1503fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner first = n 1504fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner 1505fc2693110ee8a2ba22a445ad9855fbe9e118d439David 'Digit' Turner i = i+1 15061dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 15071dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project if i > first: 15081dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project #print "### final '%s'" % repr(b.tokens[first:i]) 15091dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project blocks2.append( Block(b.tokens[first:i]) ) 15101dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 15111dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project self.blocks = blocks2 15121dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 15131dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project def insertDisclaimer(self,disclaimer="/* auto-generated file, DO NOT EDIT */"): 15141dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project """insert your standard issue disclaimer that this is an 15151dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project auto-generated file, etc..""" 15161dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project tokens = CppLineTokenizer( disclaimer ).toTokenList() 15171dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project tokens = tokens[:-1] # remove trailing tokLN 15181dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project self.blocks = [ Block(tokens) ] + self.blocks 15191dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 1520fddbafdc0a63a1b4c5e9252719a913197a4eecfaElliott Hughes def replaceTokens(self,replacements): 1521fddbafdc0a63a1b4c5e9252719a913197a4eecfaElliott Hughes """replace tokens according to the given dict""" 1522d32c8052fc2b2d3c5da344eb673df35d4a5bc052Martin Storsjo for b in self.blocks: 1523fddbafdc0a63a1b4c5e9252719a913197a4eecfaElliott Hughes made_change = False 1524fddbafdc0a63a1b4c5e9252719a913197a4eecfaElliott Hughes if b.isInclude() == None: 1525d32c8052fc2b2d3c5da344eb673df35d4a5bc052Martin Storsjo for tok in b.tokens: 1526d32c8052fc2b2d3c5da344eb673df35d4a5bc052Martin Storsjo if tok.id == tokIDENT: 1527d32c8052fc2b2d3c5da344eb673df35d4a5bc052Martin Storsjo if tok.value in replacements: 1528d32c8052fc2b2d3c5da344eb673df35d4a5bc052Martin Storsjo tok.value = replacements[tok.value] 1529fddbafdc0a63a1b4c5e9252719a913197a4eecfaElliott Hughes made_change = True 1530fddbafdc0a63a1b4c5e9252719a913197a4eecfaElliott Hughes 1531fddbafdc0a63a1b4c5e9252719a913197a4eecfaElliott Hughes if made_change and b.isIf(): 1532fddbafdc0a63a1b4c5e9252719a913197a4eecfaElliott Hughes # Keep 'expr' in sync with 'tokens'. 1533fddbafdc0a63a1b4c5e9252719a913197a4eecfaElliott Hughes b.expr = CppExpr(b.tokens) 1534d32c8052fc2b2d3c5da344eb673df35d4a5bc052Martin Storsjo 15351dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Projectclass BlockParser: 15361dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project """a class used to convert an input source file into a BlockList object""" 15371dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 15381dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project def __init__(self,tokzer=None): 15391dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project """initialize a block parser. the input source is provided through a Tokenizer 15401dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project object""" 15411dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project self.reset(tokzer) 15421dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 15431dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project def reset(self,tokzer): 15441dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project self.state = 1 15451dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project self.tokzer = tokzer 15461dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 15471dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project def getBlocks(self,tokzer=None): 15481dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project """tokenize and parse the input source, return a BlockList object 15491dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project NOTE: empty and line-numbering directives are ignored and removed 15501dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project from the result. as a consequence, it is possible to have 15511dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project two successive text blocks in the result""" 15521dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project # state 0 => in source code 15531dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project # state 1 => in source code, after a LN 15541dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project # state 2 => in source code, after LN then some space 15551dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project state = 1 15561dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project lastLN = 0 15571dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project current = [] 15581dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project blocks = [] 15591dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 15601dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project if tokzer == None: 15611dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project tokzer = self.tokzer 15621dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 15631dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project while 1: 15641dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project tok = tokzer.getToken() 15651dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project if tok.id == tokEOF: 15661dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project break 15671dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 15681dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project if tok.id == tokLN: 15691dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project state = 1 15701dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project current.append(tok) 15711dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project lastLN = len(current) 15721dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 15731dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project elif tok.id == tokSPACE: 15741dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project if state == 1: 15751dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project state = 2 15761dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project current.append(tok) 15771dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 15781dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project elif tok.id == "#": 15791dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project if state > 0: 15801dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project # this is the start of a directive 15811dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 15821dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project if lastLN > 0: 15831dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project # record previous tokens as text block 15841dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project block = Block(current[:lastLN]) 15851dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project blocks.append(block) 15861dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project lastLN = 0 15871dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 15881dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project current = [] 15891dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 15901dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project # skip spaces after the # 15911dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project while 1: 15921dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project tok = tokzer.getToken() 15931dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project if tok.id != tokSPACE: 15941dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project break 15951dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 15961dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project if tok.id != tokIDENT: 15971dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project # empty or line-numbering, ignore it 15981dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project if tok.id != tokLN and tok.id != tokEOF: 15991dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project while 1: 16001dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project tok = tokzer.getToken() 16011dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project if tok.id == tokLN or tok.id == tokEOF: 16021dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project break 16031dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project continue 16041dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 16051dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project directive = tok.value 16061dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project lineno = tok.lineno 16071dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 16081dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project # skip spaces 16091dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project tok = tokzer.getToken() 16101dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project while tok.id == tokSPACE: 16111dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project tok = tokzer.getToken() 16121dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 16131dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project # then record tokens until LN 16141dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project dirtokens = [] 16151dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project while tok.id != tokLN and tok.id != tokEOF: 16161dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project dirtokens.append(tok) 16171dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project tok = tokzer.getToken() 16181dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 16191dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project block = Block(dirtokens,directive,lineno) 16201dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project blocks.append(block) 16211dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project state = 1 16221dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 16231dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project else: 16241dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project state = 0 16251dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project current.append(tok) 16261dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 16271dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project if len(current) > 0: 16281dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project block = Block(current) 16291dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project blocks.append(block) 16301dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 16311dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project return BlockList(blocks) 16321dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 16331dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project def parse(self,tokzer): 16341dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project return self.getBlocks( tokzer ) 16351dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 16361dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project def parseLines(self,lines): 16371dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project """parse a list of text lines into a BlockList object""" 16381dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project return self.getBlocks( CppLinesTokenizer(lines) ) 16391dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 16401dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project def parseFile(self,path): 16411dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project """parse a file into a BlockList object""" 16421dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project file = open(path, "rt") 16431dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project result = self.getBlocks( CppFileTokenizer(file) ) 16441dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project file.close() 16451dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project return result 16461dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 16471dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 16481dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Projectdef test_block_parsing(lines,expected): 16491dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project blocks = BlockParser().parse( CppLinesTokenizer(lines) ) 16501dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project if len(blocks) != len(expected): 16511dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project raise BadExpectedToken, "parser.buildBlocks returned '%s' expecting '%s'" \ 16521dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project % (str(blocks), repr(expected)) 16531dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project for n in range(len(blocks)): 16541dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project if str(blocks[n]) != expected[n]: 16551dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project raise BadExpectedToken, "parser.buildBlocks()[%d] is '%s', expecting '%s'" \ 16561dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project % (n, str(blocks[n]), expected[n]) 16571dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project #for block in blocks: 16581dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project # print block 16591dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 16601dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Projectdef test_BlockParser(): 16611dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project test_block_parsing(["#error hello"],["#error hello"]) 16621dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project test_block_parsing([ "foo", "", "bar" ], [ "foo\n\nbar\n" ]) 16631dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project test_block_parsing([ "foo", " # ", "bar" ], [ "foo\n","bar\n" ]) 16641dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project test_block_parsing(\ 16651dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project [ "foo", " # ", " # /* ahah */ if defined(__KERNEL__) ", "bar", "#endif" ], 16661dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project [ "foo\n", "#ifdef __KERNEL__", "bar\n", "#endif" ] ) 16671dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 16681dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 16691dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project##################################################################################### 16701dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project##################################################################################### 16711dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project##### ##### 16721dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project##### B L O C K L I S T O P T I M I Z A T I O N ##### 16731dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project##### ##### 16741dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project##################################################################################### 16751dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project##################################################################################### 16761dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 16771dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Projectdef remove_macro_defines( blocks, excludedMacros=set() ): 16781dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project """remove macro definitions like #define <macroName> ....""" 16791dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project result = [] 16801dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project for b in blocks: 16811dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project macroName = b.isDefine() 16821dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project if macroName == None or not macroName in excludedMacros: 16831dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project result.append(b) 16841dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 16851dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project return result 16861dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 16871dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Projectdef find_matching_endif( blocks, i ): 16881dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project n = len(blocks) 16891dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project depth = 1 16901dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project while i < n: 16911dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project if blocks[i].isDirective(): 16921dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project dir = blocks[i].directive 16931dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project if dir in [ "if", "ifndef", "ifdef" ]: 16941dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project depth += 1 16951dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project elif depth == 1 and dir in [ "else", "elif" ]: 16961dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project return i 16971dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project elif dir == "endif": 16981dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project depth -= 1 16991dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project if depth == 0: 17001dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project return i 17011dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project i += 1 17021dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project return i 17031dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 17041dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Projectdef optimize_if01( blocks ): 17051dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project """remove the code between #if 0 .. #endif in a list of CppBlocks""" 17061dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project i = 0 17071dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project n = len(blocks) 17081dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project result = [] 17091dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project while i < n: 17101dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project j = i 17111dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project while j < n and not blocks[j].isIf(): 17121dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project j += 1 17131dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project if j > i: 17141dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project D2("appending lines %d to %d" % (blocks[i].lineno, blocks[j-1].lineno)) 17151dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project result += blocks[i:j] 17161dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project if j >= n: 17171dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project break 17181dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project expr = blocks[j].expr 17191dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project r = expr.toInt() 17201dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project if r == None: 17211dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project result.append(blocks[j]) 17221dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project i = j + 1 17231dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project continue 17241dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 17251dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project if r == 0: 17261dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project # if 0 => skip everything until the corresponding #endif 17271dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project j = find_matching_endif( blocks, j+1 ) 17281dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project if j >= n: 17291dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project # unterminated #if 0, finish here 17301dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project break 17311dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project dir = blocks[j].directive 17321dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project if dir == "endif": 17331dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project D2("remove 'if 0' .. 'endif' (lines %d to %d)" % (blocks[i].lineno, blocks[j].lineno)) 17341dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project i = j + 1 17351dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project elif dir == "else": 17361dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project # convert 'else' into 'if 1' 17371dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project D2("convert 'if 0' .. 'else' into 'if 1' (lines %d to %d)" % (blocks[i].lineno, blocks[j-1].lineno)) 17381dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project blocks[j].directive = "if" 17391dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project blocks[j].expr = CppExpr( CppLineTokenizer("1").toTokenList() ) 17401dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project i = j 17411dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project elif dir == "elif": 17421dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project # convert 'elif' into 'if' 17431dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project D2("convert 'if 0' .. 'elif' into 'if'") 17441dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project blocks[j].directive = "if" 17451dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project i = j 17461dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project continue 17471dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 17481dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project # if 1 => find corresponding endif and remove/transform them 17491dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project k = find_matching_endif( blocks, j+1 ) 17501dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project if k >= n: 17511dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project # unterminated #if 1, finish here 17521dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project D2("unterminated 'if 1'") 17531dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project result += blocks[j+1:k] 17541dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project break 17551dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 17561dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project dir = blocks[k].directive 17571dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project if dir == "endif": 17581dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project D2("convert 'if 1' .. 'endif' (lines %d to %d)" % (blocks[j].lineno, blocks[k].lineno)) 17591dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project result += optimize_if01(blocks[j+1:k]) 17601dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project i = k+1 17611dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project elif dir == "else": 17621dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project # convert 'else' into 'if 0' 17631dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project D2("convert 'if 1' .. 'else' (lines %d to %d)" % (blocks[j].lineno, blocks[k].lineno)) 17641dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project result += optimize_if01(blocks[j+1:k]) 17651dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project blocks[k].directive = "if" 17661dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project blocks[k].expr = CppExpr( CppLineTokenizer("0").toTokenList() ) 17671dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project i = k 17681dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project elif dir == "elif": 17691dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project # convert 'elif' into 'if 0' 17701dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project D2("convert 'if 1' .. 'elif' (lines %d to %d)" % (blocks[j].lineno, blocks[k].lineno)) 17711dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project result += optimize_if01(blocks[j+1:k]) 17721dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project blocks[k].expr = CppExpr( CppLineTokenizer("0").toTokenList() ) 17731dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project i = k 17741dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project return result 17751dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 17761dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Projectdef test_optimizeAll(): 17771dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project text = """\ 17781dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project#if 1 17791dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project#define GOOD_1 17801dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project#endif 17811dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project#if 0 17821dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project#define BAD_2 17831dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project#define BAD_3 17841dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project#endif 17851dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 17861dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project#if 1 17871dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project#define GOOD_2 17881dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project#else 17891dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project#define BAD_4 17901dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project#endif 17911dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 17921dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project#if 0 17931dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project#define BAD_5 17941dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project#else 17951dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project#define GOOD_3 17961dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project#endif 17971dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 179840596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes#if defined(__KERNEL__) 179940596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes#define BAD_KERNEL 180040596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes#endif 180140596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes 180240596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes#if defined(__KERNEL__) || !defined(__GLIBC__) || (__GLIBC__ < 2) 180340596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes#define X 180440596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes#endif 180540596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes 1806fddbafdc0a63a1b4c5e9252719a913197a4eecfaElliott Hughes#ifndef SIGRTMAX 1807fddbafdc0a63a1b4c5e9252719a913197a4eecfaElliott Hughes#define SIGRTMAX 123 1808fddbafdc0a63a1b4c5e9252719a913197a4eecfaElliott Hughes#endif /* SIGRTMAX */ 1809fddbafdc0a63a1b4c5e9252719a913197a4eecfaElliott Hughes 18101dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project#if 0 18111dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project#if 1 18121dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project#define BAD_6 18131dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project#endif 18141dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project#endif\ 18151dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project""" 18161dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 18171dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project expected = """\ 18181dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project#define GOOD_1 18191dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 18201dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project#define GOOD_2 18211dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 18221dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project#define GOOD_3 18231dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 182440596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes 182540596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes#if !defined(__GLIBC__) || __GLIBC__ < 2 182640596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes#define X 182740596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes#endif 182840596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes 1829fddbafdc0a63a1b4c5e9252719a913197a4eecfaElliott Hughes#ifndef __SIGRTMAX 1830fddbafdc0a63a1b4c5e9252719a913197a4eecfaElliott Hughes#define __SIGRTMAX 123 1831fddbafdc0a63a1b4c5e9252719a913197a4eecfaElliott Hughes#endif 1832fddbafdc0a63a1b4c5e9252719a913197a4eecfaElliott Hughes 18331dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project""" 18341dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 18351dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project out = StringOutput() 18361dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project lines = string.split(text, '\n') 18371dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project list = BlockParser().parse( CppLinesTokenizer(lines) ) 18381dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project #D_setlevel(2) 1839fddbafdc0a63a1b4c5e9252719a913197a4eecfaElliott Hughes list.replaceTokens( kernel_token_replacements ) 18401dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project list.optimizeAll( {"__KERNEL__":kCppUndefinedMacro} ) 18411dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project list.write(out) 18421dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project if out.get() != expected: 184340596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes print "[FAIL]: macro optimization failed\n" 18441dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project print "<<<< expecting '", 18451dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project print expected, 18461dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project print "'\n>>>> result '" 18471dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project print out.get(), 18481dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project print "'\n----" 184940596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes global failure_count 185040596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes failure_count += 1 18511dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 18521dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 185340596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes# -- Always run the unit tests. 18541dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 18551dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Projectdef runUnitTests(): 18561dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project """run all unit tests for this program""" 18571dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project test_CppTokenizer() 18581dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project test_CppExpr() 18591dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project test_optimizeAll() 18601dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project test_BlockParser() 18611dc9e472e19acfe6dc7f41e429236e7eef7ceda1The Android Open Source Project 186240596aa0054bcfa76148f55321bf4b979e2242beElliott Hughesfailure_count = 0 186340596aa0054bcfa76148f55321bf4b979e2242beElliott HughesrunUnitTests() 186440596aa0054bcfa76148f55321bf4b979e2242beElliott Hughesif failure_count != 0: 186540596aa0054bcfa76148f55321bf4b979e2242beElliott Hughes sys.exit(1) 1866