1583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata# ----------------------------------------------------------------------------- 2583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata# cpp.py 3583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata# 4583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata# Author: David Beazley (http://www.dabeaz.com) 5583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata# Copyright (C) 2007 6583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata# All rights reserved 7583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata# 8583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata# This module implements an ANSI-C style lexical preprocessor for PLY. 9583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata# ----------------------------------------------------------------------------- 10583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granatafrom __future__ import generators 11583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata 12583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granataimport sys 13583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata 14583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata# Some Python 3 compatibility shims 15583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granataif sys.version_info.major < 3: 16583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata STRING_TYPES = (str, unicode) 17583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granataelse: 18583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata STRING_TYPES = str 19583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata xrange = range 20583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata 21583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata# ----------------------------------------------------------------------------- 22583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata# Default preprocessor lexer definitions. These tokens are enough to get 23583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata# a basic preprocessor working. Other modules may import these if they want 24583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata# ----------------------------------------------------------------------------- 25583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata 26583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granatatokens = ( 27583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata 'CPP_ID','CPP_INTEGER', 'CPP_FLOAT', 'CPP_STRING', 'CPP_CHAR', 'CPP_WS', 'CPP_COMMENT1', 'CPP_COMMENT2', 'CPP_POUND','CPP_DPOUND' 28583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata) 29583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata 30583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granataliterals = "+-*/%|&~^<>=!?()[]{}.,;:\\\'\"" 31583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata 32583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata# Whitespace 33583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granatadef t_CPP_WS(t): 34583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata r'\s+' 35583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata t.lexer.lineno += t.value.count("\n") 36583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata return t 37583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata 38583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granatat_CPP_POUND = r'\#' 39583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granatat_CPP_DPOUND = r'\#\#' 40583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata 41583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata# Identifier 42583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granatat_CPP_ID = r'[A-Za-z_][\w_]*' 43583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata 44583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata# Integer literal 45583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granatadef CPP_INTEGER(t): 46583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata r'(((((0x)|(0X))[0-9a-fA-F]+)|(\d+))([uU][lL]|[lL][uU]|[uU]|[lL])?)' 47583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata return t 48583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata 49583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granatat_CPP_INTEGER = CPP_INTEGER 50583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata 51583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata# Floating literal 52583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granatat_CPP_FLOAT = r'((\d+)(\.\d+)(e(\+|-)?(\d+))? | (\d+)e(\+|-)?(\d+))([lL]|[fF])?' 53583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata 54583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata# String literal 55583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granatadef t_CPP_STRING(t): 56583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata r'\"([^\\\n]|(\\(.|\n)))*?\"' 57583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata t.lexer.lineno += t.value.count("\n") 58583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata return t 59583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata 60583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata# Character constant 'c' or L'c' 61583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granatadef t_CPP_CHAR(t): 62583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata r'(L)?\'([^\\\n]|(\\(.|\n)))*?\'' 63583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata t.lexer.lineno += t.value.count("\n") 64583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata return t 65583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata 66583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata# Comment 67583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granatadef t_CPP_COMMENT1(t): 68583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata r'(/\*(.|\n)*?\*/)' 69583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata ncr = t.value.count("\n") 70583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata t.lexer.lineno += ncr 71583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata # replace with one space or a number of '\n' 72583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata t.type = 'CPP_WS'; t.value = '\n' * ncr if ncr else ' ' 73583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata return t 74583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata 75583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata# Line comment 76583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granatadef t_CPP_COMMENT2(t): 77583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata r'(//.*?(\n|$))' 78583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata # replace with '/n' 79583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata t.type = 'CPP_WS'; t.value = '\n' 80583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata return t 81583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata 82583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granatadef t_error(t): 83583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata t.type = t.value[0] 84583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata t.value = t.value[0] 85583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata t.lexer.skip(1) 86583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata return t 87583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata 88583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granataimport re 89583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granataimport copy 90583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granataimport time 91583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granataimport os.path 92583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata 93583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata# ----------------------------------------------------------------------------- 94583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata# trigraph() 95583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata# 96583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata# Given an input string, this function replaces all trigraph sequences. 97583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata# The following mapping is used: 98583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata# 99583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata# ??= # 100583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata# ??/ \ 101583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata# ??' ^ 102583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata# ??( [ 103583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata# ??) ] 104583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata# ??! | 105583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata# ??< { 106583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata# ??> } 107583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata# ??- ~ 108583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata# ----------------------------------------------------------------------------- 109583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata 110583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata_trigraph_pat = re.compile(r'''\?\?[=/\'\(\)\!<>\-]''') 111583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata_trigraph_rep = { 112583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata '=':'#', 113583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata '/':'\\', 114583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata "'":'^', 115583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata '(':'[', 116583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata ')':']', 117583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata '!':'|', 118583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata '<':'{', 119583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata '>':'}', 120583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata '-':'~' 121583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata} 122583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata 123583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granatadef trigraph(input): 124583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata return _trigraph_pat.sub(lambda g: _trigraph_rep[g.group()[-1]],input) 125583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata 126583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata# ------------------------------------------------------------------ 127583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata# Macro object 128583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata# 129583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata# This object holds information about preprocessor macros 130583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata# 131583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata# .name - Macro name (string) 132583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata# .value - Macro value (a list of tokens) 133583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata# .arglist - List of argument names 134583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata# .variadic - Boolean indicating whether or not variadic macro 135583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata# .vararg - Name of the variadic parameter 136583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata# 137583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata# When a macro is created, the macro replacement token sequence is 138583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata# pre-scanned and used to create patch lists that are later used 139583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata# during macro expansion 140583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata# ------------------------------------------------------------------ 141583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata 142583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granataclass Macro(object): 143583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata def __init__(self,name,value,arglist=None,variadic=False): 144583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata self.name = name 145583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata self.value = value 146583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata self.arglist = arglist 147583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata self.variadic = variadic 148583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata if variadic: 149583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata self.vararg = arglist[-1] 150583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata self.source = None 151583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata 152583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata# ------------------------------------------------------------------ 153583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata# Preprocessor object 154583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata# 155583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata# Object representing a preprocessor. Contains macro definitions, 156583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata# include directories, and other information 157583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata# ------------------------------------------------------------------ 158583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata 159583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granataclass Preprocessor(object): 160583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata def __init__(self,lexer=None): 161583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata if lexer is None: 162583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata lexer = lex.lexer 163583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata self.lexer = lexer 164583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata self.macros = { } 165583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata self.path = [] 166583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata self.temp_path = [] 167583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata 168583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata # Probe the lexer for selected tokens 169583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata self.lexprobe() 170583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata 171583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata tm = time.localtime() 172583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata self.define("__DATE__ \"%s\"" % time.strftime("%b %d %Y",tm)) 173583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata self.define("__TIME__ \"%s\"" % time.strftime("%H:%M:%S",tm)) 174583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata self.parser = None 175583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata 176583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata # ----------------------------------------------------------------------------- 177583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata # tokenize() 178583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata # 179583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata # Utility function. Given a string of text, tokenize into a list of tokens 180583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata # ----------------------------------------------------------------------------- 181583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata 182583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata def tokenize(self,text): 183583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata tokens = [] 184583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata self.lexer.input(text) 185583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata while True: 186583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata tok = self.lexer.token() 187583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata if not tok: break 188583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata tokens.append(tok) 189583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata return tokens 190583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata 191583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata # --------------------------------------------------------------------- 192583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata # error() 193583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata # 194583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata # Report a preprocessor error/warning of some kind 195583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata # ---------------------------------------------------------------------- 196583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata 197583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata def error(self,file,line,msg): 198583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata print("%s:%d %s" % (file,line,msg)) 199583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata 200583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata # ---------------------------------------------------------------------- 201583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata # lexprobe() 202583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata # 203583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata # This method probes the preprocessor lexer object to discover 204583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata # the token types of symbols that are important to the preprocessor. 205583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata # If this works right, the preprocessor will simply "work" 206583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata # with any suitable lexer regardless of how tokens have been named. 207583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata # ---------------------------------------------------------------------- 208583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata 209583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata def lexprobe(self): 210583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata 211583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata # Determine the token type for identifiers 212583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata self.lexer.input("identifier") 213583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata tok = self.lexer.token() 214583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata if not tok or tok.value != "identifier": 215583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata print("Couldn't determine identifier type") 216583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata else: 217583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata self.t_ID = tok.type 218583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata 219583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata # Determine the token type for integers 220583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata self.lexer.input("12345") 221583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata tok = self.lexer.token() 222583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata if not tok or int(tok.value) != 12345: 223583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata print("Couldn't determine integer type") 224583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata else: 225583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata self.t_INTEGER = tok.type 226583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata self.t_INTEGER_TYPE = type(tok.value) 227583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata 228583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata # Determine the token type for strings enclosed in double quotes 229583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata self.lexer.input("\"filename\"") 230583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata tok = self.lexer.token() 231583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata if not tok or tok.value != "\"filename\"": 232583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata print("Couldn't determine string type") 233583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata else: 234583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata self.t_STRING = tok.type 235583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata 236583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata # Determine the token type for whitespace--if any 237583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata self.lexer.input(" ") 238583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata tok = self.lexer.token() 239583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata if not tok or tok.value != " ": 240583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata self.t_SPACE = None 241583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata else: 242583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata self.t_SPACE = tok.type 243583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata 244583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata # Determine the token type for newlines 245583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata self.lexer.input("\n") 246583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata tok = self.lexer.token() 247583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata if not tok or tok.value != "\n": 248583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata self.t_NEWLINE = None 249583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata print("Couldn't determine token for newlines") 250583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata else: 251583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata self.t_NEWLINE = tok.type 252583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata 253583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata self.t_WS = (self.t_SPACE, self.t_NEWLINE) 254583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata 255583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata # Check for other characters used by the preprocessor 256583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata chars = [ '<','>','#','##','\\','(',')',',','.'] 257583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata for c in chars: 258583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata self.lexer.input(c) 259583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata tok = self.lexer.token() 260583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata if not tok or tok.value != c: 261583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata print("Unable to lex '%s' required for preprocessor" % c) 262583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata 263583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata # ---------------------------------------------------------------------- 264583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata # add_path() 265583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata # 266583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata # Adds a search path to the preprocessor. 267583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata # ---------------------------------------------------------------------- 268583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata 269583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata def add_path(self,path): 270583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata self.path.append(path) 271583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata 272583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata # ---------------------------------------------------------------------- 273583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata # group_lines() 274583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata # 275583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata # Given an input string, this function splits it into lines. Trailing whitespace 276583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata # is removed. Any line ending with \ is grouped with the next line. This 277583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata # function forms the lowest level of the preprocessor---grouping into text into 278583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata # a line-by-line format. 279583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata # ---------------------------------------------------------------------- 280583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata 281583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata def group_lines(self,input): 282583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata lex = self.lexer.clone() 283583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata lines = [x.rstrip() for x in input.splitlines()] 284583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata for i in xrange(len(lines)): 285583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata j = i+1 286583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata while lines[i].endswith('\\') and (j < len(lines)): 287583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata lines[i] = lines[i][:-1]+lines[j] 288583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata lines[j] = "" 289583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata j += 1 290583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata 291583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata input = "\n".join(lines) 292583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata lex.input(input) 293583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata lex.lineno = 1 294583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata 295583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata current_line = [] 296583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata while True: 297583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata tok = lex.token() 298583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata if not tok: 299583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata break 300583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata current_line.append(tok) 301583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata if tok.type in self.t_WS and '\n' in tok.value: 302583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata yield current_line 303583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata current_line = [] 304583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata 305583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata if current_line: 306583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata yield current_line 307583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata 308583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata # ---------------------------------------------------------------------- 309583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata # tokenstrip() 310583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata # 311583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata # Remove leading/trailing whitespace tokens from a token list 312583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata # ---------------------------------------------------------------------- 313583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata 314583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata def tokenstrip(self,tokens): 315583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata i = 0 316583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata while i < len(tokens) and tokens[i].type in self.t_WS: 317583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata i += 1 318583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata del tokens[:i] 319583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata i = len(tokens)-1 320583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata while i >= 0 and tokens[i].type in self.t_WS: 321583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata i -= 1 322583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata del tokens[i+1:] 323583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata return tokens 324583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata 325583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata 326583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata # ---------------------------------------------------------------------- 327583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata # collect_args() 328583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata # 329583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata # Collects comma separated arguments from a list of tokens. The arguments 330583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata # must be enclosed in parenthesis. Returns a tuple (tokencount,args,positions) 331583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata # where tokencount is the number of tokens consumed, args is a list of arguments, 332583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata # and positions is a list of integers containing the starting index of each 333583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata # argument. Each argument is represented by a list of tokens. 334583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata # 335583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata # When collecting arguments, leading and trailing whitespace is removed 336583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata # from each argument. 337583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata # 338583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata # This function properly handles nested parenthesis and commas---these do not 339583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata # define new arguments. 340583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata # ---------------------------------------------------------------------- 341583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata 342583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata def collect_args(self,tokenlist): 343583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata args = [] 344583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata positions = [] 345583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata current_arg = [] 346583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata nesting = 1 347583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata tokenlen = len(tokenlist) 348583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata 349583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata # Search for the opening '('. 350583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata i = 0 351583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata while (i < tokenlen) and (tokenlist[i].type in self.t_WS): 352583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata i += 1 353583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata 354583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata if (i < tokenlen) and (tokenlist[i].value == '('): 355583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata positions.append(i+1) 356583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata else: 357583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata self.error(self.source,tokenlist[0].lineno,"Missing '(' in macro arguments") 358583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata return 0, [], [] 359583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata 360583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata i += 1 361583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata 362583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata while i < tokenlen: 363583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata t = tokenlist[i] 364583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata if t.value == '(': 365583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata current_arg.append(t) 366583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata nesting += 1 367583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata elif t.value == ')': 368583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata nesting -= 1 369583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata if nesting == 0: 370583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata if current_arg: 371583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata args.append(self.tokenstrip(current_arg)) 372583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata positions.append(i) 373583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata return i+1,args,positions 374583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata current_arg.append(t) 375583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata elif t.value == ',' and nesting == 1: 376583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata args.append(self.tokenstrip(current_arg)) 377583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata positions.append(i+1) 378583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata current_arg = [] 379583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata else: 380583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata current_arg.append(t) 381583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata i += 1 382583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata 383583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata # Missing end argument 384583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata self.error(self.source,tokenlist[-1].lineno,"Missing ')' in macro arguments") 385583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata return 0, [],[] 386583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata 387583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata # ---------------------------------------------------------------------- 388583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata # macro_prescan() 389583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata # 390583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata # Examine the macro value (token sequence) and identify patch points 391583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata # This is used to speed up macro expansion later on---we'll know 392583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata # right away where to apply patches to the value to form the expansion 393583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata # ---------------------------------------------------------------------- 394583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata 395583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata def macro_prescan(self,macro): 396583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata macro.patch = [] # Standard macro arguments 397583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata macro.str_patch = [] # String conversion expansion 398583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata macro.var_comma_patch = [] # Variadic macro comma patch 399583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata i = 0 400583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata while i < len(macro.value): 401583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata if macro.value[i].type == self.t_ID and macro.value[i].value in macro.arglist: 402583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata argnum = macro.arglist.index(macro.value[i].value) 403583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata # Conversion of argument to a string 404583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata if i > 0 and macro.value[i-1].value == '#': 405583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata macro.value[i] = copy.copy(macro.value[i]) 406583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata macro.value[i].type = self.t_STRING 407583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata del macro.value[i-1] 408583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata macro.str_patch.append((argnum,i-1)) 409583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata continue 410583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata # Concatenation 411583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata elif (i > 0 and macro.value[i-1].value == '##'): 412583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata macro.patch.append(('c',argnum,i-1)) 413583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata del macro.value[i-1] 414583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata continue 415583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata elif ((i+1) < len(macro.value) and macro.value[i+1].value == '##'): 416583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata macro.patch.append(('c',argnum,i)) 417583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata i += 1 418583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata continue 419583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata # Standard expansion 420583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata else: 421583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata macro.patch.append(('e',argnum,i)) 422583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata elif macro.value[i].value == '##': 423583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata if macro.variadic and (i > 0) and (macro.value[i-1].value == ',') and \ 424583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata ((i+1) < len(macro.value)) and (macro.value[i+1].type == self.t_ID) and \ 425583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata (macro.value[i+1].value == macro.vararg): 426583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata macro.var_comma_patch.append(i-1) 427583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata i += 1 428583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata macro.patch.sort(key=lambda x: x[2],reverse=True) 429583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata 430583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata # ---------------------------------------------------------------------- 431583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata # macro_expand_args() 432583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata # 433583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata # Given a Macro and list of arguments (each a token list), this method 434583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata # returns an expanded version of a macro. The return value is a token sequence 435583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata # representing the replacement macro tokens 436583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata # ---------------------------------------------------------------------- 437583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata 438583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata def macro_expand_args(self,macro,args): 439583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata # Make a copy of the macro token sequence 440583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata rep = [copy.copy(_x) for _x in macro.value] 441583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata 442583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata # Make string expansion patches. These do not alter the length of the replacement sequence 443583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata 444583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata str_expansion = {} 445583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata for argnum, i in macro.str_patch: 446583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata if argnum not in str_expansion: 447583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata str_expansion[argnum] = ('"%s"' % "".join([x.value for x in args[argnum]])).replace("\\","\\\\") 448583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata rep[i] = copy.copy(rep[i]) 449583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata rep[i].value = str_expansion[argnum] 450583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata 451583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata # Make the variadic macro comma patch. If the variadic macro argument is empty, we get rid 452583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata comma_patch = False 453583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata if macro.variadic and not args[-1]: 454583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata for i in macro.var_comma_patch: 455583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata rep[i] = None 456583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata comma_patch = True 457583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata 458583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata # Make all other patches. The order of these matters. It is assumed that the patch list 459583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata # has been sorted in reverse order of patch location since replacements will cause the 460583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata # size of the replacement sequence to expand from the patch point. 461583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata 462583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata expanded = { } 463583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata for ptype, argnum, i in macro.patch: 464583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata # Concatenation. Argument is left unexpanded 465583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata if ptype == 'c': 466583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata rep[i:i+1] = args[argnum] 467583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata # Normal expansion. Argument is macro expanded first 468583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata elif ptype == 'e': 469583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata if argnum not in expanded: 470583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata expanded[argnum] = self.expand_macros(args[argnum]) 471583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata rep[i:i+1] = expanded[argnum] 472583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata 473583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata # Get rid of removed comma if necessary 474583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata if comma_patch: 475583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata rep = [_i for _i in rep if _i] 476583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata 477583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata return rep 478583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata 479583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata 480583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata # ---------------------------------------------------------------------- 481583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata # expand_macros() 482583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata # 483583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata # Given a list of tokens, this function performs macro expansion. 484583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata # The expanded argument is a dictionary that contains macros already 485583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata # expanded. This is used to prevent infinite recursion. 486583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata # ---------------------------------------------------------------------- 487583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata 488583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata def expand_macros(self,tokens,expanded=None): 489583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata if expanded is None: 490583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata expanded = {} 491583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata i = 0 492583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata while i < len(tokens): 493583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata t = tokens[i] 494583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata if t.type == self.t_ID: 495583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata if t.value in self.macros and t.value not in expanded: 496583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata # Yes, we found a macro match 497583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata expanded[t.value] = True 498583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata 499583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata m = self.macros[t.value] 500583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata if not m.arglist: 501583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata # A simple macro 502583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata ex = self.expand_macros([copy.copy(_x) for _x in m.value],expanded) 503583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata for e in ex: 504583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata e.lineno = t.lineno 505583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata tokens[i:i+1] = ex 506583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata i += len(ex) 507583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata else: 508583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata # A macro with arguments 509583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata j = i + 1 510583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata while j < len(tokens) and tokens[j].type in self.t_WS: 511583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata j += 1 512583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata if tokens[j].value == '(': 513583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata tokcount,args,positions = self.collect_args(tokens[j:]) 514583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata if not m.variadic and len(args) != len(m.arglist): 515583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata self.error(self.source,t.lineno,"Macro %s requires %d arguments" % (t.value,len(m.arglist))) 516583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata i = j + tokcount 517583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata elif m.variadic and len(args) < len(m.arglist)-1: 518583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata if len(m.arglist) > 2: 519583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata self.error(self.source,t.lineno,"Macro %s must have at least %d arguments" % (t.value, len(m.arglist)-1)) 520583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata else: 521583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata self.error(self.source,t.lineno,"Macro %s must have at least %d argument" % (t.value, len(m.arglist)-1)) 522583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata i = j + tokcount 523583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata else: 524583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata if m.variadic: 525583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata if len(args) == len(m.arglist)-1: 526583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata args.append([]) 527583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata else: 528583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata args[len(m.arglist)-1] = tokens[j+positions[len(m.arglist)-1]:j+tokcount-1] 529583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata del args[len(m.arglist):] 530583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata 531583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata # Get macro replacement text 532583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata rep = self.macro_expand_args(m,args) 533583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata rep = self.expand_macros(rep,expanded) 534583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata for r in rep: 535583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata r.lineno = t.lineno 536583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata tokens[i:j+tokcount] = rep 537583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata i += len(rep) 538583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata del expanded[t.value] 539583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata continue 540583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata elif t.value == '__LINE__': 541583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata t.type = self.t_INTEGER 542583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata t.value = self.t_INTEGER_TYPE(t.lineno) 543583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata 544583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata i += 1 545583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata return tokens 546583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata 547583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata # ---------------------------------------------------------------------- 548583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata # evalexpr() 549583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata # 550583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata # Evaluate an expression token sequence for the purposes of evaluating 551583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata # integral expressions. 552583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata # ---------------------------------------------------------------------- 553583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata 554583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata def evalexpr(self,tokens): 555583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata # tokens = tokenize(line) 556583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata # Search for defined macros 557583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata i = 0 558583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata while i < len(tokens): 559583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata if tokens[i].type == self.t_ID and tokens[i].value == 'defined': 560583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata j = i + 1 561583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata needparen = False 562583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata result = "0L" 563583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata while j < len(tokens): 564583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata if tokens[j].type in self.t_WS: 565583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata j += 1 566583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata continue 567583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata elif tokens[j].type == self.t_ID: 568583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata if tokens[j].value in self.macros: 569583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata result = "1L" 570583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata else: 571583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata result = "0L" 572583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata if not needparen: break 573583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata elif tokens[j].value == '(': 574583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata needparen = True 575583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata elif tokens[j].value == ')': 576583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata break 577583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata else: 578583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata self.error(self.source,tokens[i].lineno,"Malformed defined()") 579583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata j += 1 580583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata tokens[i].type = self.t_INTEGER 581583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata tokens[i].value = self.t_INTEGER_TYPE(result) 582583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata del tokens[i+1:j+1] 583583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata i += 1 584583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata tokens = self.expand_macros(tokens) 585583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata for i,t in enumerate(tokens): 586583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata if t.type == self.t_ID: 587583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata tokens[i] = copy.copy(t) 588583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata tokens[i].type = self.t_INTEGER 589583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata tokens[i].value = self.t_INTEGER_TYPE("0L") 590583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata elif t.type == self.t_INTEGER: 591583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata tokens[i] = copy.copy(t) 592583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata # Strip off any trailing suffixes 593583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata tokens[i].value = str(tokens[i].value) 594583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata while tokens[i].value[-1] not in "0123456789abcdefABCDEF": 595583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata tokens[i].value = tokens[i].value[:-1] 596583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata 597583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata expr = "".join([str(x.value) for x in tokens]) 598583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata expr = expr.replace("&&"," and ") 599583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata expr = expr.replace("||"," or ") 600583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata expr = expr.replace("!"," not ") 601583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata try: 602583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata result = eval(expr) 603583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata except Exception: 604583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata self.error(self.source,tokens[0].lineno,"Couldn't evaluate expression") 605583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata result = 0 606583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata return result 607583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata 608583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata # ---------------------------------------------------------------------- 609583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata # parsegen() 610583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata # 611583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata # Parse an input string/ 612583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata # ---------------------------------------------------------------------- 613583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata def parsegen(self,input,source=None): 614583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata 615583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata # Replace trigraph sequences 616583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata t = trigraph(input) 617583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata lines = self.group_lines(t) 618583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata 619583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata if not source: 620583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata source = "" 621583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata 622583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata self.define("__FILE__ \"%s\"" % source) 623583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata 624583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata self.source = source 625583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata chunk = [] 626583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata enable = True 627583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata iftrigger = False 628583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata ifstack = [] 629583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata 630583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata for x in lines: 631583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata for i,tok in enumerate(x): 632583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata if tok.type not in self.t_WS: break 633583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata if tok.value == '#': 634583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata # Preprocessor directive 635583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata 636583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata # insert necessary whitespace instead of eaten tokens 637583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata for tok in x: 638583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata if tok.type in self.t_WS and '\n' in tok.value: 639583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata chunk.append(tok) 640583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata 641583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata dirtokens = self.tokenstrip(x[i+1:]) 642583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata if dirtokens: 643583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata name = dirtokens[0].value 644583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata args = self.tokenstrip(dirtokens[1:]) 645583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata else: 646583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata name = "" 647583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata args = [] 648583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata 649583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata if name == 'define': 650583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata if enable: 651583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata for tok in self.expand_macros(chunk): 652583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata yield tok 653583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata chunk = [] 654583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata self.define(args) 655583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata elif name == 'include': 656583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata if enable: 657583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata for tok in self.expand_macros(chunk): 658583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata yield tok 659583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata chunk = [] 660583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata oldfile = self.macros['__FILE__'] 661583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata for tok in self.include(args): 662583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata yield tok 663583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata self.macros['__FILE__'] = oldfile 664583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata self.source = source 665583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata elif name == 'undef': 666583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata if enable: 667583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata for tok in self.expand_macros(chunk): 668583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata yield tok 669583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata chunk = [] 670583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata self.undef(args) 671583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata elif name == 'ifdef': 672583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata ifstack.append((enable,iftrigger)) 673583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata if enable: 674583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata if not args[0].value in self.macros: 675583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata enable = False 676583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata iftrigger = False 677583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata else: 678583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata iftrigger = True 679583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata elif name == 'ifndef': 680583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata ifstack.append((enable,iftrigger)) 681583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata if enable: 682583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata if args[0].value in self.macros: 683583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata enable = False 684583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata iftrigger = False 685583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata else: 686583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata iftrigger = True 687583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata elif name == 'if': 688583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata ifstack.append((enable,iftrigger)) 689583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata if enable: 690583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata result = self.evalexpr(args) 691583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata if not result: 692583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata enable = False 693583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata iftrigger = False 694583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata else: 695583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata iftrigger = True 696583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata elif name == 'elif': 697583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata if ifstack: 698583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata if ifstack[-1][0]: # We only pay attention if outer "if" allows this 699583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata if enable: # If already true, we flip enable False 700583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata enable = False 701583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata elif not iftrigger: # If False, but not triggered yet, we'll check expression 702583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata result = self.evalexpr(args) 703583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata if result: 704583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata enable = True 705583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata iftrigger = True 706583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata else: 707583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata self.error(self.source,dirtokens[0].lineno,"Misplaced #elif") 708583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata 709583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata elif name == 'else': 710583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata if ifstack: 711583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata if ifstack[-1][0]: 712583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata if enable: 713583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata enable = False 714583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata elif not iftrigger: 715583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata enable = True 716583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata iftrigger = True 717583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata else: 718583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata self.error(self.source,dirtokens[0].lineno,"Misplaced #else") 719583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata 720583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata elif name == 'endif': 721583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata if ifstack: 722583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata enable,iftrigger = ifstack.pop() 723583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata else: 724583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata self.error(self.source,dirtokens[0].lineno,"Misplaced #endif") 725583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata else: 726583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata # Unknown preprocessor directive 727583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata pass 728583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata 729583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata else: 730583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata # Normal text 731583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata if enable: 732583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata chunk.extend(x) 733583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata 734583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata for tok in self.expand_macros(chunk): 735583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata yield tok 736583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata chunk = [] 737583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata 738583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata # ---------------------------------------------------------------------- 739583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata # include() 740583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata # 741583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata # Implementation of file-inclusion 742583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata # ---------------------------------------------------------------------- 743583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata 744583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata def include(self,tokens): 745583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata # Try to extract the filename and then process an include file 746583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata if not tokens: 747583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata return 748583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata if tokens: 749583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata if tokens[0].value != '<' and tokens[0].type != self.t_STRING: 750583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata tokens = self.expand_macros(tokens) 751583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata 752583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata if tokens[0].value == '<': 753583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata # Include <...> 754583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata i = 1 755583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata while i < len(tokens): 756583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata if tokens[i].value == '>': 757583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata break 758583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata i += 1 759583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata else: 760583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata print("Malformed #include <...>") 761583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata return 762583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata filename = "".join([x.value for x in tokens[1:i]]) 763583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata path = self.path + [""] + self.temp_path 764583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata elif tokens[0].type == self.t_STRING: 765583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata filename = tokens[0].value[1:-1] 766583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata path = self.temp_path + [""] + self.path 767583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata else: 768583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata print("Malformed #include statement") 769583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata return 770583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata for p in path: 771583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata iname = os.path.join(p,filename) 772583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata try: 773583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata data = open(iname,"r").read() 774583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata dname = os.path.dirname(iname) 775583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata if dname: 776583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata self.temp_path.insert(0,dname) 777583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata for tok in self.parsegen(data,filename): 778583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata yield tok 779583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata if dname: 780583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata del self.temp_path[0] 781583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata break 782583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata except IOError: 783583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata pass 784583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata else: 785583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata print("Couldn't find '%s'" % filename) 786583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata 787583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata # ---------------------------------------------------------------------- 788583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata # define() 789583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata # 790583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata # Define a new macro 791583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata # ---------------------------------------------------------------------- 792583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata 793583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata def define(self,tokens): 794583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata if isinstance(tokens,STRING_TYPES): 795583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata tokens = self.tokenize(tokens) 796583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata 797583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata linetok = tokens 798583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata try: 799583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata name = linetok[0] 800583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata if len(linetok) > 1: 801583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata mtype = linetok[1] 802583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata else: 803583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata mtype = None 804583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata if not mtype: 805583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata m = Macro(name.value,[]) 806583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata self.macros[name.value] = m 807583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata elif mtype.type in self.t_WS: 808583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata # A normal macro 809583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata m = Macro(name.value,self.tokenstrip(linetok[2:])) 810583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata self.macros[name.value] = m 811583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata elif mtype.value == '(': 812583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata # A macro with arguments 813583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata tokcount, args, positions = self.collect_args(linetok[1:]) 814583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata variadic = False 815583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata for a in args: 816583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata if variadic: 817583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata print("No more arguments may follow a variadic argument") 818583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata break 819583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata astr = "".join([str(_i.value) for _i in a]) 820583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata if astr == "...": 821583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata variadic = True 822583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata a[0].type = self.t_ID 823583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata a[0].value = '__VA_ARGS__' 824583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata variadic = True 825583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata del a[1:] 826583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata continue 827583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata elif astr[-3:] == "..." and a[0].type == self.t_ID: 828583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata variadic = True 829583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata del a[1:] 830583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata # If, for some reason, "." is part of the identifier, strip off the name for the purposes 831583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata # of macro expansion 832583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata if a[0].value[-3:] == '...': 833583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata a[0].value = a[0].value[:-3] 834583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata continue 835583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata if len(a) > 1 or a[0].type != self.t_ID: 836583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata print("Invalid macro argument") 837583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata break 838583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata else: 839583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata mvalue = self.tokenstrip(linetok[1+tokcount:]) 840583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata i = 0 841583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata while i < len(mvalue): 842583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata if i+1 < len(mvalue): 843583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata if mvalue[i].type in self.t_WS and mvalue[i+1].value == '##': 844583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata del mvalue[i] 845583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata continue 846583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata elif mvalue[i].value == '##' and mvalue[i+1].type in self.t_WS: 847583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata del mvalue[i+1] 848583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata i += 1 849583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata m = Macro(name.value,mvalue,[x[0].value for x in args],variadic) 850583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata self.macro_prescan(m) 851583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata self.macros[name.value] = m 852583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata else: 853583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata print("Bad macro definition") 854583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata except LookupError: 855583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata print("Bad macro definition") 856583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata 857583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata # ---------------------------------------------------------------------- 858583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata # undef() 859583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata # 860583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata # Undefine a macro 861583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata # ---------------------------------------------------------------------- 862583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata 863583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata def undef(self,tokens): 864583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata id = tokens[0].value 865583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata try: 866583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata del self.macros[id] 867583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata except LookupError: 868583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata pass 869583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata 870583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata # ---------------------------------------------------------------------- 871583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata # parse() 872583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata # 873583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata # Parse input text. 874583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata # ---------------------------------------------------------------------- 875583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata def parse(self,input,source=None,ignore={}): 876583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata self.ignore = ignore 877583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata self.parser = self.parsegen(input,source) 878583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata 879583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata # ---------------------------------------------------------------------- 880583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata # token() 881583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata # 882583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata # Method to return individual tokens 883583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata # ---------------------------------------------------------------------- 884583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata def token(self): 885583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata try: 886583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata while True: 887583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata tok = next(self.parser) 888583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata if tok.type not in self.ignore: return tok 889583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata except StopIteration: 890583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata self.parser = None 891583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata return None 892583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata 893583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granataif __name__ == '__main__': 894583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata import ply.lex as lex 895583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata lexer = lex.lex() 896583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata 897583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata # Run a preprocessor 898583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata import sys 899583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata f = open(sys.argv[1]) 900583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata input = f.read() 901583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata 902583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata p = Preprocessor(lexer) 903583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata p.parse(input,sys.argv[1]) 904583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata while True: 905583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata tok = p.token() 906583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata if not tok: break 907583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata print(p.source, tok) 908583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata 909583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata 910583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata 911583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata 912583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata 913583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata 914583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata 915583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata 916583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata 917583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata 918583d33c593896afeb8486a25fabfcf6e9dc9ca75Enrico Granata 919