146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan#!/usr/bin/env python 246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan# 346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan# Copyright 2008, Google Inc. 446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan# All rights reserved. 546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan# 646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan# Redistribution and use in source and binary forms, with or without 746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan# modification, are permitted provided that the following conditions are 846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan# met: 946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan# 1046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan# * Redistributions of source code must retain the above copyright 1146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan# notice, this list of conditions and the following disclaimer. 1246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan# * Redistributions in binary form must reproduce the above 1346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan# copyright notice, this list of conditions and the following disclaimer 1446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan# in the documentation and/or other materials provided with the 1546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan# distribution. 1646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan# * Neither the name of Google Inc. nor the names of its 1746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan# contributors may be used to endorse or promote products derived from 1846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan# this software without specific prior written permission. 1946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan# 2046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 2146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 2246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 2346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 2446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 2546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 2646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 2746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 2846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 2946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 3046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 3146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan 3246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan"""pump v0.2.0 - Pretty Useful for Meta Programming. 3346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan 3446108a219a4b812dd8f36fee479a0340ea5963f5Ben ChanA tool for preprocessor meta programming. Useful for generating 3546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chanrepetitive boilerplate code. Especially useful for writing C++ 3646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chanclasses, functions, macros, and templates that need to work with 3746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chanvarious number of arguments. 3846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan 3946108a219a4b812dd8f36fee479a0340ea5963f5Ben ChanUSAGE: 4046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan pump.py SOURCE_FILE 4146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan 4246108a219a4b812dd8f36fee479a0340ea5963f5Ben ChanEXAMPLES: 4346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan pump.py foo.cc.pump 4446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan Converts foo.cc.pump to foo.cc. 4546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan 4646108a219a4b812dd8f36fee479a0340ea5963f5Ben ChanGRAMMAR: 4746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan CODE ::= ATOMIC_CODE* 4846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan ATOMIC_CODE ::= $var ID = EXPRESSION 4946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan | $var ID = [[ CODE ]] 5046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan | $range ID EXPRESSION..EXPRESSION 5146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan | $for ID SEPARATOR [[ CODE ]] 5246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan | $($) 5346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan | $ID 5446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan | $(EXPRESSION) 5546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan | $if EXPRESSION [[ CODE ]] ELSE_BRANCH 5646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan | [[ CODE ]] 5746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan | RAW_CODE 5846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan SEPARATOR ::= RAW_CODE | EMPTY 5946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan ELSE_BRANCH ::= $else [[ CODE ]] 6046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan | $elif EXPRESSION [[ CODE ]] ELSE_BRANCH 6146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan | EMPTY 6246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan EXPRESSION has Python syntax. 6346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan""" 6446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan 6546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan__author__ = 'wan@google.com (Zhanyong Wan)' 6646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan 6746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chanimport os 6846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chanimport re 6946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chanimport sys 7046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan 7146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan 7246108a219a4b812dd8f36fee479a0340ea5963f5Ben ChanTOKEN_TABLE = [ 7346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan (re.compile(r'\$var\s+'), '$var'), 7446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan (re.compile(r'\$elif\s+'), '$elif'), 7546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan (re.compile(r'\$else\s+'), '$else'), 7646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan (re.compile(r'\$for\s+'), '$for'), 7746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan (re.compile(r'\$if\s+'), '$if'), 7846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan (re.compile(r'\$range\s+'), '$range'), 7946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan (re.compile(r'\$[_A-Za-z]\w*'), '$id'), 8046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan (re.compile(r'\$\(\$\)'), '$($)'), 8146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan (re.compile(r'\$'), '$'), 8246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan (re.compile(r'\[\[\n?'), '[['), 8346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan (re.compile(r'\]\]\n?'), ']]'), 8446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan ] 8546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan 8646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan 8746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chanclass Cursor: 8846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan """Represents a position (line and column) in a text file.""" 8946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan 9046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan def __init__(self, line=-1, column=-1): 9146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan self.line = line 9246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan self.column = column 9346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan 9446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan def __eq__(self, rhs): 9546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan return self.line == rhs.line and self.column == rhs.column 9646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan 9746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan def __ne__(self, rhs): 9846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan return not self == rhs 9946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan 10046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan def __lt__(self, rhs): 10146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan return self.line < rhs.line or ( 10246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan self.line == rhs.line and self.column < rhs.column) 10346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan 10446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan def __le__(self, rhs): 10546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan return self < rhs or self == rhs 10646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan 10746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan def __gt__(self, rhs): 10846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan return rhs < self 10946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan 11046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan def __ge__(self, rhs): 11146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan return rhs <= self 11246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan 11346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan def __str__(self): 11446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan if self == Eof(): 11546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan return 'EOF' 11646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan else: 11746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan return '%s(%s)' % (self.line + 1, self.column) 11846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan 11946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan def __add__(self, offset): 12046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan return Cursor(self.line, self.column + offset) 12146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan 12246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan def __sub__(self, offset): 12346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan return Cursor(self.line, self.column - offset) 12446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan 12546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan def Clone(self): 12646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan """Returns a copy of self.""" 12746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan 12846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan return Cursor(self.line, self.column) 12946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan 13046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan 13146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan# Special cursor to indicate the end-of-file. 13246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chandef Eof(): 13346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan """Returns the special cursor to denote the end-of-file.""" 13446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan return Cursor(-1, -1) 13546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan 13646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan 13746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chanclass Token: 13846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan """Represents a token in a Pump source file.""" 13946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan 14046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan def __init__(self, start=None, end=None, value=None, token_type=None): 14146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan if start is None: 14246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan self.start = Eof() 14346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan else: 14446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan self.start = start 14546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan if end is None: 14646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan self.end = Eof() 14746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan else: 14846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan self.end = end 14946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan self.value = value 15046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan self.token_type = token_type 15146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan 15246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan def __str__(self): 15346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan return 'Token @%s: \'%s\' type=%s' % ( 15446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan self.start, self.value, self.token_type) 15546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan 15646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan def Clone(self): 15746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan """Returns a copy of self.""" 15846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan 15946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan return Token(self.start.Clone(), self.end.Clone(), self.value, 16046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan self.token_type) 16146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan 16246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan 16346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chandef StartsWith(lines, pos, string): 16446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan """Returns True iff the given position in lines starts with 'string'.""" 16546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan 16646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan return lines[pos.line][pos.column:].startswith(string) 16746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan 16846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan 16946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chandef FindFirstInLine(line, token_table): 17046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan best_match_start = -1 17146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan for (regex, token_type) in token_table: 17246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan m = regex.search(line) 17346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan if m: 17446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan # We found regex in lines 17546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan if best_match_start < 0 or m.start() < best_match_start: 17646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan best_match_start = m.start() 17746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan best_match_length = m.end() - m.start() 17846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan best_match_token_type = token_type 17946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan 18046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan if best_match_start < 0: 18146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan return None 18246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan 18346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan return (best_match_start, best_match_length, best_match_token_type) 18446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan 18546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan 18646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chandef FindFirst(lines, token_table, cursor): 18746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan """Finds the first occurrence of any string in strings in lines.""" 18846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan 18946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan start = cursor.Clone() 19046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan cur_line_number = cursor.line 19146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan for line in lines[start.line:]: 19246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan if cur_line_number == start.line: 19346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan line = line[start.column:] 19446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan m = FindFirstInLine(line, token_table) 19546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan if m: 19646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan # We found a regex in line. 19746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan (start_column, length, token_type) = m 19846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan if cur_line_number == start.line: 19946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan start_column += start.column 20046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan found_start = Cursor(cur_line_number, start_column) 20146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan found_end = found_start + length 20246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan return MakeToken(lines, found_start, found_end, token_type) 20346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan cur_line_number += 1 20446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan # We failed to find str in lines 20546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan return None 20646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan 20746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan 20846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chandef SubString(lines, start, end): 20946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan """Returns a substring in lines.""" 21046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan 21146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan if end == Eof(): 21246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan end = Cursor(len(lines) - 1, len(lines[-1])) 21346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan 21446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan if start >= end: 21546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan return '' 21646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan 21746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan if start.line == end.line: 21846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan return lines[start.line][start.column:end.column] 21946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan 22046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan result_lines = ([lines[start.line][start.column:]] + 22146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan lines[start.line + 1:end.line] + 22246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan [lines[end.line][:end.column]]) 22346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan return ''.join(result_lines) 22446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan 22546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan 22646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chandef StripMetaComments(str): 22746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan """Strip meta comments from each line in the given string.""" 22846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan 22946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan # First, completely remove lines containing nothing but a meta 23046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan # comment, including the trailing \n. 23146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan str = re.sub(r'^\s*\$\$.*\n', '', str) 23246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan 23346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan # Then, remove meta comments from contentful lines. 23446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan return re.sub(r'\s*\$\$.*', '', str) 23546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan 23646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan 23746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chandef MakeToken(lines, start, end, token_type): 23846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan """Creates a new instance of Token.""" 23946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan 24046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan return Token(start, end, SubString(lines, start, end), token_type) 24146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan 24246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan 24346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chandef ParseToken(lines, pos, regex, token_type): 24446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan line = lines[pos.line][pos.column:] 24546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan m = regex.search(line) 24646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan if m and not m.start(): 24746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan return MakeToken(lines, pos, pos + m.end(), token_type) 24846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan else: 24946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan print 'ERROR: %s expected at %s.' % (token_type, pos) 25046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan sys.exit(1) 25146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan 25246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan 25346108a219a4b812dd8f36fee479a0340ea5963f5Ben ChanID_REGEX = re.compile(r'[_A-Za-z]\w*') 25446108a219a4b812dd8f36fee479a0340ea5963f5Ben ChanEQ_REGEX = re.compile(r'=') 25546108a219a4b812dd8f36fee479a0340ea5963f5Ben ChanREST_OF_LINE_REGEX = re.compile(r'.*?(?=$|\$\$)') 25646108a219a4b812dd8f36fee479a0340ea5963f5Ben ChanOPTIONAL_WHITE_SPACES_REGEX = re.compile(r'\s*') 25746108a219a4b812dd8f36fee479a0340ea5963f5Ben ChanWHITE_SPACE_REGEX = re.compile(r'\s') 25846108a219a4b812dd8f36fee479a0340ea5963f5Ben ChanDOT_DOT_REGEX = re.compile(r'\.\.') 25946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan 26046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan 26146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chandef Skip(lines, pos, regex): 26246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan line = lines[pos.line][pos.column:] 26346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan m = re.search(regex, line) 26446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan if m and not m.start(): 26546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan return pos + m.end() 26646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan else: 26746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan return pos 26846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan 26946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan 27046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chandef SkipUntil(lines, pos, regex, token_type): 27146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan line = lines[pos.line][pos.column:] 27246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan m = re.search(regex, line) 27346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan if m: 27446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan return pos + m.start() 27546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan else: 27646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan print ('ERROR: %s expected on line %s after column %s.' % 27746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan (token_type, pos.line + 1, pos.column)) 27846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan sys.exit(1) 27946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan 28046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan 28146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chandef ParseExpTokenInParens(lines, pos): 28246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan def ParseInParens(pos): 28346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan pos = Skip(lines, pos, OPTIONAL_WHITE_SPACES_REGEX) 28446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan pos = Skip(lines, pos, r'\(') 28546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan pos = Parse(pos) 28646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan pos = Skip(lines, pos, r'\)') 28746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan return pos 28846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan 28946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan def Parse(pos): 29046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan pos = SkipUntil(lines, pos, r'\(|\)', ')') 29146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan if SubString(lines, pos, pos + 1) == '(': 29246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan pos = Parse(pos + 1) 29346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan pos = Skip(lines, pos, r'\)') 29446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan return Parse(pos) 29546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan else: 29646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan return pos 29746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan 29846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan start = pos.Clone() 29946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan pos = ParseInParens(pos) 30046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan return MakeToken(lines, start, pos, 'exp') 30146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan 30246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan 30346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chandef RStripNewLineFromToken(token): 30446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan if token.value.endswith('\n'): 30546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan return Token(token.start, token.end, token.value[:-1], token.token_type) 30646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan else: 30746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan return token 30846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan 30946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan 31046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chandef TokenizeLines(lines, pos): 31146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan while True: 31246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan found = FindFirst(lines, TOKEN_TABLE, pos) 31346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan if not found: 31446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan yield MakeToken(lines, pos, Eof(), 'code') 31546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan return 31646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan 31746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan if found.start == pos: 31846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan prev_token = None 31946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan prev_token_rstripped = None 32046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan else: 32146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan prev_token = MakeToken(lines, pos, found.start, 'code') 32246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan prev_token_rstripped = RStripNewLineFromToken(prev_token) 32346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan 32446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan if found.token_type == '$var': 32546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan if prev_token_rstripped: 32646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan yield prev_token_rstripped 32746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan yield found 32846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan id_token = ParseToken(lines, found.end, ID_REGEX, 'id') 32946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan yield id_token 33046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan pos = Skip(lines, id_token.end, OPTIONAL_WHITE_SPACES_REGEX) 33146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan 33246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan eq_token = ParseToken(lines, pos, EQ_REGEX, '=') 33346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan yield eq_token 33446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan pos = Skip(lines, eq_token.end, r'\s*') 33546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan 33646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan if SubString(lines, pos, pos + 2) != '[[': 33746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan exp_token = ParseToken(lines, pos, REST_OF_LINE_REGEX, 'exp') 33846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan yield exp_token 33946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan pos = Cursor(exp_token.end.line + 1, 0) 34046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan elif found.token_type == '$for': 34146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan if prev_token_rstripped: 34246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan yield prev_token_rstripped 34346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan yield found 34446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan id_token = ParseToken(lines, found.end, ID_REGEX, 'id') 34546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan yield id_token 34646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan pos = Skip(lines, id_token.end, WHITE_SPACE_REGEX) 34746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan elif found.token_type == '$range': 34846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan if prev_token_rstripped: 34946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan yield prev_token_rstripped 35046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan yield found 35146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan id_token = ParseToken(lines, found.end, ID_REGEX, 'id') 35246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan yield id_token 35346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan pos = Skip(lines, id_token.end, OPTIONAL_WHITE_SPACES_REGEX) 35446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan 35546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan dots_pos = SkipUntil(lines, pos, DOT_DOT_REGEX, '..') 35646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan yield MakeToken(lines, pos, dots_pos, 'exp') 35746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan yield MakeToken(lines, dots_pos, dots_pos + 2, '..') 35846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan pos = dots_pos + 2 35946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan new_pos = Cursor(pos.line + 1, 0) 36046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan yield MakeToken(lines, pos, new_pos, 'exp') 36146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan pos = new_pos 36246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan elif found.token_type == '$': 36346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan if prev_token: 36446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan yield prev_token 36546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan yield found 36646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan exp_token = ParseExpTokenInParens(lines, found.end) 36746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan yield exp_token 36846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan pos = exp_token.end 36946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan elif (found.token_type == ']]' or found.token_type == '$if' or 37046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan found.token_type == '$elif' or found.token_type == '$else'): 37146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan if prev_token_rstripped: 37246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan yield prev_token_rstripped 37346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan yield found 37446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan pos = found.end 37546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan else: 37646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan if prev_token: 37746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan yield prev_token 37846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan yield found 37946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan pos = found.end 38046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan 38146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan 38246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chandef Tokenize(s): 38346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan """A generator that yields the tokens in the given string.""" 38446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan if s != '': 38546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan lines = s.splitlines(True) 38646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan for token in TokenizeLines(lines, Cursor(0, 0)): 38746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan yield token 38846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan 38946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan 39046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chanclass CodeNode: 39146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan def __init__(self, atomic_code_list=None): 39246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan self.atomic_code = atomic_code_list 39346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan 39446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan 39546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chanclass VarNode: 39646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan def __init__(self, identifier=None, atomic_code=None): 39746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan self.identifier = identifier 39846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan self.atomic_code = atomic_code 39946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan 40046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan 40146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chanclass RangeNode: 40246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan def __init__(self, identifier=None, exp1=None, exp2=None): 40346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan self.identifier = identifier 40446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan self.exp1 = exp1 40546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan self.exp2 = exp2 40646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan 40746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan 40846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chanclass ForNode: 40946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan def __init__(self, identifier=None, sep=None, code=None): 41046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan self.identifier = identifier 41146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan self.sep = sep 41246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan self.code = code 41346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan 41446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan 41546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chanclass ElseNode: 41646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan def __init__(self, else_branch=None): 41746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan self.else_branch = else_branch 41846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan 41946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan 42046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chanclass IfNode: 42146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan def __init__(self, exp=None, then_branch=None, else_branch=None): 42246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan self.exp = exp 42346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan self.then_branch = then_branch 42446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan self.else_branch = else_branch 42546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan 42646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan 42746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chanclass RawCodeNode: 42846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan def __init__(self, token=None): 42946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan self.raw_code = token 43046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan 43146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan 43246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chanclass LiteralDollarNode: 43346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan def __init__(self, token): 43446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan self.token = token 43546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan 43646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan 43746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chanclass ExpNode: 43846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan def __init__(self, token, python_exp): 43946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan self.token = token 44046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan self.python_exp = python_exp 44146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan 44246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan 44346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chandef PopFront(a_list): 44446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan head = a_list[0] 44546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan a_list[:1] = [] 44646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan return head 44746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan 44846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan 44946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chandef PushFront(a_list, elem): 45046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan a_list[:0] = [elem] 45146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan 45246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan 45346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chandef PopToken(a_list, token_type=None): 45446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan token = PopFront(a_list) 45546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan if token_type is not None and token.token_type != token_type: 45646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan print 'ERROR: %s expected at %s' % (token_type, token.start) 45746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan print 'ERROR: %s found instead' % (token,) 45846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan sys.exit(1) 45946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan 46046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan return token 46146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan 46246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan 46346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chandef PeekToken(a_list): 46446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan if not a_list: 46546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan return None 46646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan 46746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan return a_list[0] 46846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan 46946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan 47046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chandef ParseExpNode(token): 47146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan python_exp = re.sub(r'([_A-Za-z]\w*)', r'self.GetValue("\1")', token.value) 47246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan return ExpNode(token, python_exp) 47346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan 47446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan 47546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chandef ParseElseNode(tokens): 47646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan def Pop(token_type=None): 47746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan return PopToken(tokens, token_type) 47846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan 47946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan next = PeekToken(tokens) 48046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan if not next: 48146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan return None 48246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan if next.token_type == '$else': 48346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan Pop('$else') 48446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan Pop('[[') 48546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan code_node = ParseCodeNode(tokens) 48646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan Pop(']]') 48746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan return code_node 48846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan elif next.token_type == '$elif': 48946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan Pop('$elif') 49046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan exp = Pop('code') 49146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan Pop('[[') 49246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan code_node = ParseCodeNode(tokens) 49346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan Pop(']]') 49446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan inner_else_node = ParseElseNode(tokens) 49546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan return CodeNode([IfNode(ParseExpNode(exp), code_node, inner_else_node)]) 49646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan elif not next.value.strip(): 49746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan Pop('code') 49846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan return ParseElseNode(tokens) 49946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan else: 50046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan return None 50146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan 50246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan 50346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chandef ParseAtomicCodeNode(tokens): 50446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan def Pop(token_type=None): 50546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan return PopToken(tokens, token_type) 50646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan 50746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan head = PopFront(tokens) 50846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan t = head.token_type 50946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan if t == 'code': 51046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan return RawCodeNode(head) 51146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan elif t == '$var': 51246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan id_token = Pop('id') 51346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan Pop('=') 51446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan next = PeekToken(tokens) 51546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan if next.token_type == 'exp': 51646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan exp_token = Pop() 51746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan return VarNode(id_token, ParseExpNode(exp_token)) 51846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan Pop('[[') 51946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan code_node = ParseCodeNode(tokens) 52046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan Pop(']]') 52146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan return VarNode(id_token, code_node) 52246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan elif t == '$for': 52346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan id_token = Pop('id') 52446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan next_token = PeekToken(tokens) 52546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan if next_token.token_type == 'code': 52646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan sep_token = next_token 52746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan Pop('code') 52846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan else: 52946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan sep_token = None 53046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan Pop('[[') 53146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan code_node = ParseCodeNode(tokens) 53246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan Pop(']]') 53346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan return ForNode(id_token, sep_token, code_node) 53446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan elif t == '$if': 53546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan exp_token = Pop('code') 53646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan Pop('[[') 53746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan code_node = ParseCodeNode(tokens) 53846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan Pop(']]') 53946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan else_node = ParseElseNode(tokens) 54046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan return IfNode(ParseExpNode(exp_token), code_node, else_node) 54146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan elif t == '$range': 54246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan id_token = Pop('id') 54346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan exp1_token = Pop('exp') 54446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan Pop('..') 54546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan exp2_token = Pop('exp') 54646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan return RangeNode(id_token, ParseExpNode(exp1_token), 54746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan ParseExpNode(exp2_token)) 54846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan elif t == '$id': 54946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan return ParseExpNode(Token(head.start + 1, head.end, head.value[1:], 'id')) 55046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan elif t == '$($)': 55146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan return LiteralDollarNode(head) 55246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan elif t == '$': 55346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan exp_token = Pop('exp') 55446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan return ParseExpNode(exp_token) 55546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan elif t == '[[': 55646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan code_node = ParseCodeNode(tokens) 55746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan Pop(']]') 55846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan return code_node 55946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan else: 56046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan PushFront(tokens, head) 56146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan return None 56246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan 56346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan 56446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chandef ParseCodeNode(tokens): 56546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan atomic_code_list = [] 56646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan while True: 56746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan if not tokens: 56846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan break 56946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan atomic_code_node = ParseAtomicCodeNode(tokens) 57046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan if atomic_code_node: 57146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan atomic_code_list.append(atomic_code_node) 57246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan else: 57346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan break 57446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan return CodeNode(atomic_code_list) 57546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan 57646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan 57746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chandef ParseToAST(pump_src_text): 57846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan """Convert the given Pump source text into an AST.""" 57946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan tokens = list(Tokenize(pump_src_text)) 58046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan code_node = ParseCodeNode(tokens) 58146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan return code_node 58246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan 58346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan 58446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chanclass Env: 58546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan def __init__(self): 58646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan self.variables = [] 58746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan self.ranges = [] 58846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan 58946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan def Clone(self): 59046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan clone = Env() 59146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan clone.variables = self.variables[:] 59246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan clone.ranges = self.ranges[:] 59346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan return clone 59446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan 59546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan def PushVariable(self, var, value): 59646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan # If value looks like an int, store it as an int. 59746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan try: 59846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan int_value = int(value) 59946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan if ('%s' % int_value) == value: 60046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan value = int_value 60146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan except Exception: 60246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan pass 60346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan self.variables[:0] = [(var, value)] 60446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan 60546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan def PopVariable(self): 60646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan self.variables[:1] = [] 60746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan 60846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan def PushRange(self, var, lower, upper): 60946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan self.ranges[:0] = [(var, lower, upper)] 61046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan 61146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan def PopRange(self): 61246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan self.ranges[:1] = [] 61346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan 61446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan def GetValue(self, identifier): 61546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan for (var, value) in self.variables: 61646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan if identifier == var: 61746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan return value 61846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan 61946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan print 'ERROR: meta variable %s is undefined.' % (identifier,) 62046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan sys.exit(1) 62146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan 62246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan def EvalExp(self, exp): 62346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan try: 62446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan result = eval(exp.python_exp) 62546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan except Exception, e: 62646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan print 'ERROR: caught exception %s: %s' % (e.__class__.__name__, e) 62746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan print ('ERROR: failed to evaluate meta expression %s at %s' % 62846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan (exp.python_exp, exp.token.start)) 62946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan sys.exit(1) 63046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan return result 63146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan 63246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan def GetRange(self, identifier): 63346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan for (var, lower, upper) in self.ranges: 63446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan if identifier == var: 63546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan return (lower, upper) 63646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan 63746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan print 'ERROR: range %s is undefined.' % (identifier,) 63846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan sys.exit(1) 63946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan 64046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan 64146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chanclass Output: 64246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan def __init__(self): 64346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan self.string = '' 64446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan 64546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan def GetLastLine(self): 64646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan index = self.string.rfind('\n') 64746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan if index < 0: 64846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan return '' 64946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan 65046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan return self.string[index + 1:] 65146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan 65246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan def Append(self, s): 65346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan self.string += s 65446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan 65546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan 65646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chandef RunAtomicCode(env, node, output): 65746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan if isinstance(node, VarNode): 65846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan identifier = node.identifier.value.strip() 65946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan result = Output() 66046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan RunAtomicCode(env.Clone(), node.atomic_code, result) 66146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan value = result.string 66246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan env.PushVariable(identifier, value) 66346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan elif isinstance(node, RangeNode): 66446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan identifier = node.identifier.value.strip() 66546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan lower = int(env.EvalExp(node.exp1)) 66646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan upper = int(env.EvalExp(node.exp2)) 66746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan env.PushRange(identifier, lower, upper) 66846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan elif isinstance(node, ForNode): 66946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan identifier = node.identifier.value.strip() 67046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan if node.sep is None: 67146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan sep = '' 67246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan else: 67346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan sep = node.sep.value 67446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan (lower, upper) = env.GetRange(identifier) 67546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan for i in range(lower, upper + 1): 67646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan new_env = env.Clone() 67746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan new_env.PushVariable(identifier, i) 67846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan RunCode(new_env, node.code, output) 67946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan if i != upper: 68046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan output.Append(sep) 68146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan elif isinstance(node, RawCodeNode): 68246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan output.Append(node.raw_code.value) 68346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan elif isinstance(node, IfNode): 68446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan cond = env.EvalExp(node.exp) 68546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan if cond: 68646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan RunCode(env.Clone(), node.then_branch, output) 68746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan elif node.else_branch is not None: 68846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan RunCode(env.Clone(), node.else_branch, output) 68946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan elif isinstance(node, ExpNode): 69046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan value = env.EvalExp(node) 69146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan output.Append('%s' % (value,)) 69246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan elif isinstance(node, LiteralDollarNode): 69346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan output.Append('$') 69446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan elif isinstance(node, CodeNode): 69546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan RunCode(env.Clone(), node, output) 69646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan else: 69746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan print 'BAD' 69846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan print node 69946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan sys.exit(1) 70046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan 70146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan 70246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chandef RunCode(env, code_node, output): 70346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan for atomic_code in code_node.atomic_code: 70446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan RunAtomicCode(env, atomic_code, output) 70546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan 70646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan 70746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chandef IsComment(cur_line): 70846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan return '//' in cur_line 70946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan 71046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan 71146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chandef IsInPreprocessorDirevative(prev_lines, cur_line): 71246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan if cur_line.lstrip().startswith('#'): 71346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan return True 71446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan return prev_lines != [] and prev_lines[-1].endswith('\\') 71546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan 71646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan 71746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chandef WrapComment(line, output): 71846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan loc = line.find('//') 71946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan before_comment = line[:loc].rstrip() 72046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan if before_comment == '': 72146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan indent = loc 72246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan else: 72346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan output.append(before_comment) 72446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan indent = len(before_comment) - len(before_comment.lstrip()) 72546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan prefix = indent*' ' + '// ' 72646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan max_len = 80 - len(prefix) 72746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan comment = line[loc + 2:].strip() 72846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan segs = [seg for seg in re.split(r'(\w+\W*)', comment) if seg != ''] 72946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan cur_line = '' 73046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan for seg in segs: 73146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan if len((cur_line + seg).rstrip()) < max_len: 73246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan cur_line += seg 73346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan else: 73446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan if cur_line.strip() != '': 73546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan output.append(prefix + cur_line.rstrip()) 73646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan cur_line = seg.lstrip() 73746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan if cur_line.strip() != '': 73846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan output.append(prefix + cur_line.strip()) 73946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan 74046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan 74146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chandef WrapCode(line, line_concat, output): 74246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan indent = len(line) - len(line.lstrip()) 74346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan prefix = indent*' ' # Prefix of the current line 74446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan max_len = 80 - indent - len(line_concat) # Maximum length of the current line 74546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan new_prefix = prefix + 4*' ' # Prefix of a continuation line 74646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan new_max_len = max_len - 4 # Maximum length of a continuation line 74746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan # Prefers to wrap a line after a ',' or ';'. 74846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan segs = [seg for seg in re.split(r'([^,;]+[,;]?)', line.strip()) if seg != ''] 74946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan cur_line = '' # The current line without leading spaces. 75046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan for seg in segs: 75146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan # If the line is still too long, wrap at a space. 75246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan while cur_line == '' and len(seg.strip()) > max_len: 75346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan seg = seg.lstrip() 75446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan split_at = seg.rfind(' ', 0, max_len) 75546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan output.append(prefix + seg[:split_at].strip() + line_concat) 75646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan seg = seg[split_at + 1:] 75746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan prefix = new_prefix 75846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan max_len = new_max_len 75946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan 76046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan if len((cur_line + seg).rstrip()) < max_len: 76146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan cur_line = (cur_line + seg).lstrip() 76246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan else: 76346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan output.append(prefix + cur_line.rstrip() + line_concat) 76446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan prefix = new_prefix 76546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan max_len = new_max_len 76646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan cur_line = seg.lstrip() 76746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan if cur_line.strip() != '': 76846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan output.append(prefix + cur_line.strip()) 76946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan 77046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan 77146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chandef WrapPreprocessorDirevative(line, output): 77246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan WrapCode(line, ' \\', output) 77346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan 77446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan 77546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chandef WrapPlainCode(line, output): 77646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan WrapCode(line, '', output) 77746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan 77846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan 77946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chandef IsHeaderGuardOrInclude(line): 78046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan return (re.match(r'^#(ifndef|define|endif\s*//)\s*[\w_]+\s*$', line) or 78146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan re.match(r'^#include\s', line)) 78246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan 78346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan 78446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chandef WrapLongLine(line, output): 78546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan line = line.rstrip() 78646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan if len(line) <= 80: 78746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan output.append(line) 78846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan elif IsComment(line): 78946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan if IsHeaderGuardOrInclude(line): 79046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan # The style guide made an exception to allow long header guard lines 79146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan # and includes. 79246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan output.append(line) 79346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan else: 79446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan WrapComment(line, output) 79546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan elif IsInPreprocessorDirevative(output, line): 79646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan if IsHeaderGuardOrInclude(line): 79746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan # The style guide made an exception to allow long header guard lines 79846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan # and includes. 79946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan output.append(line) 80046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan else: 80146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan WrapPreprocessorDirevative(line, output) 80246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan else: 80346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan WrapPlainCode(line, output) 80446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan 80546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan 80646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chandef BeautifyCode(string): 80746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan lines = string.splitlines() 80846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan output = [] 80946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan for line in lines: 81046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan WrapLongLine(line, output) 81146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan output2 = [line.rstrip() for line in output] 81246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan return '\n'.join(output2) + '\n' 81346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan 81446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan 81546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chandef ConvertFromPumpSource(src_text): 81646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan """Return the text generated from the given Pump source text.""" 81746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan ast = ParseToAST(StripMetaComments(src_text)) 81846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan output = Output() 81946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan RunCode(Env(), ast, output) 82046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan return BeautifyCode(output.string) 82146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan 82246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan 82346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chandef main(argv): 82446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan if len(argv) == 1: 82546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan print __doc__ 82646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan sys.exit(1) 82746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan 82846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan file_path = argv[-1] 82946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan output_str = ConvertFromPumpSource(file(file_path, 'r').read()) 83046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan if file_path.endswith('.pump'): 83146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan output_file_path = file_path[:-5] 83246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan else: 83346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan output_file_path = '-' 83446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan if output_file_path == '-': 83546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan print output_str, 83646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan else: 83746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan output_file = file(output_file_path, 'w') 83846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan output_file.write('// This file was GENERATED by command:\n') 83946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan output_file.write('// %s %s\n' % 84046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan (os.path.basename(__file__), os.path.basename(file_path))) 84146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan output_file.write('// DO NOT EDIT BY HAND!!!\n\n') 84246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan output_file.write(output_str) 84346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan output_file.close() 84446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan 84546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan 84646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chanif __name__ == '__main__': 84746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan main(sys.argv) 848