141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot#!/usr/bin/env python 241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot# 341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot# Copyright 2008, Google Inc. 441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot# All rights reserved. 541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot# 641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot# Redistribution and use in source and binary forms, with or without 741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot# modification, are permitted provided that the following conditions are 841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot# met: 941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot# 1041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot# * Redistributions of source code must retain the above copyright 1141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot# notice, this list of conditions and the following disclaimer. 1241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot# * Redistributions in binary form must reproduce the above 1341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot# copyright notice, this list of conditions and the following disclaimer 1441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot# in the documentation and/or other materials provided with the 1541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot# distribution. 1641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot# * Neither the name of Google Inc. nor the names of its 1741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot# contributors may be used to endorse or promote products derived from 1841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot# this software without specific prior written permission. 1941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot# 2041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 2141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 2241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 2341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 2441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 2541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 2641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 2741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 2841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 2941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 3041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 3141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot 3241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot"""pump v0.2.0 - Pretty Useful for Meta Programming. 3341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot 3441d0579e8de9ef4ff178fc4991043c61a19943f7Brett ChabotA tool for preprocessor meta programming. Useful for generating 3541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabotrepetitive boilerplate code. Especially useful for writing C++ 3641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabotclasses, functions, macros, and templates that need to work with 3741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabotvarious number of arguments. 3841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot 3941d0579e8de9ef4ff178fc4991043c61a19943f7Brett ChabotUSAGE: 4041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot pump.py SOURCE_FILE 4141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot 4241d0579e8de9ef4ff178fc4991043c61a19943f7Brett ChabotEXAMPLES: 4341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot pump.py foo.cc.pump 4441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot Converts foo.cc.pump to foo.cc. 4541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot 4641d0579e8de9ef4ff178fc4991043c61a19943f7Brett ChabotGRAMMAR: 4741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot CODE ::= ATOMIC_CODE* 4841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot ATOMIC_CODE ::= $var ID = EXPRESSION 4941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot | $var ID = [[ CODE ]] 5041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot | $range ID EXPRESSION..EXPRESSION 5141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot | $for ID SEPARATOR [[ CODE ]] 5241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot | $($) 5341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot | $ID 5441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot | $(EXPRESSION) 5541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot | $if EXPRESSION [[ CODE ]] ELSE_BRANCH 5641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot | [[ CODE ]] 5741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot | RAW_CODE 5841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot SEPARATOR ::= RAW_CODE | EMPTY 5941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot ELSE_BRANCH ::= $else [[ CODE ]] 6041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot | $elif EXPRESSION [[ CODE ]] ELSE_BRANCH 6141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot | EMPTY 6241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot EXPRESSION has Python syntax. 6341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot""" 6441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot 6541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot__author__ = 'wan@google.com (Zhanyong Wan)' 6641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot 6741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabotimport os 6841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabotimport re 6941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabotimport sys 7041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot 7141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot 7241d0579e8de9ef4ff178fc4991043c61a19943f7Brett ChabotTOKEN_TABLE = [ 7341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot (re.compile(r'\$var\s+'), '$var'), 7441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot (re.compile(r'\$elif\s+'), '$elif'), 7541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot (re.compile(r'\$else\s+'), '$else'), 7641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot (re.compile(r'\$for\s+'), '$for'), 7741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot (re.compile(r'\$if\s+'), '$if'), 7841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot (re.compile(r'\$range\s+'), '$range'), 7941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot (re.compile(r'\$[_A-Za-z]\w*'), '$id'), 8041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot (re.compile(r'\$\(\$\)'), '$($)'), 8141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot (re.compile(r'\$'), '$'), 8241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot (re.compile(r'\[\[\n?'), '[['), 8341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot (re.compile(r'\]\]\n?'), ']]'), 8441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot ] 8541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot 8641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot 8741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabotclass Cursor: 8841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot """Represents a position (line and column) in a text file.""" 8941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot 9041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot def __init__(self, line=-1, column=-1): 9141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot self.line = line 9241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot self.column = column 9341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot 9441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot def __eq__(self, rhs): 9541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot return self.line == rhs.line and self.column == rhs.column 9641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot 9741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot def __ne__(self, rhs): 9841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot return not self == rhs 9941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot 10041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot def __lt__(self, rhs): 10141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot return self.line < rhs.line or ( 10241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot self.line == rhs.line and self.column < rhs.column) 10341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot 10441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot def __le__(self, rhs): 10541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot return self < rhs or self == rhs 10641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot 10741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot def __gt__(self, rhs): 10841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot return rhs < self 10941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot 11041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot def __ge__(self, rhs): 11141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot return rhs <= self 11241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot 11341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot def __str__(self): 11441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot if self == Eof(): 11541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot return 'EOF' 11641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot else: 11741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot return '%s(%s)' % (self.line + 1, self.column) 11841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot 11941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot def __add__(self, offset): 12041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot return Cursor(self.line, self.column + offset) 12141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot 12241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot def __sub__(self, offset): 12341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot return Cursor(self.line, self.column - offset) 12441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot 12541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot def Clone(self): 12641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot """Returns a copy of self.""" 12741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot 12841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot return Cursor(self.line, self.column) 12941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot 13041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot 13141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot# Special cursor to indicate the end-of-file. 13241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabotdef Eof(): 13341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot """Returns the special cursor to denote the end-of-file.""" 13441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot return Cursor(-1, -1) 13541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot 13641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot 13741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabotclass Token: 13841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot """Represents a token in a Pump source file.""" 13941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot 14041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot def __init__(self, start=None, end=None, value=None, token_type=None): 14141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot if start is None: 14241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot self.start = Eof() 14341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot else: 14441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot self.start = start 14541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot if end is None: 14641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot self.end = Eof() 14741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot else: 14841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot self.end = end 14941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot self.value = value 15041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot self.token_type = token_type 15141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot 15241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot def __str__(self): 15341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot return 'Token @%s: \'%s\' type=%s' % ( 15441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot self.start, self.value, self.token_type) 15541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot 15641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot def Clone(self): 15741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot """Returns a copy of self.""" 15841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot 15941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot return Token(self.start.Clone(), self.end.Clone(), self.value, 16041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot self.token_type) 16141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot 16241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot 16341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabotdef StartsWith(lines, pos, string): 16441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot """Returns True iff the given position in lines starts with 'string'.""" 16541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot 16641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot return lines[pos.line][pos.column:].startswith(string) 16741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot 16841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot 16941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabotdef FindFirstInLine(line, token_table): 17041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot best_match_start = -1 17141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot for (regex, token_type) in token_table: 17241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot m = regex.search(line) 17341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot if m: 17441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot # We found regex in lines 17541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot if best_match_start < 0 or m.start() < best_match_start: 17641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot best_match_start = m.start() 17741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot best_match_length = m.end() - m.start() 17841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot best_match_token_type = token_type 17941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot 18041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot if best_match_start < 0: 18141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot return None 18241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot 18341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot return (best_match_start, best_match_length, best_match_token_type) 18441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot 18541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot 18641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabotdef FindFirst(lines, token_table, cursor): 18741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot """Finds the first occurrence of any string in strings in lines.""" 18841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot 18941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot start = cursor.Clone() 19041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot cur_line_number = cursor.line 19141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot for line in lines[start.line:]: 19241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot if cur_line_number == start.line: 19341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot line = line[start.column:] 19441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot m = FindFirstInLine(line, token_table) 19541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot if m: 19641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot # We found a regex in line. 19741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot (start_column, length, token_type) = m 19841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot if cur_line_number == start.line: 19941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot start_column += start.column 20041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot found_start = Cursor(cur_line_number, start_column) 20141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot found_end = found_start + length 20241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot return MakeToken(lines, found_start, found_end, token_type) 20341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot cur_line_number += 1 20441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot # We failed to find str in lines 20541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot return None 20641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot 20741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot 20841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabotdef SubString(lines, start, end): 20941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot """Returns a substring in lines.""" 21041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot 21141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot if end == Eof(): 21241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot end = Cursor(len(lines) - 1, len(lines[-1])) 21341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot 21441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot if start >= end: 21541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot return '' 21641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot 21741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot if start.line == end.line: 21841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot return lines[start.line][start.column:end.column] 21941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot 22041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot result_lines = ([lines[start.line][start.column:]] + 22141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot lines[start.line + 1:end.line] + 22241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot [lines[end.line][:end.column]]) 22341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot return ''.join(result_lines) 22441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot 22541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot 22641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabotdef StripMetaComments(str): 22741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot """Strip meta comments from each line in the given string.""" 22841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot 22941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot # First, completely remove lines containing nothing but a meta 23041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot # comment, including the trailing \n. 23141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot str = re.sub(r'^\s*\$\$.*\n', '', str) 23241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot 23341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot # Then, remove meta comments from contentful lines. 23441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot return re.sub(r'\s*\$\$.*', '', str) 23541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot 23641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot 23741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabotdef MakeToken(lines, start, end, token_type): 23841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot """Creates a new instance of Token.""" 23941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot 24041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot return Token(start, end, SubString(lines, start, end), token_type) 24141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot 24241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot 24341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabotdef ParseToken(lines, pos, regex, token_type): 24441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot line = lines[pos.line][pos.column:] 24541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot m = regex.search(line) 24641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot if m and not m.start(): 24741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot return MakeToken(lines, pos, pos + m.end(), token_type) 24841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot else: 24941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot print 'ERROR: %s expected at %s.' % (token_type, pos) 25041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot sys.exit(1) 25141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot 25241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot 25341d0579e8de9ef4ff178fc4991043c61a19943f7Brett ChabotID_REGEX = re.compile(r'[_A-Za-z]\w*') 25441d0579e8de9ef4ff178fc4991043c61a19943f7Brett ChabotEQ_REGEX = re.compile(r'=') 25541d0579e8de9ef4ff178fc4991043c61a19943f7Brett ChabotREST_OF_LINE_REGEX = re.compile(r'.*?(?=$|\$\$)') 25641d0579e8de9ef4ff178fc4991043c61a19943f7Brett ChabotOPTIONAL_WHITE_SPACES_REGEX = re.compile(r'\s*') 25741d0579e8de9ef4ff178fc4991043c61a19943f7Brett ChabotWHITE_SPACE_REGEX = re.compile(r'\s') 25841d0579e8de9ef4ff178fc4991043c61a19943f7Brett ChabotDOT_DOT_REGEX = re.compile(r'\.\.') 25941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot 26041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot 26141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabotdef Skip(lines, pos, regex): 26241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot line = lines[pos.line][pos.column:] 26341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot m = re.search(regex, line) 26441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot if m and not m.start(): 26541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot return pos + m.end() 26641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot else: 26741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot return pos 26841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot 26941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot 27041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabotdef SkipUntil(lines, pos, regex, token_type): 27141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot line = lines[pos.line][pos.column:] 27241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot m = re.search(regex, line) 27341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot if m: 27441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot return pos + m.start() 27541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot else: 27641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot print ('ERROR: %s expected on line %s after column %s.' % 27741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot (token_type, pos.line + 1, pos.column)) 27841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot sys.exit(1) 27941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot 28041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot 28141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabotdef ParseExpTokenInParens(lines, pos): 28241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot def ParseInParens(pos): 28341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot pos = Skip(lines, pos, OPTIONAL_WHITE_SPACES_REGEX) 28441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot pos = Skip(lines, pos, r'\(') 28541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot pos = Parse(pos) 28641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot pos = Skip(lines, pos, r'\)') 28741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot return pos 28841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot 28941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot def Parse(pos): 29041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot pos = SkipUntil(lines, pos, r'\(|\)', ')') 29141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot if SubString(lines, pos, pos + 1) == '(': 29241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot pos = Parse(pos + 1) 29341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot pos = Skip(lines, pos, r'\)') 29441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot return Parse(pos) 29541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot else: 29641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot return pos 29741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot 29841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot start = pos.Clone() 29941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot pos = ParseInParens(pos) 30041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot return MakeToken(lines, start, pos, 'exp') 30141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot 30241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot 30341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabotdef RStripNewLineFromToken(token): 30441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot if token.value.endswith('\n'): 30541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot return Token(token.start, token.end, token.value[:-1], token.token_type) 30641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot else: 30741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot return token 30841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot 30941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot 31041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabotdef TokenizeLines(lines, pos): 31141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot while True: 31241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot found = FindFirst(lines, TOKEN_TABLE, pos) 31341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot if not found: 31441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot yield MakeToken(lines, pos, Eof(), 'code') 31541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot return 31641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot 31741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot if found.start == pos: 31841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot prev_token = None 31941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot prev_token_rstripped = None 32041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot else: 32141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot prev_token = MakeToken(lines, pos, found.start, 'code') 32241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot prev_token_rstripped = RStripNewLineFromToken(prev_token) 32341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot 32441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot if found.token_type == '$var': 32541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot if prev_token_rstripped: 32641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot yield prev_token_rstripped 32741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot yield found 32841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot id_token = ParseToken(lines, found.end, ID_REGEX, 'id') 32941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot yield id_token 33041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot pos = Skip(lines, id_token.end, OPTIONAL_WHITE_SPACES_REGEX) 33141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot 33241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot eq_token = ParseToken(lines, pos, EQ_REGEX, '=') 33341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot yield eq_token 33441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot pos = Skip(lines, eq_token.end, r'\s*') 33541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot 33641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot if SubString(lines, pos, pos + 2) != '[[': 33741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot exp_token = ParseToken(lines, pos, REST_OF_LINE_REGEX, 'exp') 33841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot yield exp_token 33941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot pos = Cursor(exp_token.end.line + 1, 0) 34041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot elif found.token_type == '$for': 34141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot if prev_token_rstripped: 34241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot yield prev_token_rstripped 34341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot yield found 34441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot id_token = ParseToken(lines, found.end, ID_REGEX, 'id') 34541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot yield id_token 34641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot pos = Skip(lines, id_token.end, WHITE_SPACE_REGEX) 34741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot elif found.token_type == '$range': 34841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot if prev_token_rstripped: 34941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot yield prev_token_rstripped 35041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot yield found 35141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot id_token = ParseToken(lines, found.end, ID_REGEX, 'id') 35241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot yield id_token 35341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot pos = Skip(lines, id_token.end, OPTIONAL_WHITE_SPACES_REGEX) 35441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot 35541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot dots_pos = SkipUntil(lines, pos, DOT_DOT_REGEX, '..') 35641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot yield MakeToken(lines, pos, dots_pos, 'exp') 35741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot yield MakeToken(lines, dots_pos, dots_pos + 2, '..') 35841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot pos = dots_pos + 2 35941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot new_pos = Cursor(pos.line + 1, 0) 36041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot yield MakeToken(lines, pos, new_pos, 'exp') 36141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot pos = new_pos 36241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot elif found.token_type == '$': 36341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot if prev_token: 36441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot yield prev_token 36541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot yield found 36641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot exp_token = ParseExpTokenInParens(lines, found.end) 36741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot yield exp_token 36841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot pos = exp_token.end 36941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot elif (found.token_type == ']]' or found.token_type == '$if' or 37041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot found.token_type == '$elif' or found.token_type == '$else'): 37141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot if prev_token_rstripped: 37241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot yield prev_token_rstripped 37341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot yield found 37441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot pos = found.end 37541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot else: 37641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot if prev_token: 37741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot yield prev_token 37841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot yield found 37941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot pos = found.end 38041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot 38141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot 38241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabotdef Tokenize(s): 38341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot """A generator that yields the tokens in the given string.""" 38441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot if s != '': 38541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot lines = s.splitlines(True) 38641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot for token in TokenizeLines(lines, Cursor(0, 0)): 38741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot yield token 38841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot 38941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot 39041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabotclass CodeNode: 39141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot def __init__(self, atomic_code_list=None): 39241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot self.atomic_code = atomic_code_list 39341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot 39441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot 39541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabotclass VarNode: 39641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot def __init__(self, identifier=None, atomic_code=None): 39741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot self.identifier = identifier 39841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot self.atomic_code = atomic_code 39941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot 40041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot 40141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabotclass RangeNode: 40241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot def __init__(self, identifier=None, exp1=None, exp2=None): 40341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot self.identifier = identifier 40441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot self.exp1 = exp1 40541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot self.exp2 = exp2 40641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot 40741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot 40841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabotclass ForNode: 40941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot def __init__(self, identifier=None, sep=None, code=None): 41041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot self.identifier = identifier 41141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot self.sep = sep 41241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot self.code = code 41341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot 41441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot 41541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabotclass ElseNode: 41641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot def __init__(self, else_branch=None): 41741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot self.else_branch = else_branch 41841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot 41941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot 42041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabotclass IfNode: 42141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot def __init__(self, exp=None, then_branch=None, else_branch=None): 42241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot self.exp = exp 42341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot self.then_branch = then_branch 42441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot self.else_branch = else_branch 42541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot 42641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot 42741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabotclass RawCodeNode: 42841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot def __init__(self, token=None): 42941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot self.raw_code = token 43041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot 43141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot 43241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabotclass LiteralDollarNode: 43341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot def __init__(self, token): 43441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot self.token = token 43541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot 43641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot 43741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabotclass ExpNode: 43841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot def __init__(self, token, python_exp): 43941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot self.token = token 44041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot self.python_exp = python_exp 44141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot 44241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot 44341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabotdef PopFront(a_list): 44441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot head = a_list[0] 44541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot a_list[:1] = [] 44641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot return head 44741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot 44841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot 44941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabotdef PushFront(a_list, elem): 45041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot a_list[:0] = [elem] 45141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot 45241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot 45341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabotdef PopToken(a_list, token_type=None): 45441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot token = PopFront(a_list) 45541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot if token_type is not None and token.token_type != token_type: 45641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot print 'ERROR: %s expected at %s' % (token_type, token.start) 45741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot print 'ERROR: %s found instead' % (token,) 45841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot sys.exit(1) 45941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot 46041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot return token 46141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot 46241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot 46341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabotdef PeekToken(a_list): 46441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot if not a_list: 46541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot return None 46641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot 46741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot return a_list[0] 46841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot 46941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot 47041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabotdef ParseExpNode(token): 47141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot python_exp = re.sub(r'([_A-Za-z]\w*)', r'self.GetValue("\1")', token.value) 47241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot return ExpNode(token, python_exp) 47341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot 47441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot 47541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabotdef ParseElseNode(tokens): 47641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot def Pop(token_type=None): 47741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot return PopToken(tokens, token_type) 47841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot 47941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot next = PeekToken(tokens) 48041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot if not next: 48141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot return None 48241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot if next.token_type == '$else': 48341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot Pop('$else') 48441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot Pop('[[') 48541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot code_node = ParseCodeNode(tokens) 48641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot Pop(']]') 48741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot return code_node 48841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot elif next.token_type == '$elif': 48941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot Pop('$elif') 49041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot exp = Pop('code') 49141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot Pop('[[') 49241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot code_node = ParseCodeNode(tokens) 49341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot Pop(']]') 49441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot inner_else_node = ParseElseNode(tokens) 49541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot return CodeNode([IfNode(ParseExpNode(exp), code_node, inner_else_node)]) 49641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot elif not next.value.strip(): 49741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot Pop('code') 49841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot return ParseElseNode(tokens) 49941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot else: 50041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot return None 50141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot 50241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot 50341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabotdef ParseAtomicCodeNode(tokens): 50441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot def Pop(token_type=None): 50541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot return PopToken(tokens, token_type) 50641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot 50741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot head = PopFront(tokens) 50841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot t = head.token_type 50941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot if t == 'code': 51041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot return RawCodeNode(head) 51141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot elif t == '$var': 51241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot id_token = Pop('id') 51341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot Pop('=') 51441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot next = PeekToken(tokens) 51541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot if next.token_type == 'exp': 51641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot exp_token = Pop() 51741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot return VarNode(id_token, ParseExpNode(exp_token)) 51841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot Pop('[[') 51941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot code_node = ParseCodeNode(tokens) 52041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot Pop(']]') 52141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot return VarNode(id_token, code_node) 52241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot elif t == '$for': 52341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot id_token = Pop('id') 52441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot next_token = PeekToken(tokens) 52541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot if next_token.token_type == 'code': 52641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot sep_token = next_token 52741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot Pop('code') 52841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot else: 52941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot sep_token = None 53041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot Pop('[[') 53141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot code_node = ParseCodeNode(tokens) 53241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot Pop(']]') 53341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot return ForNode(id_token, sep_token, code_node) 53441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot elif t == '$if': 53541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot exp_token = Pop('code') 53641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot Pop('[[') 53741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot code_node = ParseCodeNode(tokens) 53841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot Pop(']]') 53941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot else_node = ParseElseNode(tokens) 54041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot return IfNode(ParseExpNode(exp_token), code_node, else_node) 54141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot elif t == '$range': 54241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot id_token = Pop('id') 54341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot exp1_token = Pop('exp') 54441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot Pop('..') 54541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot exp2_token = Pop('exp') 54641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot return RangeNode(id_token, ParseExpNode(exp1_token), 54741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot ParseExpNode(exp2_token)) 54841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot elif t == '$id': 54941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot return ParseExpNode(Token(head.start + 1, head.end, head.value[1:], 'id')) 55041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot elif t == '$($)': 55141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot return LiteralDollarNode(head) 55241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot elif t == '$': 55341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot exp_token = Pop('exp') 55441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot return ParseExpNode(exp_token) 55541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot elif t == '[[': 55641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot code_node = ParseCodeNode(tokens) 55741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot Pop(']]') 55841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot return code_node 55941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot else: 56041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot PushFront(tokens, head) 56141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot return None 56241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot 56341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot 56441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabotdef ParseCodeNode(tokens): 56541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot atomic_code_list = [] 56641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot while True: 56741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot if not tokens: 56841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot break 56941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot atomic_code_node = ParseAtomicCodeNode(tokens) 57041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot if atomic_code_node: 57141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot atomic_code_list.append(atomic_code_node) 57241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot else: 57341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot break 57441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot return CodeNode(atomic_code_list) 57541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot 57641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot 57741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabotdef ParseToAST(pump_src_text): 57841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot """Convert the given Pump source text into an AST.""" 57941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot tokens = list(Tokenize(pump_src_text)) 58041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot code_node = ParseCodeNode(tokens) 58141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot return code_node 58241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot 58341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot 58441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabotclass Env: 58541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot def __init__(self): 58641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot self.variables = [] 58741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot self.ranges = [] 58841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot 58941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot def Clone(self): 59041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot clone = Env() 59141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot clone.variables = self.variables[:] 59241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot clone.ranges = self.ranges[:] 59341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot return clone 59441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot 59541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot def PushVariable(self, var, value): 59641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot # If value looks like an int, store it as an int. 59741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot try: 59841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot int_value = int(value) 59941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot if ('%s' % int_value) == value: 60041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot value = int_value 60141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot except Exception: 60241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot pass 60341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot self.variables[:0] = [(var, value)] 60441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot 60541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot def PopVariable(self): 60641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot self.variables[:1] = [] 60741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot 60841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot def PushRange(self, var, lower, upper): 60941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot self.ranges[:0] = [(var, lower, upper)] 61041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot 61141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot def PopRange(self): 61241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot self.ranges[:1] = [] 61341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot 61441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot def GetValue(self, identifier): 61541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot for (var, value) in self.variables: 61641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot if identifier == var: 61741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot return value 61841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot 61941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot print 'ERROR: meta variable %s is undefined.' % (identifier,) 62041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot sys.exit(1) 62141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot 62241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot def EvalExp(self, exp): 62341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot try: 62441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot result = eval(exp.python_exp) 62541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot except Exception, e: 62641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot print 'ERROR: caught exception %s: %s' % (e.__class__.__name__, e) 62741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot print ('ERROR: failed to evaluate meta expression %s at %s' % 62841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot (exp.python_exp, exp.token.start)) 62941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot sys.exit(1) 63041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot return result 63141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot 63241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot def GetRange(self, identifier): 63341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot for (var, lower, upper) in self.ranges: 63441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot if identifier == var: 63541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot return (lower, upper) 63641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot 63741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot print 'ERROR: range %s is undefined.' % (identifier,) 63841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot sys.exit(1) 63941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot 64041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot 64141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabotclass Output: 64241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot def __init__(self): 64341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot self.string = '' 64441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot 64541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot def GetLastLine(self): 64641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot index = self.string.rfind('\n') 64741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot if index < 0: 64841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot return '' 64941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot 65041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot return self.string[index + 1:] 65141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot 65241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot def Append(self, s): 65341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot self.string += s 65441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot 65541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot 65641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabotdef RunAtomicCode(env, node, output): 65741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot if isinstance(node, VarNode): 65841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot identifier = node.identifier.value.strip() 65941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot result = Output() 66041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot RunAtomicCode(env.Clone(), node.atomic_code, result) 66141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot value = result.string 66241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot env.PushVariable(identifier, value) 66341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot elif isinstance(node, RangeNode): 66441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot identifier = node.identifier.value.strip() 66541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot lower = int(env.EvalExp(node.exp1)) 66641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot upper = int(env.EvalExp(node.exp2)) 66741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot env.PushRange(identifier, lower, upper) 66841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot elif isinstance(node, ForNode): 66941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot identifier = node.identifier.value.strip() 67041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot if node.sep is None: 67141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot sep = '' 67241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot else: 67341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot sep = node.sep.value 67441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot (lower, upper) = env.GetRange(identifier) 67541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot for i in range(lower, upper + 1): 67641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot new_env = env.Clone() 67741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot new_env.PushVariable(identifier, i) 67841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot RunCode(new_env, node.code, output) 67941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot if i != upper: 68041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot output.Append(sep) 68141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot elif isinstance(node, RawCodeNode): 68241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot output.Append(node.raw_code.value) 68341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot elif isinstance(node, IfNode): 68441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot cond = env.EvalExp(node.exp) 68541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot if cond: 68641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot RunCode(env.Clone(), node.then_branch, output) 68741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot elif node.else_branch is not None: 68841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot RunCode(env.Clone(), node.else_branch, output) 68941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot elif isinstance(node, ExpNode): 69041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot value = env.EvalExp(node) 69141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot output.Append('%s' % (value,)) 69241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot elif isinstance(node, LiteralDollarNode): 69341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot output.Append('$') 69441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot elif isinstance(node, CodeNode): 69541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot RunCode(env.Clone(), node, output) 69641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot else: 69741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot print 'BAD' 69841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot print node 69941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot sys.exit(1) 70041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot 70141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot 70241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabotdef RunCode(env, code_node, output): 70341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot for atomic_code in code_node.atomic_code: 70441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot RunAtomicCode(env, atomic_code, output) 70541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot 70641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot 70741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabotdef IsComment(cur_line): 70841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot return '//' in cur_line 70941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot 71041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot 71141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabotdef IsInPreprocessorDirevative(prev_lines, cur_line): 71241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot if cur_line.lstrip().startswith('#'): 71341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot return True 71441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot return prev_lines != [] and prev_lines[-1].endswith('\\') 71541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot 71641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot 71741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabotdef WrapComment(line, output): 71841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot loc = line.find('//') 71941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot before_comment = line[:loc].rstrip() 72041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot if before_comment == '': 72141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot indent = loc 72241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot else: 72341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot output.append(before_comment) 72441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot indent = len(before_comment) - len(before_comment.lstrip()) 72541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot prefix = indent*' ' + '// ' 72641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot max_len = 80 - len(prefix) 72741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot comment = line[loc + 2:].strip() 72841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot segs = [seg for seg in re.split(r'(\w+\W*)', comment) if seg != ''] 72941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot cur_line = '' 73041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot for seg in segs: 73141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot if len((cur_line + seg).rstrip()) < max_len: 73241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot cur_line += seg 73341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot else: 73441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot if cur_line.strip() != '': 73541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot output.append(prefix + cur_line.rstrip()) 73641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot cur_line = seg.lstrip() 73741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot if cur_line.strip() != '': 73841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot output.append(prefix + cur_line.strip()) 73941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot 74041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot 74141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabotdef WrapCode(line, line_concat, output): 74241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot indent = len(line) - len(line.lstrip()) 74341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot prefix = indent*' ' # Prefix of the current line 74441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot max_len = 80 - indent - len(line_concat) # Maximum length of the current line 74541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot new_prefix = prefix + 4*' ' # Prefix of a continuation line 74641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot new_max_len = max_len - 4 # Maximum length of a continuation line 74741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot # Prefers to wrap a line after a ',' or ';'. 74841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot segs = [seg for seg in re.split(r'([^,;]+[,;]?)', line.strip()) if seg != ''] 74941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot cur_line = '' # The current line without leading spaces. 75041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot for seg in segs: 75141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot # If the line is still too long, wrap at a space. 75241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot while cur_line == '' and len(seg.strip()) > max_len: 75341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot seg = seg.lstrip() 75441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot split_at = seg.rfind(' ', 0, max_len) 75541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot output.append(prefix + seg[:split_at].strip() + line_concat) 75641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot seg = seg[split_at + 1:] 75741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot prefix = new_prefix 75841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot max_len = new_max_len 75941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot 76041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot if len((cur_line + seg).rstrip()) < max_len: 76141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot cur_line = (cur_line + seg).lstrip() 76241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot else: 76341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot output.append(prefix + cur_line.rstrip() + line_concat) 76441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot prefix = new_prefix 76541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot max_len = new_max_len 76641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot cur_line = seg.lstrip() 76741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot if cur_line.strip() != '': 76841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot output.append(prefix + cur_line.strip()) 76941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot 77041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot 77141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabotdef WrapPreprocessorDirevative(line, output): 77241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot WrapCode(line, ' \\', output) 77341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot 77441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot 77541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabotdef WrapPlainCode(line, output): 77641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot WrapCode(line, '', output) 77741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot 77841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot 77941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabotdef IsHeaderGuardOrInclude(line): 78041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot return (re.match(r'^#(ifndef|define|endif\s*//)\s*[\w_]+\s*$', line) or 78141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot re.match(r'^#include\s', line)) 78241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot 78341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot 78441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabotdef WrapLongLine(line, output): 78541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot line = line.rstrip() 78641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot if len(line) <= 80: 78741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot output.append(line) 78841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot elif IsComment(line): 78941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot if IsHeaderGuardOrInclude(line): 79041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot # The style guide made an exception to allow long header guard lines 79141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot # and includes. 79241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot output.append(line) 79341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot else: 79441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot WrapComment(line, output) 79541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot elif IsInPreprocessorDirevative(output, line): 79641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot if IsHeaderGuardOrInclude(line): 79741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot # The style guide made an exception to allow long header guard lines 79841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot # and includes. 79941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot output.append(line) 80041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot else: 80141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot WrapPreprocessorDirevative(line, output) 80241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot else: 80341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot WrapPlainCode(line, output) 80441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot 80541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot 80641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabotdef BeautifyCode(string): 80741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot lines = string.splitlines() 80841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot output = [] 80941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot for line in lines: 81041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot WrapLongLine(line, output) 81141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot output2 = [line.rstrip() for line in output] 81241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot return '\n'.join(output2) + '\n' 81341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot 81441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot 81541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabotdef ConvertFromPumpSource(src_text): 81641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot """Return the text generated from the given Pump source text.""" 81741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot ast = ParseToAST(StripMetaComments(src_text)) 81841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot output = Output() 81941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot RunCode(Env(), ast, output) 82041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot return BeautifyCode(output.string) 82141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot 82241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot 82341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabotdef main(argv): 82441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot if len(argv) == 1: 82541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot print __doc__ 82641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot sys.exit(1) 82741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot 82841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot file_path = argv[-1] 82941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot output_str = ConvertFromPumpSource(file(file_path, 'r').read()) 83041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot if file_path.endswith('.pump'): 83141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot output_file_path = file_path[:-5] 83241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot else: 83341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot output_file_path = '-' 83441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot if output_file_path == '-': 83541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot print output_str, 83641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot else: 83741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot output_file = file(output_file_path, 'w') 83841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot output_file.write('// This file was GENERATED by command:\n') 83941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot output_file.write('// %s %s\n' % 84041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot (os.path.basename(__file__), os.path.basename(file_path))) 84141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot output_file.write('// DO NOT EDIT BY HAND!!!\n\n') 84241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot output_file.write(output_str) 84341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot output_file.close() 84441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot 84541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot 84641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabotif __name__ == '__main__': 84741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot main(sys.argv) 848