1c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch#!/usr/bin/env python 2c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch# 3c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch# Copyright 2008, Google Inc. 4c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch# All rights reserved. 5c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch# 6c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch# Redistribution and use in source and binary forms, with or without 7c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch# modification, are permitted provided that the following conditions are 8c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch# met: 9c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch# 10c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch# * Redistributions of source code must retain the above copyright 11c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch# notice, this list of conditions and the following disclaimer. 12c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch# * Redistributions in binary form must reproduce the above 13c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch# copyright notice, this list of conditions and the following disclaimer 14c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch# in the documentation and/or other materials provided with the 15c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch# distribution. 16c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch# * Neither the name of Google Inc. nor the names of its 17c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch# contributors may be used to endorse or promote products derived from 18c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch# this software without specific prior written permission. 19c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch# 20c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 21c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 22c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 23c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 24c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 25c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 26c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 27c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 28c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 29c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 30c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 31c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch 32dc0f95d653279beabeb9817299e2902918ba123eKristian Monsen"""pump v0.2.0 - Pretty Useful for Meta Programming. 33c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch 34c407dc5cd9bdc5668497f21b26b09d988ab439deBen MurdochA tool for preprocessor meta programming. Useful for generating 35c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdochrepetitive boilerplate code. Especially useful for writing C++ 36c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdochclasses, functions, macros, and templates that need to work with 37c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdochvarious number of arguments. 38c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch 39c407dc5cd9bdc5668497f21b26b09d988ab439deBen MurdochUSAGE: 40c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch pump.py SOURCE_FILE 41c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch 42c407dc5cd9bdc5668497f21b26b09d988ab439deBen MurdochEXAMPLES: 43c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch pump.py foo.cc.pump 44c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch Converts foo.cc.pump to foo.cc. 45c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch 46c407dc5cd9bdc5668497f21b26b09d988ab439deBen MurdochGRAMMAR: 47c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch CODE ::= ATOMIC_CODE* 48c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch ATOMIC_CODE ::= $var ID = EXPRESSION 49c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch | $var ID = [[ CODE ]] 50c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch | $range ID EXPRESSION..EXPRESSION 51c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch | $for ID SEPARATOR [[ CODE ]] 52c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch | $($) 53c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch | $ID 54c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch | $(EXPRESSION) 55c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch | $if EXPRESSION [[ CODE ]] ELSE_BRANCH 56c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch | [[ CODE ]] 57c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch | RAW_CODE 58c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch SEPARATOR ::= RAW_CODE | EMPTY 59c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch ELSE_BRANCH ::= $else [[ CODE ]] 60c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch | $elif EXPRESSION [[ CODE ]] ELSE_BRANCH 61c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch | EMPTY 62c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch EXPRESSION has Python syntax. 63c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch""" 64c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch 65c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch__author__ = 'wan@google.com (Zhanyong Wan)' 66c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch 67c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdochimport os 68c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdochimport re 69c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdochimport sys 70c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch 71c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch 72c407dc5cd9bdc5668497f21b26b09d988ab439deBen MurdochTOKEN_TABLE = [ 73c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch (re.compile(r'\$var\s+'), '$var'), 74c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch (re.compile(r'\$elif\s+'), '$elif'), 75c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch (re.compile(r'\$else\s+'), '$else'), 76c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch (re.compile(r'\$for\s+'), '$for'), 77c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch (re.compile(r'\$if\s+'), '$if'), 78c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch (re.compile(r'\$range\s+'), '$range'), 79c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch (re.compile(r'\$[_A-Za-z]\w*'), '$id'), 80c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch (re.compile(r'\$\(\$\)'), '$($)'), 81c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch (re.compile(r'\$'), '$'), 82c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch (re.compile(r'\[\[\n?'), '[['), 83c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch (re.compile(r'\]\]\n?'), ']]'), 84c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch ] 85c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch 86c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch 87c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdochclass Cursor: 88c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch """Represents a position (line and column) in a text file.""" 89c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch 90c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch def __init__(self, line=-1, column=-1): 91c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch self.line = line 92c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch self.column = column 93c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch 94c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch def __eq__(self, rhs): 95c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch return self.line == rhs.line and self.column == rhs.column 96c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch 97c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch def __ne__(self, rhs): 98c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch return not self == rhs 99c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch 100c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch def __lt__(self, rhs): 101c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch return self.line < rhs.line or ( 102c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch self.line == rhs.line and self.column < rhs.column) 103c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch 104c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch def __le__(self, rhs): 105c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch return self < rhs or self == rhs 106c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch 107c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch def __gt__(self, rhs): 108c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch return rhs < self 109c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch 110c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch def __ge__(self, rhs): 111c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch return rhs <= self 112c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch 113c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch def __str__(self): 114c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch if self == Eof(): 115c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch return 'EOF' 116c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch else: 117c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch return '%s(%s)' % (self.line + 1, self.column) 118c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch 119c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch def __add__(self, offset): 120c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch return Cursor(self.line, self.column + offset) 121c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch 122c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch def __sub__(self, offset): 123c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch return Cursor(self.line, self.column - offset) 124c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch 125c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch def Clone(self): 126c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch """Returns a copy of self.""" 127c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch 128c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch return Cursor(self.line, self.column) 129c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch 130c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch 131c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch# Special cursor to indicate the end-of-file. 132c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdochdef Eof(): 133c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch """Returns the special cursor to denote the end-of-file.""" 134c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch return Cursor(-1, -1) 135c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch 136c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch 137c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdochclass Token: 138c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch """Represents a token in a Pump source file.""" 139c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch 140c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch def __init__(self, start=None, end=None, value=None, token_type=None): 141c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch if start is None: 142c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch self.start = Eof() 143c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch else: 144c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch self.start = start 145c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch if end is None: 146c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch self.end = Eof() 147c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch else: 148c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch self.end = end 149c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch self.value = value 150c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch self.token_type = token_type 151c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch 152c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch def __str__(self): 153c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch return 'Token @%s: \'%s\' type=%s' % ( 154c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch self.start, self.value, self.token_type) 155c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch 156c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch def Clone(self): 157c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch """Returns a copy of self.""" 158c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch 159c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch return Token(self.start.Clone(), self.end.Clone(), self.value, 160c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch self.token_type) 161c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch 162c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch 163c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdochdef StartsWith(lines, pos, string): 164c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch """Returns True iff the given position in lines starts with 'string'.""" 165c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch 166c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch return lines[pos.line][pos.column:].startswith(string) 167c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch 168c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch 169c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdochdef FindFirstInLine(line, token_table): 170c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch best_match_start = -1 171c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch for (regex, token_type) in token_table: 172c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch m = regex.search(line) 173c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch if m: 174c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch # We found regex in lines 175c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch if best_match_start < 0 or m.start() < best_match_start: 176c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch best_match_start = m.start() 177c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch best_match_length = m.end() - m.start() 178c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch best_match_token_type = token_type 179c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch 180c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch if best_match_start < 0: 181c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch return None 182c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch 183c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch return (best_match_start, best_match_length, best_match_token_type) 184c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch 185c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch 186c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdochdef FindFirst(lines, token_table, cursor): 187c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch """Finds the first occurrence of any string in strings in lines.""" 188c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch 189c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch start = cursor.Clone() 190c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch cur_line_number = cursor.line 191c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch for line in lines[start.line:]: 192c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch if cur_line_number == start.line: 193c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch line = line[start.column:] 194c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch m = FindFirstInLine(line, token_table) 195c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch if m: 196c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch # We found a regex in line. 197c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch (start_column, length, token_type) = m 198c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch if cur_line_number == start.line: 199c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch start_column += start.column 200c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch found_start = Cursor(cur_line_number, start_column) 201c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch found_end = found_start + length 202c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch return MakeToken(lines, found_start, found_end, token_type) 203c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch cur_line_number += 1 204c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch # We failed to find str in lines 205c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch return None 206c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch 207c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch 208c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdochdef SubString(lines, start, end): 209c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch """Returns a substring in lines.""" 210c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch 211c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch if end == Eof(): 212c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch end = Cursor(len(lines) - 1, len(lines[-1])) 213c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch 214c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch if start >= end: 215c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch return '' 216c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch 217c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch if start.line == end.line: 218c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch return lines[start.line][start.column:end.column] 219c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch 220c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch result_lines = ([lines[start.line][start.column:]] + 221c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch lines[start.line + 1:end.line] + 222c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch [lines[end.line][:end.column]]) 223c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch return ''.join(result_lines) 224c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch 225c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch 226dc0f95d653279beabeb9817299e2902918ba123eKristian Monsendef StripMetaComments(str): 227dc0f95d653279beabeb9817299e2902918ba123eKristian Monsen """Strip meta comments from each line in the given string.""" 228dc0f95d653279beabeb9817299e2902918ba123eKristian Monsen 229dc0f95d653279beabeb9817299e2902918ba123eKristian Monsen # First, completely remove lines containing nothing but a meta 230dc0f95d653279beabeb9817299e2902918ba123eKristian Monsen # comment, including the trailing \n. 231dc0f95d653279beabeb9817299e2902918ba123eKristian Monsen str = re.sub(r'^\s*\$\$.*\n', '', str) 232dc0f95d653279beabeb9817299e2902918ba123eKristian Monsen 233dc0f95d653279beabeb9817299e2902918ba123eKristian Monsen # Then, remove meta comments from contentful lines. 234dc0f95d653279beabeb9817299e2902918ba123eKristian Monsen return re.sub(r'\s*\$\$.*', '', str) 235dc0f95d653279beabeb9817299e2902918ba123eKristian Monsen 236dc0f95d653279beabeb9817299e2902918ba123eKristian Monsen 237c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdochdef MakeToken(lines, start, end, token_type): 238c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch """Creates a new instance of Token.""" 239c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch 240c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch return Token(start, end, SubString(lines, start, end), token_type) 241c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch 242c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch 243c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdochdef ParseToken(lines, pos, regex, token_type): 244c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch line = lines[pos.line][pos.column:] 245c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch m = regex.search(line) 246c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch if m and not m.start(): 247c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch return MakeToken(lines, pos, pos + m.end(), token_type) 248c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch else: 249c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch print 'ERROR: %s expected at %s.' % (token_type, pos) 250c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch sys.exit(1) 251c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch 252c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch 253c407dc5cd9bdc5668497f21b26b09d988ab439deBen MurdochID_REGEX = re.compile(r'[_A-Za-z]\w*') 254c407dc5cd9bdc5668497f21b26b09d988ab439deBen MurdochEQ_REGEX = re.compile(r'=') 255c407dc5cd9bdc5668497f21b26b09d988ab439deBen MurdochREST_OF_LINE_REGEX = re.compile(r'.*?(?=$|\$\$)') 256c407dc5cd9bdc5668497f21b26b09d988ab439deBen MurdochOPTIONAL_WHITE_SPACES_REGEX = re.compile(r'\s*') 257c407dc5cd9bdc5668497f21b26b09d988ab439deBen MurdochWHITE_SPACE_REGEX = re.compile(r'\s') 258c407dc5cd9bdc5668497f21b26b09d988ab439deBen MurdochDOT_DOT_REGEX = re.compile(r'\.\.') 259c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch 260c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch 261c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdochdef Skip(lines, pos, regex): 262c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch line = lines[pos.line][pos.column:] 263c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch m = re.search(regex, line) 264c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch if m and not m.start(): 265c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch return pos + m.end() 266c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch else: 267c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch return pos 268c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch 269c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch 270c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdochdef SkipUntil(lines, pos, regex, token_type): 271c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch line = lines[pos.line][pos.column:] 272c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch m = re.search(regex, line) 273c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch if m: 274c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch return pos + m.start() 275c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch else: 276c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch print ('ERROR: %s expected on line %s after column %s.' % 277c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch (token_type, pos.line + 1, pos.column)) 278c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch sys.exit(1) 279c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch 280c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch 281c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdochdef ParseExpTokenInParens(lines, pos): 282c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch def ParseInParens(pos): 283c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch pos = Skip(lines, pos, OPTIONAL_WHITE_SPACES_REGEX) 284c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch pos = Skip(lines, pos, r'\(') 285c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch pos = Parse(pos) 286c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch pos = Skip(lines, pos, r'\)') 287c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch return pos 288c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch 289c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch def Parse(pos): 290c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch pos = SkipUntil(lines, pos, r'\(|\)', ')') 291c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch if SubString(lines, pos, pos + 1) == '(': 292c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch pos = Parse(pos + 1) 293c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch pos = Skip(lines, pos, r'\)') 294c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch return Parse(pos) 295c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch else: 296c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch return pos 297c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch 298c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch start = pos.Clone() 299c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch pos = ParseInParens(pos) 300c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch return MakeToken(lines, start, pos, 'exp') 301c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch 302c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch 303c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdochdef RStripNewLineFromToken(token): 304c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch if token.value.endswith('\n'): 305c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch return Token(token.start, token.end, token.value[:-1], token.token_type) 306c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch else: 307c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch return token 308c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch 309c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch 310c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdochdef TokenizeLines(lines, pos): 311c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch while True: 312c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch found = FindFirst(lines, TOKEN_TABLE, pos) 313c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch if not found: 314c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch yield MakeToken(lines, pos, Eof(), 'code') 315c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch return 316c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch 317c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch if found.start == pos: 318c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch prev_token = None 319c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch prev_token_rstripped = None 320c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch else: 321c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch prev_token = MakeToken(lines, pos, found.start, 'code') 322c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch prev_token_rstripped = RStripNewLineFromToken(prev_token) 323c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch 324dc0f95d653279beabeb9817299e2902918ba123eKristian Monsen if found.token_type == '$var': 325c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch if prev_token_rstripped: 326c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch yield prev_token_rstripped 327c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch yield found 328c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch id_token = ParseToken(lines, found.end, ID_REGEX, 'id') 329c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch yield id_token 330c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch pos = Skip(lines, id_token.end, OPTIONAL_WHITE_SPACES_REGEX) 331c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch 332c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch eq_token = ParseToken(lines, pos, EQ_REGEX, '=') 333c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch yield eq_token 334c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch pos = Skip(lines, eq_token.end, r'\s*') 335c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch 336c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch if SubString(lines, pos, pos + 2) != '[[': 337c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch exp_token = ParseToken(lines, pos, REST_OF_LINE_REGEX, 'exp') 338c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch yield exp_token 339c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch pos = Cursor(exp_token.end.line + 1, 0) 340c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch elif found.token_type == '$for': 341c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch if prev_token_rstripped: 342c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch yield prev_token_rstripped 343c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch yield found 344c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch id_token = ParseToken(lines, found.end, ID_REGEX, 'id') 345c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch yield id_token 346c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch pos = Skip(lines, id_token.end, WHITE_SPACE_REGEX) 347c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch elif found.token_type == '$range': 348c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch if prev_token_rstripped: 349c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch yield prev_token_rstripped 350c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch yield found 351c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch id_token = ParseToken(lines, found.end, ID_REGEX, 'id') 352c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch yield id_token 353c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch pos = Skip(lines, id_token.end, OPTIONAL_WHITE_SPACES_REGEX) 354c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch 355c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch dots_pos = SkipUntil(lines, pos, DOT_DOT_REGEX, '..') 356c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch yield MakeToken(lines, pos, dots_pos, 'exp') 357c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch yield MakeToken(lines, dots_pos, dots_pos + 2, '..') 358c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch pos = dots_pos + 2 359c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch new_pos = Cursor(pos.line + 1, 0) 360c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch yield MakeToken(lines, pos, new_pos, 'exp') 361c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch pos = new_pos 362c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch elif found.token_type == '$': 363c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch if prev_token: 364c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch yield prev_token 365c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch yield found 366c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch exp_token = ParseExpTokenInParens(lines, found.end) 367c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch yield exp_token 368c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch pos = exp_token.end 369c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch elif (found.token_type == ']]' or found.token_type == '$if' or 370c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch found.token_type == '$elif' or found.token_type == '$else'): 371c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch if prev_token_rstripped: 372c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch yield prev_token_rstripped 373c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch yield found 374c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch pos = found.end 375c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch else: 376c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch if prev_token: 377c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch yield prev_token 378c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch yield found 379c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch pos = found.end 380c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch 381c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch 382c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdochdef Tokenize(s): 383dc0f95d653279beabeb9817299e2902918ba123eKristian Monsen """A generator that yields the tokens in the given string.""" 384dc0f95d653279beabeb9817299e2902918ba123eKristian Monsen if s != '': 385dc0f95d653279beabeb9817299e2902918ba123eKristian Monsen lines = s.splitlines(True) 386dc0f95d653279beabeb9817299e2902918ba123eKristian Monsen for token in TokenizeLines(lines, Cursor(0, 0)): 387dc0f95d653279beabeb9817299e2902918ba123eKristian Monsen yield token 388c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch 389c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch 390c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdochclass CodeNode: 391c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch def __init__(self, atomic_code_list=None): 392c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch self.atomic_code = atomic_code_list 393c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch 394c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch 395c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdochclass VarNode: 396c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch def __init__(self, identifier=None, atomic_code=None): 397c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch self.identifier = identifier 398c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch self.atomic_code = atomic_code 399c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch 400c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch 401c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdochclass RangeNode: 402c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch def __init__(self, identifier=None, exp1=None, exp2=None): 403c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch self.identifier = identifier 404c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch self.exp1 = exp1 405c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch self.exp2 = exp2 406c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch 407c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch 408c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdochclass ForNode: 409c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch def __init__(self, identifier=None, sep=None, code=None): 410c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch self.identifier = identifier 411c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch self.sep = sep 412c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch self.code = code 413c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch 414c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch 415c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdochclass ElseNode: 416c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch def __init__(self, else_branch=None): 417c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch self.else_branch = else_branch 418c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch 419c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch 420c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdochclass IfNode: 421c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch def __init__(self, exp=None, then_branch=None, else_branch=None): 422c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch self.exp = exp 423c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch self.then_branch = then_branch 424c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch self.else_branch = else_branch 425c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch 426c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch 427c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdochclass RawCodeNode: 428c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch def __init__(self, token=None): 429c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch self.raw_code = token 430c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch 431c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch 432c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdochclass LiteralDollarNode: 433c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch def __init__(self, token): 434c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch self.token = token 435c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch 436c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch 437c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdochclass ExpNode: 438c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch def __init__(self, token, python_exp): 439c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch self.token = token 440c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch self.python_exp = python_exp 441c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch 442c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch 443c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdochdef PopFront(a_list): 444c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch head = a_list[0] 445c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch a_list[:1] = [] 446c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch return head 447c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch 448c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch 449c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdochdef PushFront(a_list, elem): 450c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch a_list[:0] = [elem] 451c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch 452c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch 453c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdochdef PopToken(a_list, token_type=None): 454c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch token = PopFront(a_list) 455c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch if token_type is not None and token.token_type != token_type: 456c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch print 'ERROR: %s expected at %s' % (token_type, token.start) 457c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch print 'ERROR: %s found instead' % (token,) 458c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch sys.exit(1) 459c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch 460c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch return token 461c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch 462c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch 463c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdochdef PeekToken(a_list): 464c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch if not a_list: 465c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch return None 466c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch 467c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch return a_list[0] 468c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch 469c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch 470c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdochdef ParseExpNode(token): 471c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch python_exp = re.sub(r'([_A-Za-z]\w*)', r'self.GetValue("\1")', token.value) 472c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch return ExpNode(token, python_exp) 473c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch 474c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch 475c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdochdef ParseElseNode(tokens): 476c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch def Pop(token_type=None): 477c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch return PopToken(tokens, token_type) 478c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch 479c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch next = PeekToken(tokens) 480c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch if not next: 481c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch return None 482c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch if next.token_type == '$else': 483c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch Pop('$else') 484c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch Pop('[[') 485c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch code_node = ParseCodeNode(tokens) 486c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch Pop(']]') 487c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch return code_node 488c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch elif next.token_type == '$elif': 489c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch Pop('$elif') 490c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch exp = Pop('code') 491c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch Pop('[[') 492c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch code_node = ParseCodeNode(tokens) 493c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch Pop(']]') 494c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch inner_else_node = ParseElseNode(tokens) 495c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch return CodeNode([IfNode(ParseExpNode(exp), code_node, inner_else_node)]) 496c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch elif not next.value.strip(): 497c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch Pop('code') 498c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch return ParseElseNode(tokens) 499c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch else: 500c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch return None 501c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch 502c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch 503c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdochdef ParseAtomicCodeNode(tokens): 504c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch def Pop(token_type=None): 505c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch return PopToken(tokens, token_type) 506c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch 507c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch head = PopFront(tokens) 508c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch t = head.token_type 509c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch if t == 'code': 510c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch return RawCodeNode(head) 511c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch elif t == '$var': 512c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch id_token = Pop('id') 513c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch Pop('=') 514c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch next = PeekToken(tokens) 515c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch if next.token_type == 'exp': 516c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch exp_token = Pop() 517c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch return VarNode(id_token, ParseExpNode(exp_token)) 518c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch Pop('[[') 519c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch code_node = ParseCodeNode(tokens) 520c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch Pop(']]') 521c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch return VarNode(id_token, code_node) 522c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch elif t == '$for': 523c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch id_token = Pop('id') 524c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch next_token = PeekToken(tokens) 525c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch if next_token.token_type == 'code': 526c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch sep_token = next_token 527c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch Pop('code') 528c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch else: 529c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch sep_token = None 530c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch Pop('[[') 531c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch code_node = ParseCodeNode(tokens) 532c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch Pop(']]') 533c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch return ForNode(id_token, sep_token, code_node) 534c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch elif t == '$if': 535c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch exp_token = Pop('code') 536c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch Pop('[[') 537c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch code_node = ParseCodeNode(tokens) 538c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch Pop(']]') 539c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch else_node = ParseElseNode(tokens) 540c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch return IfNode(ParseExpNode(exp_token), code_node, else_node) 541c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch elif t == '$range': 542c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch id_token = Pop('id') 543c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch exp1_token = Pop('exp') 544c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch Pop('..') 545c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch exp2_token = Pop('exp') 546c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch return RangeNode(id_token, ParseExpNode(exp1_token), 547c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch ParseExpNode(exp2_token)) 548c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch elif t == '$id': 549c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch return ParseExpNode(Token(head.start + 1, head.end, head.value[1:], 'id')) 550c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch elif t == '$($)': 551c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch return LiteralDollarNode(head) 552c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch elif t == '$': 553c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch exp_token = Pop('exp') 554c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch return ParseExpNode(exp_token) 555c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch elif t == '[[': 556c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch code_node = ParseCodeNode(tokens) 557c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch Pop(']]') 558c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch return code_node 559c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch else: 560c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch PushFront(tokens, head) 561c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch return None 562c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch 563c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch 564c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdochdef ParseCodeNode(tokens): 565c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch atomic_code_list = [] 566c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch while True: 567c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch if not tokens: 568c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch break 569c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch atomic_code_node = ParseAtomicCodeNode(tokens) 570c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch if atomic_code_node: 571c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch atomic_code_list.append(atomic_code_node) 572c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch else: 573c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch break 574c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch return CodeNode(atomic_code_list) 575c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch 576c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch 577dc0f95d653279beabeb9817299e2902918ba123eKristian Monsendef ParseToAST(pump_src_text): 578dc0f95d653279beabeb9817299e2902918ba123eKristian Monsen """Convert the given Pump source text into an AST.""" 579dc0f95d653279beabeb9817299e2902918ba123eKristian Monsen tokens = list(Tokenize(pump_src_text)) 580c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch code_node = ParseCodeNode(tokens) 581c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch return code_node 582c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch 583c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch 584c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdochclass Env: 585c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch def __init__(self): 586c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch self.variables = [] 587c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch self.ranges = [] 588c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch 589c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch def Clone(self): 590c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch clone = Env() 591c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch clone.variables = self.variables[:] 592c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch clone.ranges = self.ranges[:] 593c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch return clone 594c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch 595c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch def PushVariable(self, var, value): 596c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch # If value looks like an int, store it as an int. 597c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch try: 598c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch int_value = int(value) 599c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch if ('%s' % int_value) == value: 600c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch value = int_value 601c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch except Exception: 602c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch pass 603c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch self.variables[:0] = [(var, value)] 604c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch 605c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch def PopVariable(self): 606c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch self.variables[:1] = [] 607c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch 608c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch def PushRange(self, var, lower, upper): 609c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch self.ranges[:0] = [(var, lower, upper)] 610c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch 611c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch def PopRange(self): 612c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch self.ranges[:1] = [] 613c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch 614c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch def GetValue(self, identifier): 615c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch for (var, value) in self.variables: 616c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch if identifier == var: 617c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch return value 618c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch 619c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch print 'ERROR: meta variable %s is undefined.' % (identifier,) 620c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch sys.exit(1) 621c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch 622c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch def EvalExp(self, exp): 623c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch try: 624c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch result = eval(exp.python_exp) 625c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch except Exception, e: 626c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch print 'ERROR: caught exception %s: %s' % (e.__class__.__name__, e) 627c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch print ('ERROR: failed to evaluate meta expression %s at %s' % 628c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch (exp.python_exp, exp.token.start)) 629c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch sys.exit(1) 630c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch return result 631c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch 632c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch def GetRange(self, identifier): 633c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch for (var, lower, upper) in self.ranges: 634c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch if identifier == var: 635c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch return (lower, upper) 636c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch 637c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch print 'ERROR: range %s is undefined.' % (identifier,) 638c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch sys.exit(1) 639c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch 640c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch 641c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdochclass Output: 642c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch def __init__(self): 643c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch self.string = '' 644c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch 645c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch def GetLastLine(self): 646c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch index = self.string.rfind('\n') 647c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch if index < 0: 648c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch return '' 649c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch 650c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch return self.string[index + 1:] 651c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch 652c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch def Append(self, s): 653c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch self.string += s 654c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch 655c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch 656c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdochdef RunAtomicCode(env, node, output): 657c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch if isinstance(node, VarNode): 658c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch identifier = node.identifier.value.strip() 659c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch result = Output() 660c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch RunAtomicCode(env.Clone(), node.atomic_code, result) 661c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch value = result.string 662c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch env.PushVariable(identifier, value) 663c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch elif isinstance(node, RangeNode): 664c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch identifier = node.identifier.value.strip() 665c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch lower = int(env.EvalExp(node.exp1)) 666c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch upper = int(env.EvalExp(node.exp2)) 667c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch env.PushRange(identifier, lower, upper) 668c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch elif isinstance(node, ForNode): 669c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch identifier = node.identifier.value.strip() 670c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch if node.sep is None: 671c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch sep = '' 672c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch else: 673c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch sep = node.sep.value 674c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch (lower, upper) = env.GetRange(identifier) 675c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch for i in range(lower, upper + 1): 676c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch new_env = env.Clone() 677c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch new_env.PushVariable(identifier, i) 678c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch RunCode(new_env, node.code, output) 679c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch if i != upper: 680c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch output.Append(sep) 681c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch elif isinstance(node, RawCodeNode): 682c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch output.Append(node.raw_code.value) 683c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch elif isinstance(node, IfNode): 684c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch cond = env.EvalExp(node.exp) 685c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch if cond: 686c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch RunCode(env.Clone(), node.then_branch, output) 687c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch elif node.else_branch is not None: 688c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch RunCode(env.Clone(), node.else_branch, output) 689c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch elif isinstance(node, ExpNode): 690c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch value = env.EvalExp(node) 691c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch output.Append('%s' % (value,)) 692c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch elif isinstance(node, LiteralDollarNode): 693c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch output.Append('$') 694c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch elif isinstance(node, CodeNode): 695c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch RunCode(env.Clone(), node, output) 696c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch else: 697c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch print 'BAD' 698c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch print node 699c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch sys.exit(1) 700c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch 701c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch 702c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdochdef RunCode(env, code_node, output): 703c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch for atomic_code in code_node.atomic_code: 704c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch RunAtomicCode(env, atomic_code, output) 705c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch 706c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch 707c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdochdef IsComment(cur_line): 708c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch return '//' in cur_line 709c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch 710c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch 711c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdochdef IsInPreprocessorDirevative(prev_lines, cur_line): 712c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch if cur_line.lstrip().startswith('#'): 713c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch return True 714c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch return prev_lines != [] and prev_lines[-1].endswith('\\') 715c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch 716c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch 717c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdochdef WrapComment(line, output): 718c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch loc = line.find('//') 719c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch before_comment = line[:loc].rstrip() 720c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch if before_comment == '': 721c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch indent = loc 722c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch else: 723c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch output.append(before_comment) 724c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch indent = len(before_comment) - len(before_comment.lstrip()) 725c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch prefix = indent*' ' + '// ' 726c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch max_len = 80 - len(prefix) 727c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch comment = line[loc + 2:].strip() 728c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch segs = [seg for seg in re.split(r'(\w+\W*)', comment) if seg != ''] 729c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch cur_line = '' 730c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch for seg in segs: 731c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch if len((cur_line + seg).rstrip()) < max_len: 732c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch cur_line += seg 733c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch else: 734c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch if cur_line.strip() != '': 735c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch output.append(prefix + cur_line.rstrip()) 736c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch cur_line = seg.lstrip() 737c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch if cur_line.strip() != '': 738c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch output.append(prefix + cur_line.strip()) 739c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch 740c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch 741c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdochdef WrapCode(line, line_concat, output): 742c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch indent = len(line) - len(line.lstrip()) 743c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch prefix = indent*' ' # Prefix of the current line 744c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch max_len = 80 - indent - len(line_concat) # Maximum length of the current line 745c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch new_prefix = prefix + 4*' ' # Prefix of a continuation line 746c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch new_max_len = max_len - 4 # Maximum length of a continuation line 747c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch # Prefers to wrap a line after a ',' or ';'. 748c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch segs = [seg for seg in re.split(r'([^,;]+[,;]?)', line.strip()) if seg != ''] 749c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch cur_line = '' # The current line without leading spaces. 750c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch for seg in segs: 751c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch # If the line is still too long, wrap at a space. 752c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch while cur_line == '' and len(seg.strip()) > max_len: 753c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch seg = seg.lstrip() 754c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch split_at = seg.rfind(' ', 0, max_len) 755c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch output.append(prefix + seg[:split_at].strip() + line_concat) 756c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch seg = seg[split_at + 1:] 757c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch prefix = new_prefix 758c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch max_len = new_max_len 759c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch 760c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch if len((cur_line + seg).rstrip()) < max_len: 761c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch cur_line = (cur_line + seg).lstrip() 762c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch else: 763c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch output.append(prefix + cur_line.rstrip() + line_concat) 764c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch prefix = new_prefix 765c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch max_len = new_max_len 766c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch cur_line = seg.lstrip() 767c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch if cur_line.strip() != '': 768c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch output.append(prefix + cur_line.strip()) 769c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch 770c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch 771c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdochdef WrapPreprocessorDirevative(line, output): 772c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch WrapCode(line, ' \\', output) 773c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch 774c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch 775c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdochdef WrapPlainCode(line, output): 776c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch WrapCode(line, '', output) 777c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch 778c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch 779c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdochdef IsHeaderGuardOrInclude(line): 780c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch return (re.match(r'^#(ifndef|define|endif\s*//)\s*[\w_]+\s*$', line) or 781c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch re.match(r'^#include\s', line)) 782c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch 783c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch 784c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdochdef WrapLongLine(line, output): 785c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch line = line.rstrip() 786c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch if len(line) <= 80: 787c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch output.append(line) 788c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch elif IsComment(line): 789c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch if IsHeaderGuardOrInclude(line): 790c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch # The style guide made an exception to allow long header guard lines 791c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch # and includes. 792c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch output.append(line) 793c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch else: 794c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch WrapComment(line, output) 795c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch elif IsInPreprocessorDirevative(output, line): 796c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch if IsHeaderGuardOrInclude(line): 797c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch # The style guide made an exception to allow long header guard lines 798c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch # and includes. 799c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch output.append(line) 800c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch else: 801c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch WrapPreprocessorDirevative(line, output) 802c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch else: 803c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch WrapPlainCode(line, output) 804c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch 805c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch 806c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdochdef BeautifyCode(string): 807c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch lines = string.splitlines() 808c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch output = [] 809c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch for line in lines: 810c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch WrapLongLine(line, output) 811c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch output2 = [line.rstrip() for line in output] 812c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch return '\n'.join(output2) + '\n' 813c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch 814c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch 815dc0f95d653279beabeb9817299e2902918ba123eKristian Monsendef ConvertFromPumpSource(src_text): 816dc0f95d653279beabeb9817299e2902918ba123eKristian Monsen """Return the text generated from the given Pump source text.""" 817dc0f95d653279beabeb9817299e2902918ba123eKristian Monsen ast = ParseToAST(StripMetaComments(src_text)) 818dc0f95d653279beabeb9817299e2902918ba123eKristian Monsen output = Output() 819dc0f95d653279beabeb9817299e2902918ba123eKristian Monsen RunCode(Env(), ast, output) 820dc0f95d653279beabeb9817299e2902918ba123eKristian Monsen return BeautifyCode(output.string) 821dc0f95d653279beabeb9817299e2902918ba123eKristian Monsen 822dc0f95d653279beabeb9817299e2902918ba123eKristian Monsen 823c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdochdef main(argv): 824c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch if len(argv) == 1: 825c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch print __doc__ 826c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch sys.exit(1) 827c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch 828c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch file_path = argv[-1] 829dc0f95d653279beabeb9817299e2902918ba123eKristian Monsen output_str = ConvertFromPumpSource(file(file_path, 'r').read()) 830c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch if file_path.endswith('.pump'): 831c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch output_file_path = file_path[:-5] 832c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch else: 833c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch output_file_path = '-' 834c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch if output_file_path == '-': 835c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch print output_str, 836c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch else: 837c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch output_file = file(output_file_path, 'w') 838c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch output_file.write('// This file was GENERATED by command:\n') 839c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch output_file.write('// %s %s\n' % 840c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch (os.path.basename(__file__), os.path.basename(file_path))) 841c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch output_file.write('// DO NOT EDIT BY HAND!!!\n\n') 842c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch output_file.write(output_str) 843c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch output_file.close() 844c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch 845c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch 846c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdochif __name__ == '__main__': 847c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch main(sys.argv) 848