12cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan#!/usr/bin/env python 22cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan# 32cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan# Copyright 2008, Google Inc. 42cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan# All rights reserved. 52cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan# 62cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan# Redistribution and use in source and binary forms, with or without 72cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan# modification, are permitted provided that the following conditions are 82cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan# met: 92cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan# 102cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan# * Redistributions of source code must retain the above copyright 112cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan# notice, this list of conditions and the following disclaimer. 122cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan# * Redistributions in binary form must reproduce the above 132cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan# copyright notice, this list of conditions and the following disclaimer 142cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan# in the documentation and/or other materials provided with the 152cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan# distribution. 162cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan# * Neither the name of Google Inc. nor the names of its 172cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan# contributors may be used to endorse or promote products derived from 182cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan# this software without specific prior written permission. 192cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan# 202cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 212cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 222cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 232cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 242cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 252cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 262cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 272cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 282cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 292cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 302cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 312cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan 32ff61f4eeebd7537d421ca18136adeadd71288355zhanyong.wan"""pump v0.2.0 - Pretty Useful for Meta Programming. 332cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan 342cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wanA tool for preprocessor meta programming. Useful for generating 352cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wanrepetitive boilerplate code. Especially useful for writing C++ 362cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wanclasses, functions, macros, and templates that need to work with 372cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wanvarious number of arguments. 382cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan 392cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wanUSAGE: 402cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan pump.py SOURCE_FILE 412cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan 422cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wanEXAMPLES: 432cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan pump.py foo.cc.pump 442cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan Converts foo.cc.pump to foo.cc. 452cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan 462cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wanGRAMMAR: 472cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan CODE ::= ATOMIC_CODE* 482cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan ATOMIC_CODE ::= $var ID = EXPRESSION 492cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan | $var ID = [[ CODE ]] 502cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan | $range ID EXPRESSION..EXPRESSION 512cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan | $for ID SEPARATOR [[ CODE ]] 522cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan | $($) 532cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan | $ID 542cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan | $(EXPRESSION) 552cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan | $if EXPRESSION [[ CODE ]] ELSE_BRANCH 562cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan | [[ CODE ]] 572cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan | RAW_CODE 582cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan SEPARATOR ::= RAW_CODE | EMPTY 592cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan ELSE_BRANCH ::= $else [[ CODE ]] 602cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan | $elif EXPRESSION [[ CODE ]] ELSE_BRANCH 612cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan | EMPTY 622cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan EXPRESSION has Python syntax. 632cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan""" 642cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan 652cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan__author__ = 'wan@google.com (Zhanyong Wan)' 662cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan 672cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wanimport os 682cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wanimport re 692cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wanimport sys 702cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan 712cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan 722cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wanTOKEN_TABLE = [ 732cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan (re.compile(r'\$var\s+'), '$var'), 742cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan (re.compile(r'\$elif\s+'), '$elif'), 752cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan (re.compile(r'\$else\s+'), '$else'), 762cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan (re.compile(r'\$for\s+'), '$for'), 772cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan (re.compile(r'\$if\s+'), '$if'), 782cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan (re.compile(r'\$range\s+'), '$range'), 792cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan (re.compile(r'\$[_A-Za-z]\w*'), '$id'), 802cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan (re.compile(r'\$\(\$\)'), '$($)'), 812cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan (re.compile(r'\$'), '$'), 822cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan (re.compile(r'\[\[\n?'), '[['), 832cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan (re.compile(r'\]\]\n?'), ']]'), 842cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan ] 852cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan 862cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan 872cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wanclass Cursor: 882cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan """Represents a position (line and column) in a text file.""" 892cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan 902cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan def __init__(self, line=-1, column=-1): 912cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan self.line = line 922cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan self.column = column 932cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan 942cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan def __eq__(self, rhs): 952cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan return self.line == rhs.line and self.column == rhs.column 962cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan 972cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan def __ne__(self, rhs): 982cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan return not self == rhs 992cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan 1002cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan def __lt__(self, rhs): 1012cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan return self.line < rhs.line or ( 1022cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan self.line == rhs.line and self.column < rhs.column) 1032cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan 1042cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan def __le__(self, rhs): 1052cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan return self < rhs or self == rhs 1062cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan 1072cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan def __gt__(self, rhs): 1082cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan return rhs < self 1092cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan 1102cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan def __ge__(self, rhs): 1112cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan return rhs <= self 1122cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan 1132cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan def __str__(self): 1142cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan if self == Eof(): 1152cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan return 'EOF' 1162cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan else: 1172cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan return '%s(%s)' % (self.line + 1, self.column) 1182cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan 1192cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan def __add__(self, offset): 1202cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan return Cursor(self.line, self.column + offset) 1212cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan 1222cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan def __sub__(self, offset): 1232cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan return Cursor(self.line, self.column - offset) 1242cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan 1252cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan def Clone(self): 1262cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan """Returns a copy of self.""" 1272cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan 1282cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan return Cursor(self.line, self.column) 1292cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan 1302cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan 1312cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan# Special cursor to indicate the end-of-file. 1322cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wandef Eof(): 1332cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan """Returns the special cursor to denote the end-of-file.""" 1342cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan return Cursor(-1, -1) 1352cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan 1362cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan 1372cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wanclass Token: 1382cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan """Represents a token in a Pump source file.""" 1392cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan 1402cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan def __init__(self, start=None, end=None, value=None, token_type=None): 1412cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan if start is None: 1422cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan self.start = Eof() 1432cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan else: 1442cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan self.start = start 1452cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan if end is None: 1462cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan self.end = Eof() 1472cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan else: 1482cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan self.end = end 1492cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan self.value = value 1502cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan self.token_type = token_type 1512cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan 1522cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan def __str__(self): 1532cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan return 'Token @%s: \'%s\' type=%s' % ( 1542cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan self.start, self.value, self.token_type) 1552cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan 1562cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan def Clone(self): 1572cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan """Returns a copy of self.""" 1582cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan 1592cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan return Token(self.start.Clone(), self.end.Clone(), self.value, 1602cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan self.token_type) 1612cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan 1622cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan 1632cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wandef StartsWith(lines, pos, string): 1642cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan """Returns True iff the given position in lines starts with 'string'.""" 1652cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan 1662cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan return lines[pos.line][pos.column:].startswith(string) 1672cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan 1682cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan 1692cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wandef FindFirstInLine(line, token_table): 1702cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan best_match_start = -1 1712cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan for (regex, token_type) in token_table: 1722cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan m = regex.search(line) 1732cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan if m: 1742cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan # We found regex in lines 1752cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan if best_match_start < 0 or m.start() < best_match_start: 1762cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan best_match_start = m.start() 1772cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan best_match_length = m.end() - m.start() 1782cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan best_match_token_type = token_type 1792cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan 1802cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan if best_match_start < 0: 1812cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan return None 1822cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan 1832cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan return (best_match_start, best_match_length, best_match_token_type) 1842cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan 1852cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan 1862cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wandef FindFirst(lines, token_table, cursor): 1872cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan """Finds the first occurrence of any string in strings in lines.""" 1882cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan 1892cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan start = cursor.Clone() 1902cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan cur_line_number = cursor.line 1912cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan for line in lines[start.line:]: 1922cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan if cur_line_number == start.line: 1932cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan line = line[start.column:] 1942cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan m = FindFirstInLine(line, token_table) 1952cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan if m: 1962cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan # We found a regex in line. 1972cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan (start_column, length, token_type) = m 1982cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan if cur_line_number == start.line: 1992cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan start_column += start.column 2002cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan found_start = Cursor(cur_line_number, start_column) 2012cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan found_end = found_start + length 2022cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan return MakeToken(lines, found_start, found_end, token_type) 2032cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan cur_line_number += 1 2042cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan # We failed to find str in lines 2052cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan return None 2062cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan 2072cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan 2082cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wandef SubString(lines, start, end): 2092cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan """Returns a substring in lines.""" 2102cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan 2112cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan if end == Eof(): 2122cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan end = Cursor(len(lines) - 1, len(lines[-1])) 2132cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan 2142cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan if start >= end: 2152cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan return '' 2162cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan 2172cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan if start.line == end.line: 2182cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan return lines[start.line][start.column:end.column] 2192cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan 2202cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan result_lines = ([lines[start.line][start.column:]] + 2212cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan lines[start.line + 1:end.line] + 2222cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan [lines[end.line][:end.column]]) 2232cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan return ''.join(result_lines) 2242cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan 2252cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan 226ff61f4eeebd7537d421ca18136adeadd71288355zhanyong.wandef StripMetaComments(str): 227ff61f4eeebd7537d421ca18136adeadd71288355zhanyong.wan """Strip meta comments from each line in the given string.""" 228ff61f4eeebd7537d421ca18136adeadd71288355zhanyong.wan 229ff61f4eeebd7537d421ca18136adeadd71288355zhanyong.wan # First, completely remove lines containing nothing but a meta 230ff61f4eeebd7537d421ca18136adeadd71288355zhanyong.wan # comment, including the trailing \n. 231ff61f4eeebd7537d421ca18136adeadd71288355zhanyong.wan str = re.sub(r'^\s*\$\$.*\n', '', str) 232ff61f4eeebd7537d421ca18136adeadd71288355zhanyong.wan 233ff61f4eeebd7537d421ca18136adeadd71288355zhanyong.wan # Then, remove meta comments from contentful lines. 234ff61f4eeebd7537d421ca18136adeadd71288355zhanyong.wan return re.sub(r'\s*\$\$.*', '', str) 235ff61f4eeebd7537d421ca18136adeadd71288355zhanyong.wan 236ff61f4eeebd7537d421ca18136adeadd71288355zhanyong.wan 2372cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wandef MakeToken(lines, start, end, token_type): 2382cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan """Creates a new instance of Token.""" 2392cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan 2402cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan return Token(start, end, SubString(lines, start, end), token_type) 2412cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan 2422cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan 2432cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wandef ParseToken(lines, pos, regex, token_type): 2442cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan line = lines[pos.line][pos.column:] 2452cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan m = regex.search(line) 2462cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan if m and not m.start(): 2472cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan return MakeToken(lines, pos, pos + m.end(), token_type) 2482cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan else: 2492cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan print 'ERROR: %s expected at %s.' % (token_type, pos) 2502cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan sys.exit(1) 2512cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan 2522cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan 2532cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wanID_REGEX = re.compile(r'[_A-Za-z]\w*') 2542cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wanEQ_REGEX = re.compile(r'=') 2552cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wanREST_OF_LINE_REGEX = re.compile(r'.*?(?=$|\$\$)') 2562cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wanOPTIONAL_WHITE_SPACES_REGEX = re.compile(r'\s*') 2572cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wanWHITE_SPACE_REGEX = re.compile(r'\s') 2582cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wanDOT_DOT_REGEX = re.compile(r'\.\.') 2592cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan 2602cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan 2612cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wandef Skip(lines, pos, regex): 2622cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan line = lines[pos.line][pos.column:] 2632cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan m = re.search(regex, line) 2642cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan if m and not m.start(): 2652cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan return pos + m.end() 2662cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan else: 2672cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan return pos 2682cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan 2692cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan 2702cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wandef SkipUntil(lines, pos, regex, token_type): 2712cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan line = lines[pos.line][pos.column:] 2722cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan m = re.search(regex, line) 2732cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan if m: 2742cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan return pos + m.start() 2752cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan else: 2762cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan print ('ERROR: %s expected on line %s after column %s.' % 2772cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan (token_type, pos.line + 1, pos.column)) 2782cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan sys.exit(1) 2792cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan 2802cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan 2812cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wandef ParseExpTokenInParens(lines, pos): 2822cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan def ParseInParens(pos): 2832cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan pos = Skip(lines, pos, OPTIONAL_WHITE_SPACES_REGEX) 2842cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan pos = Skip(lines, pos, r'\(') 2852cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan pos = Parse(pos) 2862cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan pos = Skip(lines, pos, r'\)') 2872cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan return pos 2882cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan 2892cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan def Parse(pos): 2902cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan pos = SkipUntil(lines, pos, r'\(|\)', ')') 2912cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan if SubString(lines, pos, pos + 1) == '(': 2922cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan pos = Parse(pos + 1) 2932cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan pos = Skip(lines, pos, r'\)') 2942cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan return Parse(pos) 2952cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan else: 2962cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan return pos 2972cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan 2982cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan start = pos.Clone() 2992cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan pos = ParseInParens(pos) 3002cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan return MakeToken(lines, start, pos, 'exp') 3012cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan 3022cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan 3032cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wandef RStripNewLineFromToken(token): 3042cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan if token.value.endswith('\n'): 3052cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan return Token(token.start, token.end, token.value[:-1], token.token_type) 3062cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan else: 3072cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan return token 3082cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan 3092cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan 3102cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wandef TokenizeLines(lines, pos): 3112cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan while True: 3122cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan found = FindFirst(lines, TOKEN_TABLE, pos) 3132cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan if not found: 3142cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan yield MakeToken(lines, pos, Eof(), 'code') 3152cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan return 3162cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan 3172cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan if found.start == pos: 3182cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan prev_token = None 3192cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan prev_token_rstripped = None 3202cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan else: 3212cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan prev_token = MakeToken(lines, pos, found.start, 'code') 3222cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan prev_token_rstripped = RStripNewLineFromToken(prev_token) 3232cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan 324ff61f4eeebd7537d421ca18136adeadd71288355zhanyong.wan if found.token_type == '$var': 3252cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan if prev_token_rstripped: 3262cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan yield prev_token_rstripped 3272cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan yield found 3282cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan id_token = ParseToken(lines, found.end, ID_REGEX, 'id') 3292cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan yield id_token 3302cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan pos = Skip(lines, id_token.end, OPTIONAL_WHITE_SPACES_REGEX) 3312cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan 3322cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan eq_token = ParseToken(lines, pos, EQ_REGEX, '=') 3332cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan yield eq_token 3342cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan pos = Skip(lines, eq_token.end, r'\s*') 3352cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan 3362cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan if SubString(lines, pos, pos + 2) != '[[': 3372cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan exp_token = ParseToken(lines, pos, REST_OF_LINE_REGEX, 'exp') 3382cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan yield exp_token 3392cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan pos = Cursor(exp_token.end.line + 1, 0) 3402cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan elif found.token_type == '$for': 3412cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan if prev_token_rstripped: 3422cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan yield prev_token_rstripped 3432cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan yield found 3442cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan id_token = ParseToken(lines, found.end, ID_REGEX, 'id') 3452cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan yield id_token 3462cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan pos = Skip(lines, id_token.end, WHITE_SPACE_REGEX) 3472cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan elif found.token_type == '$range': 3482cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan if prev_token_rstripped: 3492cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan yield prev_token_rstripped 3502cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan yield found 3512cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan id_token = ParseToken(lines, found.end, ID_REGEX, 'id') 3522cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan yield id_token 3532cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan pos = Skip(lines, id_token.end, OPTIONAL_WHITE_SPACES_REGEX) 3542cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan 3552cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan dots_pos = SkipUntil(lines, pos, DOT_DOT_REGEX, '..') 3562cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan yield MakeToken(lines, pos, dots_pos, 'exp') 3572cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan yield MakeToken(lines, dots_pos, dots_pos + 2, '..') 3582cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan pos = dots_pos + 2 3592cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan new_pos = Cursor(pos.line + 1, 0) 3602cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan yield MakeToken(lines, pos, new_pos, 'exp') 3612cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan pos = new_pos 3622cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan elif found.token_type == '$': 3632cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan if prev_token: 3642cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan yield prev_token 3652cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan yield found 3662cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan exp_token = ParseExpTokenInParens(lines, found.end) 3672cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan yield exp_token 3682cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan pos = exp_token.end 3692cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan elif (found.token_type == ']]' or found.token_type == '$if' or 3702cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan found.token_type == '$elif' or found.token_type == '$else'): 3712cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan if prev_token_rstripped: 3722cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan yield prev_token_rstripped 3732cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan yield found 3742cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan pos = found.end 3752cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan else: 3762cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan if prev_token: 3772cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan yield prev_token 3782cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan yield found 3792cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan pos = found.end 3802cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan 3812cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan 3822cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wandef Tokenize(s): 383ff61f4eeebd7537d421ca18136adeadd71288355zhanyong.wan """A generator that yields the tokens in the given string.""" 384ff61f4eeebd7537d421ca18136adeadd71288355zhanyong.wan if s != '': 385ff61f4eeebd7537d421ca18136adeadd71288355zhanyong.wan lines = s.splitlines(True) 386ff61f4eeebd7537d421ca18136adeadd71288355zhanyong.wan for token in TokenizeLines(lines, Cursor(0, 0)): 387ff61f4eeebd7537d421ca18136adeadd71288355zhanyong.wan yield token 3882cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan 3892cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan 3902cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wanclass CodeNode: 3912cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan def __init__(self, atomic_code_list=None): 3922cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan self.atomic_code = atomic_code_list 3932cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan 3942cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan 3952cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wanclass VarNode: 3962cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan def __init__(self, identifier=None, atomic_code=None): 3972cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan self.identifier = identifier 3982cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan self.atomic_code = atomic_code 3992cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan 4002cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan 4012cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wanclass RangeNode: 4022cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan def __init__(self, identifier=None, exp1=None, exp2=None): 4032cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan self.identifier = identifier 4042cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan self.exp1 = exp1 4052cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan self.exp2 = exp2 4062cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan 4072cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan 4082cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wanclass ForNode: 4092cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan def __init__(self, identifier=None, sep=None, code=None): 4102cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan self.identifier = identifier 4112cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan self.sep = sep 4122cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan self.code = code 4132cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan 4142cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan 4152cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wanclass ElseNode: 4162cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan def __init__(self, else_branch=None): 4172cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan self.else_branch = else_branch 4182cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan 4192cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan 4202cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wanclass IfNode: 4212cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan def __init__(self, exp=None, then_branch=None, else_branch=None): 4222cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan self.exp = exp 4232cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan self.then_branch = then_branch 4242cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan self.else_branch = else_branch 4252cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan 4262cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan 4272cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wanclass RawCodeNode: 4282cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan def __init__(self, token=None): 4292cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan self.raw_code = token 4302cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan 4312cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan 4322cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wanclass LiteralDollarNode: 4332cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan def __init__(self, token): 4342cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan self.token = token 4352cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan 4362cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan 4372cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wanclass ExpNode: 4382cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan def __init__(self, token, python_exp): 4392cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan self.token = token 4402cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan self.python_exp = python_exp 4412cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan 4422cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan 4432cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wandef PopFront(a_list): 4442cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan head = a_list[0] 4452cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan a_list[:1] = [] 4462cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan return head 4472cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan 4482cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan 4492cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wandef PushFront(a_list, elem): 4502cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan a_list[:0] = [elem] 4512cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan 4522cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan 4532cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wandef PopToken(a_list, token_type=None): 4542cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan token = PopFront(a_list) 4552cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan if token_type is not None and token.token_type != token_type: 4562cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan print 'ERROR: %s expected at %s' % (token_type, token.start) 4572cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan print 'ERROR: %s found instead' % (token,) 4582cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan sys.exit(1) 4592cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan 4602cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan return token 4612cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan 4622cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan 4632cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wandef PeekToken(a_list): 4642cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan if not a_list: 4652cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan return None 4662cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan 4672cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan return a_list[0] 4682cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan 4692cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan 4702cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wandef ParseExpNode(token): 4712cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan python_exp = re.sub(r'([_A-Za-z]\w*)', r'self.GetValue("\1")', token.value) 4722cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan return ExpNode(token, python_exp) 4732cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan 4742cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan 4752cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wandef ParseElseNode(tokens): 4762cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan def Pop(token_type=None): 4772cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan return PopToken(tokens, token_type) 4782cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan 4792cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan next = PeekToken(tokens) 4802cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan if not next: 4812cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan return None 4822cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan if next.token_type == '$else': 4832cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan Pop('$else') 4842cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan Pop('[[') 4852cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan code_node = ParseCodeNode(tokens) 4862cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan Pop(']]') 4872cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan return code_node 4882cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan elif next.token_type == '$elif': 4892cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan Pop('$elif') 4902cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan exp = Pop('code') 4912cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan Pop('[[') 4922cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan code_node = ParseCodeNode(tokens) 4932cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan Pop(']]') 4942cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan inner_else_node = ParseElseNode(tokens) 4952cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan return CodeNode([IfNode(ParseExpNode(exp), code_node, inner_else_node)]) 4962cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan elif not next.value.strip(): 4972cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan Pop('code') 4982cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan return ParseElseNode(tokens) 4992cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan else: 5002cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan return None 5012cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan 5022cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan 5032cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wandef ParseAtomicCodeNode(tokens): 5042cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan def Pop(token_type=None): 5052cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan return PopToken(tokens, token_type) 5062cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan 5072cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan head = PopFront(tokens) 5082cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan t = head.token_type 5092cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan if t == 'code': 5102cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan return RawCodeNode(head) 5112cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan elif t == '$var': 5122cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan id_token = Pop('id') 5132cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan Pop('=') 5142cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan next = PeekToken(tokens) 5152cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan if next.token_type == 'exp': 5162cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan exp_token = Pop() 5172cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan return VarNode(id_token, ParseExpNode(exp_token)) 5182cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan Pop('[[') 5192cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan code_node = ParseCodeNode(tokens) 5202cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan Pop(']]') 5212cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan return VarNode(id_token, code_node) 5222cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan elif t == '$for': 5232cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan id_token = Pop('id') 5242cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan next_token = PeekToken(tokens) 5252cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan if next_token.token_type == 'code': 5262cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan sep_token = next_token 5272cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan Pop('code') 5282cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan else: 5292cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan sep_token = None 5302cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan Pop('[[') 5312cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan code_node = ParseCodeNode(tokens) 5322cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan Pop(']]') 5332cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan return ForNode(id_token, sep_token, code_node) 5342cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan elif t == '$if': 5352cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan exp_token = Pop('code') 5362cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan Pop('[[') 5372cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan code_node = ParseCodeNode(tokens) 5382cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan Pop(']]') 5392cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan else_node = ParseElseNode(tokens) 5402cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan return IfNode(ParseExpNode(exp_token), code_node, else_node) 5412cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan elif t == '$range': 5422cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan id_token = Pop('id') 5432cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan exp1_token = Pop('exp') 5442cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan Pop('..') 5452cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan exp2_token = Pop('exp') 5462cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan return RangeNode(id_token, ParseExpNode(exp1_token), 5472cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan ParseExpNode(exp2_token)) 5482cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan elif t == '$id': 5492cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan return ParseExpNode(Token(head.start + 1, head.end, head.value[1:], 'id')) 5502cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan elif t == '$($)': 5512cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan return LiteralDollarNode(head) 5522cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan elif t == '$': 5532cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan exp_token = Pop('exp') 5542cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan return ParseExpNode(exp_token) 5552cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan elif t == '[[': 5562cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan code_node = ParseCodeNode(tokens) 5572cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan Pop(']]') 5582cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan return code_node 5592cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan else: 5602cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan PushFront(tokens, head) 5612cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan return None 5622cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan 5632cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan 5642cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wandef ParseCodeNode(tokens): 5652cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan atomic_code_list = [] 5662cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan while True: 5672cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan if not tokens: 5682cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan break 5692cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan atomic_code_node = ParseAtomicCodeNode(tokens) 5702cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan if atomic_code_node: 5712cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan atomic_code_list.append(atomic_code_node) 5722cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan else: 5732cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan break 5742cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan return CodeNode(atomic_code_list) 5752cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan 5762cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan 577ff61f4eeebd7537d421ca18136adeadd71288355zhanyong.wandef ParseToAST(pump_src_text): 578ff61f4eeebd7537d421ca18136adeadd71288355zhanyong.wan """Convert the given Pump source text into an AST.""" 579ff61f4eeebd7537d421ca18136adeadd71288355zhanyong.wan tokens = list(Tokenize(pump_src_text)) 5802cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan code_node = ParseCodeNode(tokens) 5812cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan return code_node 5822cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan 5832cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan 5842cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wanclass Env: 5852cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan def __init__(self): 5862cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan self.variables = [] 5872cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan self.ranges = [] 5882cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan 5892cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan def Clone(self): 5902cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan clone = Env() 5912cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan clone.variables = self.variables[:] 5922cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan clone.ranges = self.ranges[:] 5932cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan return clone 5942cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan 5952cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan def PushVariable(self, var, value): 5962cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan # If value looks like an int, store it as an int. 5972cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan try: 5982cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan int_value = int(value) 5992cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan if ('%s' % int_value) == value: 6002cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan value = int_value 6012cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan except Exception: 6022cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan pass 6032cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan self.variables[:0] = [(var, value)] 6042cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan 6052cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan def PopVariable(self): 6062cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan self.variables[:1] = [] 6072cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan 6082cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan def PushRange(self, var, lower, upper): 6092cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan self.ranges[:0] = [(var, lower, upper)] 6102cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan 6112cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan def PopRange(self): 6122cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan self.ranges[:1] = [] 6132cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan 6142cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan def GetValue(self, identifier): 6152cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan for (var, value) in self.variables: 6162cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan if identifier == var: 6172cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan return value 6182cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan 6192cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan print 'ERROR: meta variable %s is undefined.' % (identifier,) 6202cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan sys.exit(1) 6212cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan 6222cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan def EvalExp(self, exp): 6232cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan try: 6242cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan result = eval(exp.python_exp) 6252cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan except Exception, e: 6262cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan print 'ERROR: caught exception %s: %s' % (e.__class__.__name__, e) 6272cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan print ('ERROR: failed to evaluate meta expression %s at %s' % 6282cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan (exp.python_exp, exp.token.start)) 6292cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan sys.exit(1) 6302cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan return result 6312cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan 6322cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan def GetRange(self, identifier): 6332cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan for (var, lower, upper) in self.ranges: 6342cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan if identifier == var: 6352cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan return (lower, upper) 6362cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan 6372cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan print 'ERROR: range %s is undefined.' % (identifier,) 6382cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan sys.exit(1) 6392cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan 6402cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan 6412cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wanclass Output: 6422cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan def __init__(self): 6432cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan self.string = '' 6442cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan 6452cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan def GetLastLine(self): 6462cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan index = self.string.rfind('\n') 6472cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan if index < 0: 6482cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan return '' 6492cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan 6502cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan return self.string[index + 1:] 6512cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan 6522cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan def Append(self, s): 6532cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan self.string += s 6542cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan 6552cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan 6562cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wandef RunAtomicCode(env, node, output): 6572cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan if isinstance(node, VarNode): 6582cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan identifier = node.identifier.value.strip() 6592cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan result = Output() 6602cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan RunAtomicCode(env.Clone(), node.atomic_code, result) 6612cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan value = result.string 6622cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan env.PushVariable(identifier, value) 6632cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan elif isinstance(node, RangeNode): 6642cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan identifier = node.identifier.value.strip() 6652cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan lower = int(env.EvalExp(node.exp1)) 6662cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan upper = int(env.EvalExp(node.exp2)) 6672cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan env.PushRange(identifier, lower, upper) 6682cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan elif isinstance(node, ForNode): 6692cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan identifier = node.identifier.value.strip() 6702cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan if node.sep is None: 6712cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan sep = '' 6722cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan else: 6732cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan sep = node.sep.value 6742cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan (lower, upper) = env.GetRange(identifier) 6752cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan for i in range(lower, upper + 1): 6762cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan new_env = env.Clone() 6772cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan new_env.PushVariable(identifier, i) 6782cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan RunCode(new_env, node.code, output) 6792cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan if i != upper: 6802cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan output.Append(sep) 6812cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan elif isinstance(node, RawCodeNode): 6822cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan output.Append(node.raw_code.value) 6832cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan elif isinstance(node, IfNode): 6842cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan cond = env.EvalExp(node.exp) 6852cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan if cond: 6862cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan RunCode(env.Clone(), node.then_branch, output) 6872cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan elif node.else_branch is not None: 6882cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan RunCode(env.Clone(), node.else_branch, output) 6892cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan elif isinstance(node, ExpNode): 6902cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan value = env.EvalExp(node) 6912cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan output.Append('%s' % (value,)) 6922cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan elif isinstance(node, LiteralDollarNode): 6932cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan output.Append('$') 6942cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan elif isinstance(node, CodeNode): 6952cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan RunCode(env.Clone(), node, output) 6962cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan else: 6972cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan print 'BAD' 6982cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan print node 6992cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan sys.exit(1) 7002cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan 7012cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan 7022cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wandef RunCode(env, code_node, output): 7032cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan for atomic_code in code_node.atomic_code: 7042cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan RunAtomicCode(env, atomic_code, output) 7052cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan 7062cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan 70711c6ddfb54a39ccf8fe3ef8083053fae0ae4b9f6vladlosevdef IsSingleLineComment(cur_line): 7082cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan return '//' in cur_line 7092cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan 7102cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan 71111c6ddfb54a39ccf8fe3ef8083053fae0ae4b9f6vladlosevdef IsInPreprocessorDirective(prev_lines, cur_line): 7122cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan if cur_line.lstrip().startswith('#'): 7132cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan return True 71411c6ddfb54a39ccf8fe3ef8083053fae0ae4b9f6vladlosev return prev_lines and prev_lines[-1].endswith('\\') 7152cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan 7162cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan 7172cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wandef WrapComment(line, output): 7182cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan loc = line.find('//') 7192cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan before_comment = line[:loc].rstrip() 7202cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan if before_comment == '': 7212cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan indent = loc 7222cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan else: 7232cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan output.append(before_comment) 7242cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan indent = len(before_comment) - len(before_comment.lstrip()) 7252cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan prefix = indent*' ' + '// ' 7262cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan max_len = 80 - len(prefix) 7272cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan comment = line[loc + 2:].strip() 7282cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan segs = [seg for seg in re.split(r'(\w+\W*)', comment) if seg != ''] 7292cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan cur_line = '' 7302cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan for seg in segs: 7312cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan if len((cur_line + seg).rstrip()) < max_len: 7322cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan cur_line += seg 7332cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan else: 7342cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan if cur_line.strip() != '': 7352cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan output.append(prefix + cur_line.rstrip()) 7362cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan cur_line = seg.lstrip() 7372cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan if cur_line.strip() != '': 7382cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan output.append(prefix + cur_line.strip()) 7392cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan 7402cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan 7412cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wandef WrapCode(line, line_concat, output): 7422cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan indent = len(line) - len(line.lstrip()) 7432cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan prefix = indent*' ' # Prefix of the current line 7442cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan max_len = 80 - indent - len(line_concat) # Maximum length of the current line 7452cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan new_prefix = prefix + 4*' ' # Prefix of a continuation line 7462cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan new_max_len = max_len - 4 # Maximum length of a continuation line 7472cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan # Prefers to wrap a line after a ',' or ';'. 7482cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan segs = [seg for seg in re.split(r'([^,;]+[,;]?)', line.strip()) if seg != ''] 7492cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan cur_line = '' # The current line without leading spaces. 7502cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan for seg in segs: 7512cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan # If the line is still too long, wrap at a space. 7522cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan while cur_line == '' and len(seg.strip()) > max_len: 7532cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan seg = seg.lstrip() 7542cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan split_at = seg.rfind(' ', 0, max_len) 7552cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan output.append(prefix + seg[:split_at].strip() + line_concat) 7562cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan seg = seg[split_at + 1:] 7572cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan prefix = new_prefix 7582cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan max_len = new_max_len 7592cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan 7602cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan if len((cur_line + seg).rstrip()) < max_len: 7612cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan cur_line = (cur_line + seg).lstrip() 7622cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan else: 7632cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan output.append(prefix + cur_line.rstrip() + line_concat) 7642cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan prefix = new_prefix 7652cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan max_len = new_max_len 7662cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan cur_line = seg.lstrip() 7672cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan if cur_line.strip() != '': 7682cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan output.append(prefix + cur_line.strip()) 7692cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan 7702cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan 77111c6ddfb54a39ccf8fe3ef8083053fae0ae4b9f6vladlosevdef WrapPreprocessorDirective(line, output): 7722cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan WrapCode(line, ' \\', output) 7732cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan 7742cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan 7752cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wandef WrapPlainCode(line, output): 7762cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan WrapCode(line, '', output) 7772cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan 7782cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan 77911c6ddfb54a39ccf8fe3ef8083053fae0ae4b9f6vladlosevdef IsMultiLineIWYUPragma(line): 78011c6ddfb54a39ccf8fe3ef8083053fae0ae4b9f6vladlosev return re.search(r'/\* IWYU pragma: ', line) 78111c6ddfb54a39ccf8fe3ef8083053fae0ae4b9f6vladlosev 78211c6ddfb54a39ccf8fe3ef8083053fae0ae4b9f6vladlosev 78311c6ddfb54a39ccf8fe3ef8083053fae0ae4b9f6vladlosevdef IsHeaderGuardIncludeOrOneLineIWYUPragma(line): 7842cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan return (re.match(r'^#(ifndef|define|endif\s*//)\s*[\w_]+\s*$', line) or 78511c6ddfb54a39ccf8fe3ef8083053fae0ae4b9f6vladlosev re.match(r'^#include\s', line) or 78611c6ddfb54a39ccf8fe3ef8083053fae0ae4b9f6vladlosev # Don't break IWYU pragmas, either; that causes iwyu.py problems. 78711c6ddfb54a39ccf8fe3ef8083053fae0ae4b9f6vladlosev re.search(r'// IWYU pragma: ', line)) 7882cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan 7892cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan 7902cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wandef WrapLongLine(line, output): 7912cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan line = line.rstrip() 7922cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan if len(line) <= 80: 7932cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan output.append(line) 79411c6ddfb54a39ccf8fe3ef8083053fae0ae4b9f6vladlosev elif IsSingleLineComment(line): 79511c6ddfb54a39ccf8fe3ef8083053fae0ae4b9f6vladlosev if IsHeaderGuardIncludeOrOneLineIWYUPragma(line): 79611c6ddfb54a39ccf8fe3ef8083053fae0ae4b9f6vladlosev # The style guide made an exception to allow long header guard lines, 79711c6ddfb54a39ccf8fe3ef8083053fae0ae4b9f6vladlosev # includes and IWYU pragmas. 7982cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan output.append(line) 7992cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan else: 8002cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan WrapComment(line, output) 80111c6ddfb54a39ccf8fe3ef8083053fae0ae4b9f6vladlosev elif IsInPreprocessorDirective(output, line): 80211c6ddfb54a39ccf8fe3ef8083053fae0ae4b9f6vladlosev if IsHeaderGuardIncludeOrOneLineIWYUPragma(line): 80311c6ddfb54a39ccf8fe3ef8083053fae0ae4b9f6vladlosev # The style guide made an exception to allow long header guard lines, 80411c6ddfb54a39ccf8fe3ef8083053fae0ae4b9f6vladlosev # includes and IWYU pragmas. 8052cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan output.append(line) 8062cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan else: 80711c6ddfb54a39ccf8fe3ef8083053fae0ae4b9f6vladlosev WrapPreprocessorDirective(line, output) 80811c6ddfb54a39ccf8fe3ef8083053fae0ae4b9f6vladlosev elif IsMultiLineIWYUPragma(line): 80911c6ddfb54a39ccf8fe3ef8083053fae0ae4b9f6vladlosev output.append(line) 8102cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan else: 8112cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan WrapPlainCode(line, output) 8122cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan 8132cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan 8142cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wandef BeautifyCode(string): 8152cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan lines = string.splitlines() 8162cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan output = [] 8172cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan for line in lines: 8182cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan WrapLongLine(line, output) 8192cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan output2 = [line.rstrip() for line in output] 8202cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan return '\n'.join(output2) + '\n' 8212cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan 8222cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan 823ff61f4eeebd7537d421ca18136adeadd71288355zhanyong.wandef ConvertFromPumpSource(src_text): 824ff61f4eeebd7537d421ca18136adeadd71288355zhanyong.wan """Return the text generated from the given Pump source text.""" 825ff61f4eeebd7537d421ca18136adeadd71288355zhanyong.wan ast = ParseToAST(StripMetaComments(src_text)) 826ff61f4eeebd7537d421ca18136adeadd71288355zhanyong.wan output = Output() 827ff61f4eeebd7537d421ca18136adeadd71288355zhanyong.wan RunCode(Env(), ast, output) 828ff61f4eeebd7537d421ca18136adeadd71288355zhanyong.wan return BeautifyCode(output.string) 829ff61f4eeebd7537d421ca18136adeadd71288355zhanyong.wan 830ff61f4eeebd7537d421ca18136adeadd71288355zhanyong.wan 8312cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wandef main(argv): 8322cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan if len(argv) == 1: 8332cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan print __doc__ 8342cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan sys.exit(1) 8352cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan 8362cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan file_path = argv[-1] 837ff61f4eeebd7537d421ca18136adeadd71288355zhanyong.wan output_str = ConvertFromPumpSource(file(file_path, 'r').read()) 8382cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan if file_path.endswith('.pump'): 8392cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan output_file_path = file_path[:-5] 8402cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan else: 8412cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan output_file_path = '-' 8422cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan if output_file_path == '-': 8432cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan print output_str, 8442cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan else: 8452cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan output_file = file(output_file_path, 'w') 8462cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan output_file.write('// This file was GENERATED by command:\n') 8472cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan output_file.write('// %s %s\n' % 8482cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan (os.path.basename(__file__), os.path.basename(file_path))) 8492cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan output_file.write('// DO NOT EDIT BY HAND!!!\n\n') 8502cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan output_file.write(output_str) 8512cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan output_file.close() 8522cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan 8532cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan 8542cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wanif __name__ == '__main__': 8552cd90032a44bd1b1cf9756a7561a86557de204d5zhanyong.wan main(sys.argv) 856