1aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter#!/usr/bin/env python 2aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter# 3aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter# Copyright 2008, Google Inc. 4aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter# All rights reserved. 5aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter# 6aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter# Redistribution and use in source and binary forms, with or without 7aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter# modification, are permitted provided that the following conditions are 8aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter# met: 9aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter# 10aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter# * Redistributions of source code must retain the above copyright 11aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter# notice, this list of conditions and the following disclaimer. 12aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter# * Redistributions in binary form must reproduce the above 13aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter# copyright notice, this list of conditions and the following disclaimer 14aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter# in the documentation and/or other materials provided with the 15aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter# distribution. 16aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter# * Neither the name of Google Inc. nor the names of its 17aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter# contributors may be used to endorse or promote products derived from 18aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter# this software without specific prior written permission. 19aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter# 20aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 21aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 22aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 23aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 24aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 25aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 26aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 27aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 28aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 29aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 30aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 31aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 32aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter"""pump v0.2.0 - Pretty Useful for Meta Programming. 33aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 34aa46da279e2426caf1e103eb079dfec8124c5feeCourtney GoeltzenleuchterA tool for preprocessor meta programming. Useful for generating 35aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchterrepetitive boilerplate code. Especially useful for writing C++ 36aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchterclasses, functions, macros, and templates that need to work with 37aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchtervarious number of arguments. 38aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 39aa46da279e2426caf1e103eb079dfec8124c5feeCourtney GoeltzenleuchterUSAGE: 40aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter pump.py SOURCE_FILE 41aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 42aa46da279e2426caf1e103eb079dfec8124c5feeCourtney GoeltzenleuchterEXAMPLES: 43aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter pump.py foo.cc.pump 44aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter Converts foo.cc.pump to foo.cc. 45aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 46aa46da279e2426caf1e103eb079dfec8124c5feeCourtney GoeltzenleuchterGRAMMAR: 47aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter CODE ::= ATOMIC_CODE* 48aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter ATOMIC_CODE ::= $var ID = EXPRESSION 49aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter | $var ID = [[ CODE ]] 50aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter | $range ID EXPRESSION..EXPRESSION 51aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter | $for ID SEPARATOR [[ CODE ]] 52aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter | $($) 53aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter | $ID 54aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter | $(EXPRESSION) 55aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter | $if EXPRESSION [[ CODE ]] ELSE_BRANCH 56aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter | [[ CODE ]] 57aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter | RAW_CODE 58aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter SEPARATOR ::= RAW_CODE | EMPTY 59aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter ELSE_BRANCH ::= $else [[ CODE ]] 60aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter | $elif EXPRESSION [[ CODE ]] ELSE_BRANCH 61aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter | EMPTY 62aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter EXPRESSION has Python syntax. 63aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter""" 64aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 65aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter__author__ = 'wan@google.com (Zhanyong Wan)' 66aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 67aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchterimport os 68aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchterimport re 69aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchterimport sys 70aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 71aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 72aa46da279e2426caf1e103eb079dfec8124c5feeCourtney GoeltzenleuchterTOKEN_TABLE = [ 73aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter (re.compile(r'\$var\s+'), '$var'), 74aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter (re.compile(r'\$elif\s+'), '$elif'), 75aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter (re.compile(r'\$else\s+'), '$else'), 76aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter (re.compile(r'\$for\s+'), '$for'), 77aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter (re.compile(r'\$if\s+'), '$if'), 78aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter (re.compile(r'\$range\s+'), '$range'), 79aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter (re.compile(r'\$[_A-Za-z]\w*'), '$id'), 80aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter (re.compile(r'\$\(\$\)'), '$($)'), 81aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter (re.compile(r'\$'), '$'), 82aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter (re.compile(r'\[\[\n?'), '[['), 83aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter (re.compile(r'\]\]\n?'), ']]'), 84aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter ] 85aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 86aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 87aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchterclass Cursor: 88aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter """Represents a position (line and column) in a text file.""" 89aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 90aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter def __init__(self, line=-1, column=-1): 91aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter self.line = line 92aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter self.column = column 93aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 94aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter def __eq__(self, rhs): 95aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter return self.line == rhs.line and self.column == rhs.column 96aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 97aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter def __ne__(self, rhs): 98aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter return not self == rhs 99aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 100aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter def __lt__(self, rhs): 101aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter return self.line < rhs.line or ( 102aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter self.line == rhs.line and self.column < rhs.column) 103aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 104aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter def __le__(self, rhs): 105aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter return self < rhs or self == rhs 106aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 107aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter def __gt__(self, rhs): 108aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter return rhs < self 109aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 110aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter def __ge__(self, rhs): 111aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter return rhs <= self 112aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 113aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter def __str__(self): 114aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter if self == Eof(): 115aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter return 'EOF' 116aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter else: 117aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter return '%s(%s)' % (self.line + 1, self.column) 118aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 119aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter def __add__(self, offset): 120aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter return Cursor(self.line, self.column + offset) 121aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 122aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter def __sub__(self, offset): 123aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter return Cursor(self.line, self.column - offset) 124aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 125aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter def Clone(self): 126aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter """Returns a copy of self.""" 127aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 128aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter return Cursor(self.line, self.column) 129aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 130aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 131aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter# Special cursor to indicate the end-of-file. 132aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchterdef Eof(): 133aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter """Returns the special cursor to denote the end-of-file.""" 134aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter return Cursor(-1, -1) 135aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 136aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 137aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchterclass Token: 138aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter """Represents a token in a Pump source file.""" 139aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 140aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter def __init__(self, start=None, end=None, value=None, token_type=None): 141aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter if start is None: 142aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter self.start = Eof() 143aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter else: 144aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter self.start = start 145aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter if end is None: 146aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter self.end = Eof() 147aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter else: 148aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter self.end = end 149aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter self.value = value 150aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter self.token_type = token_type 151aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 152aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter def __str__(self): 153aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter return 'Token @%s: \'%s\' type=%s' % ( 154aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter self.start, self.value, self.token_type) 155aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 156aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter def Clone(self): 157aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter """Returns a copy of self.""" 158aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 159aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter return Token(self.start.Clone(), self.end.Clone(), self.value, 160aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter self.token_type) 161aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 162aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 163aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchterdef StartsWith(lines, pos, string): 164aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter """Returns True iff the given position in lines starts with 'string'.""" 165aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 166aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter return lines[pos.line][pos.column:].startswith(string) 167aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 168aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 169aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchterdef FindFirstInLine(line, token_table): 170aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter best_match_start = -1 171aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter for (regex, token_type) in token_table: 172aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter m = regex.search(line) 173aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter if m: 174aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter # We found regex in lines 175aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter if best_match_start < 0 or m.start() < best_match_start: 176aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter best_match_start = m.start() 177aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter best_match_length = m.end() - m.start() 178aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter best_match_token_type = token_type 179aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 180aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter if best_match_start < 0: 181aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter return None 182aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 183aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter return (best_match_start, best_match_length, best_match_token_type) 184aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 185aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 186aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchterdef FindFirst(lines, token_table, cursor): 187aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter """Finds the first occurrence of any string in strings in lines.""" 188aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 189aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter start = cursor.Clone() 190aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter cur_line_number = cursor.line 191aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter for line in lines[start.line:]: 192aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter if cur_line_number == start.line: 193aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter line = line[start.column:] 194aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter m = FindFirstInLine(line, token_table) 195aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter if m: 196aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter # We found a regex in line. 197aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter (start_column, length, token_type) = m 198aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter if cur_line_number == start.line: 199aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter start_column += start.column 200aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter found_start = Cursor(cur_line_number, start_column) 201aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter found_end = found_start + length 202aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter return MakeToken(lines, found_start, found_end, token_type) 203aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter cur_line_number += 1 204aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter # We failed to find str in lines 205aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter return None 206aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 207aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 208aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchterdef SubString(lines, start, end): 209aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter """Returns a substring in lines.""" 210aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 211aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter if end == Eof(): 212aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter end = Cursor(len(lines) - 1, len(lines[-1])) 213aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 214aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter if start >= end: 215aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter return '' 216aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 217aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter if start.line == end.line: 218aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter return lines[start.line][start.column:end.column] 219aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 220aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter result_lines = ([lines[start.line][start.column:]] + 221aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter lines[start.line + 1:end.line] + 222aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter [lines[end.line][:end.column]]) 223aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter return ''.join(result_lines) 224aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 225aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 226aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchterdef StripMetaComments(str): 227aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter """Strip meta comments from each line in the given string.""" 228aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 229aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter # First, completely remove lines containing nothing but a meta 230aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter # comment, including the trailing \n. 231aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter str = re.sub(r'^\s*\$\$.*\n', '', str) 232aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 233aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter # Then, remove meta comments from contentful lines. 234aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter return re.sub(r'\s*\$\$.*', '', str) 235aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 236aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 237aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchterdef MakeToken(lines, start, end, token_type): 238aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter """Creates a new instance of Token.""" 239aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 240aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter return Token(start, end, SubString(lines, start, end), token_type) 241aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 242aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 243aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchterdef ParseToken(lines, pos, regex, token_type): 244aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter line = lines[pos.line][pos.column:] 245aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter m = regex.search(line) 246aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter if m and not m.start(): 247aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter return MakeToken(lines, pos, pos + m.end(), token_type) 248aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter else: 249aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter print 'ERROR: %s expected at %s.' % (token_type, pos) 250aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter sys.exit(1) 251aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 252aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 253aa46da279e2426caf1e103eb079dfec8124c5feeCourtney GoeltzenleuchterID_REGEX = re.compile(r'[_A-Za-z]\w*') 254aa46da279e2426caf1e103eb079dfec8124c5feeCourtney GoeltzenleuchterEQ_REGEX = re.compile(r'=') 255aa46da279e2426caf1e103eb079dfec8124c5feeCourtney GoeltzenleuchterREST_OF_LINE_REGEX = re.compile(r'.*?(?=$|\$\$)') 256aa46da279e2426caf1e103eb079dfec8124c5feeCourtney GoeltzenleuchterOPTIONAL_WHITE_SPACES_REGEX = re.compile(r'\s*') 257aa46da279e2426caf1e103eb079dfec8124c5feeCourtney GoeltzenleuchterWHITE_SPACE_REGEX = re.compile(r'\s') 258aa46da279e2426caf1e103eb079dfec8124c5feeCourtney GoeltzenleuchterDOT_DOT_REGEX = re.compile(r'\.\.') 259aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 260aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 261aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchterdef Skip(lines, pos, regex): 262aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter line = lines[pos.line][pos.column:] 263aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter m = re.search(regex, line) 264aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter if m and not m.start(): 265aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter return pos + m.end() 266aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter else: 267aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter return pos 268aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 269aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 270aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchterdef SkipUntil(lines, pos, regex, token_type): 271aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter line = lines[pos.line][pos.column:] 272aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter m = re.search(regex, line) 273aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter if m: 274aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter return pos + m.start() 275aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter else: 276aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter print ('ERROR: %s expected on line %s after column %s.' % 277aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter (token_type, pos.line + 1, pos.column)) 278aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter sys.exit(1) 279aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 280aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 281aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchterdef ParseExpTokenInParens(lines, pos): 282aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter def ParseInParens(pos): 283aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter pos = Skip(lines, pos, OPTIONAL_WHITE_SPACES_REGEX) 284aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter pos = Skip(lines, pos, r'\(') 285aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter pos = Parse(pos) 286aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter pos = Skip(lines, pos, r'\)') 287aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter return pos 288aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 289aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter def Parse(pos): 290aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter pos = SkipUntil(lines, pos, r'\(|\)', ')') 291aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter if SubString(lines, pos, pos + 1) == '(': 292aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter pos = Parse(pos + 1) 293aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter pos = Skip(lines, pos, r'\)') 294aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter return Parse(pos) 295aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter else: 296aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter return pos 297aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 298aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter start = pos.Clone() 299aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter pos = ParseInParens(pos) 300aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter return MakeToken(lines, start, pos, 'exp') 301aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 302aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 303aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchterdef RStripNewLineFromToken(token): 304aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter if token.value.endswith('\n'): 305aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter return Token(token.start, token.end, token.value[:-1], token.token_type) 306aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter else: 307aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter return token 308aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 309aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 310aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchterdef TokenizeLines(lines, pos): 311aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter while True: 312aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter found = FindFirst(lines, TOKEN_TABLE, pos) 313aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter if not found: 314aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter yield MakeToken(lines, pos, Eof(), 'code') 315aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter return 316aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 317aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter if found.start == pos: 318aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter prev_token = None 319aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter prev_token_rstripped = None 320aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter else: 321aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter prev_token = MakeToken(lines, pos, found.start, 'code') 322aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter prev_token_rstripped = RStripNewLineFromToken(prev_token) 323aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 324aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter if found.token_type == '$var': 325aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter if prev_token_rstripped: 326aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter yield prev_token_rstripped 327aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter yield found 328aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter id_token = ParseToken(lines, found.end, ID_REGEX, 'id') 329aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter yield id_token 330aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter pos = Skip(lines, id_token.end, OPTIONAL_WHITE_SPACES_REGEX) 331aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 332aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter eq_token = ParseToken(lines, pos, EQ_REGEX, '=') 333aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter yield eq_token 334aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter pos = Skip(lines, eq_token.end, r'\s*') 335aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 336aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter if SubString(lines, pos, pos + 2) != '[[': 337aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter exp_token = ParseToken(lines, pos, REST_OF_LINE_REGEX, 'exp') 338aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter yield exp_token 339aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter pos = Cursor(exp_token.end.line + 1, 0) 340aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter elif found.token_type == '$for': 341aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter if prev_token_rstripped: 342aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter yield prev_token_rstripped 343aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter yield found 344aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter id_token = ParseToken(lines, found.end, ID_REGEX, 'id') 345aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter yield id_token 346aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter pos = Skip(lines, id_token.end, WHITE_SPACE_REGEX) 347aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter elif found.token_type == '$range': 348aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter if prev_token_rstripped: 349aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter yield prev_token_rstripped 350aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter yield found 351aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter id_token = ParseToken(lines, found.end, ID_REGEX, 'id') 352aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter yield id_token 353aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter pos = Skip(lines, id_token.end, OPTIONAL_WHITE_SPACES_REGEX) 354aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 355aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter dots_pos = SkipUntil(lines, pos, DOT_DOT_REGEX, '..') 356aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter yield MakeToken(lines, pos, dots_pos, 'exp') 357aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter yield MakeToken(lines, dots_pos, dots_pos + 2, '..') 358aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter pos = dots_pos + 2 359aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter new_pos = Cursor(pos.line + 1, 0) 360aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter yield MakeToken(lines, pos, new_pos, 'exp') 361aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter pos = new_pos 362aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter elif found.token_type == '$': 363aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter if prev_token: 364aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter yield prev_token 365aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter yield found 366aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter exp_token = ParseExpTokenInParens(lines, found.end) 367aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter yield exp_token 368aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter pos = exp_token.end 369aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter elif (found.token_type == ']]' or found.token_type == '$if' or 370aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter found.token_type == '$elif' or found.token_type == '$else'): 371aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter if prev_token_rstripped: 372aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter yield prev_token_rstripped 373aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter yield found 374aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter pos = found.end 375aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter else: 376aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter if prev_token: 377aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter yield prev_token 378aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter yield found 379aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter pos = found.end 380aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 381aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 382aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchterdef Tokenize(s): 383aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter """A generator that yields the tokens in the given string.""" 384aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter if s != '': 385aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter lines = s.splitlines(True) 386aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter for token in TokenizeLines(lines, Cursor(0, 0)): 387aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter yield token 388aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 389aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 390aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchterclass CodeNode: 391aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter def __init__(self, atomic_code_list=None): 392aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter self.atomic_code = atomic_code_list 393aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 394aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 395aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchterclass VarNode: 396aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter def __init__(self, identifier=None, atomic_code=None): 397aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter self.identifier = identifier 398aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter self.atomic_code = atomic_code 399aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 400aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 401aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchterclass RangeNode: 402aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter def __init__(self, identifier=None, exp1=None, exp2=None): 403aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter self.identifier = identifier 404aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter self.exp1 = exp1 405aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter self.exp2 = exp2 406aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 407aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 408aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchterclass ForNode: 409aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter def __init__(self, identifier=None, sep=None, code=None): 410aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter self.identifier = identifier 411aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter self.sep = sep 412aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter self.code = code 413aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 414aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 415aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchterclass ElseNode: 416aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter def __init__(self, else_branch=None): 417aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter self.else_branch = else_branch 418aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 419aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 420aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchterclass IfNode: 421aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter def __init__(self, exp=None, then_branch=None, else_branch=None): 422aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter self.exp = exp 423aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter self.then_branch = then_branch 424aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter self.else_branch = else_branch 425aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 426aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 427aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchterclass RawCodeNode: 428aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter def __init__(self, token=None): 429aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter self.raw_code = token 430aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 431aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 432aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchterclass LiteralDollarNode: 433aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter def __init__(self, token): 434aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter self.token = token 435aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 436aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 437aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchterclass ExpNode: 438aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter def __init__(self, token, python_exp): 439aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter self.token = token 440aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter self.python_exp = python_exp 441aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 442aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 443aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchterdef PopFront(a_list): 444aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter head = a_list[0] 445aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter a_list[:1] = [] 446aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter return head 447aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 448aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 449aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchterdef PushFront(a_list, elem): 450aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter a_list[:0] = [elem] 451aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 452aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 453aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchterdef PopToken(a_list, token_type=None): 454aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter token = PopFront(a_list) 455aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter if token_type is not None and token.token_type != token_type: 456aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter print 'ERROR: %s expected at %s' % (token_type, token.start) 457aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter print 'ERROR: %s found instead' % (token,) 458aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter sys.exit(1) 459aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 460aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter return token 461aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 462aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 463aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchterdef PeekToken(a_list): 464aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter if not a_list: 465aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter return None 466aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 467aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter return a_list[0] 468aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 469aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 470aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchterdef ParseExpNode(token): 471aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter python_exp = re.sub(r'([_A-Za-z]\w*)', r'self.GetValue("\1")', token.value) 472aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter return ExpNode(token, python_exp) 473aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 474aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 475aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchterdef ParseElseNode(tokens): 476aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter def Pop(token_type=None): 477aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter return PopToken(tokens, token_type) 478aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 479aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter next = PeekToken(tokens) 480aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter if not next: 481aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter return None 482aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter if next.token_type == '$else': 483aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter Pop('$else') 484aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter Pop('[[') 485aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter code_node = ParseCodeNode(tokens) 486aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter Pop(']]') 487aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter return code_node 488aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter elif next.token_type == '$elif': 489aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter Pop('$elif') 490aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter exp = Pop('code') 491aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter Pop('[[') 492aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter code_node = ParseCodeNode(tokens) 493aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter Pop(']]') 494aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter inner_else_node = ParseElseNode(tokens) 495aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter return CodeNode([IfNode(ParseExpNode(exp), code_node, inner_else_node)]) 496aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter elif not next.value.strip(): 497aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter Pop('code') 498aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter return ParseElseNode(tokens) 499aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter else: 500aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter return None 501aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 502aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 503aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchterdef ParseAtomicCodeNode(tokens): 504aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter def Pop(token_type=None): 505aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter return PopToken(tokens, token_type) 506aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 507aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter head = PopFront(tokens) 508aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter t = head.token_type 509aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter if t == 'code': 510aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter return RawCodeNode(head) 511aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter elif t == '$var': 512aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter id_token = Pop('id') 513aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter Pop('=') 514aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter next = PeekToken(tokens) 515aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter if next.token_type == 'exp': 516aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter exp_token = Pop() 517aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter return VarNode(id_token, ParseExpNode(exp_token)) 518aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter Pop('[[') 519aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter code_node = ParseCodeNode(tokens) 520aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter Pop(']]') 521aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter return VarNode(id_token, code_node) 522aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter elif t == '$for': 523aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter id_token = Pop('id') 524aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter next_token = PeekToken(tokens) 525aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter if next_token.token_type == 'code': 526aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter sep_token = next_token 527aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter Pop('code') 528aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter else: 529aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter sep_token = None 530aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter Pop('[[') 531aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter code_node = ParseCodeNode(tokens) 532aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter Pop(']]') 533aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter return ForNode(id_token, sep_token, code_node) 534aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter elif t == '$if': 535aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter exp_token = Pop('code') 536aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter Pop('[[') 537aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter code_node = ParseCodeNode(tokens) 538aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter Pop(']]') 539aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter else_node = ParseElseNode(tokens) 540aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter return IfNode(ParseExpNode(exp_token), code_node, else_node) 541aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter elif t == '$range': 542aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter id_token = Pop('id') 543aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter exp1_token = Pop('exp') 544aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter Pop('..') 545aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter exp2_token = Pop('exp') 546aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter return RangeNode(id_token, ParseExpNode(exp1_token), 547aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter ParseExpNode(exp2_token)) 548aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter elif t == '$id': 549aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter return ParseExpNode(Token(head.start + 1, head.end, head.value[1:], 'id')) 550aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter elif t == '$($)': 551aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter return LiteralDollarNode(head) 552aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter elif t == '$': 553aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter exp_token = Pop('exp') 554aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter return ParseExpNode(exp_token) 555aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter elif t == '[[': 556aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter code_node = ParseCodeNode(tokens) 557aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter Pop(']]') 558aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter return code_node 559aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter else: 560aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter PushFront(tokens, head) 561aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter return None 562aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 563aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 564aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchterdef ParseCodeNode(tokens): 565aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter atomic_code_list = [] 566aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter while True: 567aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter if not tokens: 568aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter break 569aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter atomic_code_node = ParseAtomicCodeNode(tokens) 570aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter if atomic_code_node: 571aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter atomic_code_list.append(atomic_code_node) 572aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter else: 573aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter break 574aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter return CodeNode(atomic_code_list) 575aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 576aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 577aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchterdef ParseToAST(pump_src_text): 578aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter """Convert the given Pump source text into an AST.""" 579aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter tokens = list(Tokenize(pump_src_text)) 580aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter code_node = ParseCodeNode(tokens) 581aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter return code_node 582aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 583aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 584aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchterclass Env: 585aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter def __init__(self): 586aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter self.variables = [] 587aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter self.ranges = [] 588aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 589aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter def Clone(self): 590aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter clone = Env() 591aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter clone.variables = self.variables[:] 592aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter clone.ranges = self.ranges[:] 593aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter return clone 594aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 595aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter def PushVariable(self, var, value): 596aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter # If value looks like an int, store it as an int. 597aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter try: 598aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter int_value = int(value) 599aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter if ('%s' % int_value) == value: 600aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter value = int_value 601aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter except Exception: 602aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter pass 603aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter self.variables[:0] = [(var, value)] 604aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 605aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter def PopVariable(self): 606aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter self.variables[:1] = [] 607aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 608aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter def PushRange(self, var, lower, upper): 609aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter self.ranges[:0] = [(var, lower, upper)] 610aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 611aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter def PopRange(self): 612aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter self.ranges[:1] = [] 613aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 614aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter def GetValue(self, identifier): 615aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter for (var, value) in self.variables: 616aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter if identifier == var: 617aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter return value 618aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 619aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter print 'ERROR: meta variable %s is undefined.' % (identifier,) 620aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter sys.exit(1) 621aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 622aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter def EvalExp(self, exp): 623aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter try: 624aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter result = eval(exp.python_exp) 625aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter except Exception, e: 626aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter print 'ERROR: caught exception %s: %s' % (e.__class__.__name__, e) 627aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter print ('ERROR: failed to evaluate meta expression %s at %s' % 628aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter (exp.python_exp, exp.token.start)) 629aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter sys.exit(1) 630aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter return result 631aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 632aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter def GetRange(self, identifier): 633aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter for (var, lower, upper) in self.ranges: 634aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter if identifier == var: 635aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter return (lower, upper) 636aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 637aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter print 'ERROR: range %s is undefined.' % (identifier,) 638aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter sys.exit(1) 639aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 640aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 641aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchterclass Output: 642aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter def __init__(self): 643aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter self.string = '' 644aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 645aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter def GetLastLine(self): 646aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter index = self.string.rfind('\n') 647aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter if index < 0: 648aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter return '' 649aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 650aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter return self.string[index + 1:] 651aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 652aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter def Append(self, s): 653aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter self.string += s 654aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 655aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 656aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchterdef RunAtomicCode(env, node, output): 657aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter if isinstance(node, VarNode): 658aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter identifier = node.identifier.value.strip() 659aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter result = Output() 660aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter RunAtomicCode(env.Clone(), node.atomic_code, result) 661aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter value = result.string 662aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter env.PushVariable(identifier, value) 663aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter elif isinstance(node, RangeNode): 664aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter identifier = node.identifier.value.strip() 665aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter lower = int(env.EvalExp(node.exp1)) 666aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter upper = int(env.EvalExp(node.exp2)) 667aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter env.PushRange(identifier, lower, upper) 668aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter elif isinstance(node, ForNode): 669aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter identifier = node.identifier.value.strip() 670aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter if node.sep is None: 671aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter sep = '' 672aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter else: 673aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter sep = node.sep.value 674aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter (lower, upper) = env.GetRange(identifier) 675aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter for i in range(lower, upper + 1): 676aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter new_env = env.Clone() 677aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter new_env.PushVariable(identifier, i) 678aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter RunCode(new_env, node.code, output) 679aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter if i != upper: 680aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter output.Append(sep) 681aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter elif isinstance(node, RawCodeNode): 682aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter output.Append(node.raw_code.value) 683aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter elif isinstance(node, IfNode): 684aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter cond = env.EvalExp(node.exp) 685aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter if cond: 686aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter RunCode(env.Clone(), node.then_branch, output) 687aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter elif node.else_branch is not None: 688aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter RunCode(env.Clone(), node.else_branch, output) 689aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter elif isinstance(node, ExpNode): 690aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter value = env.EvalExp(node) 691aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter output.Append('%s' % (value,)) 692aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter elif isinstance(node, LiteralDollarNode): 693aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter output.Append('$') 694aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter elif isinstance(node, CodeNode): 695aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter RunCode(env.Clone(), node, output) 696aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter else: 697aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter print 'BAD' 698aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter print node 699aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter sys.exit(1) 700aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 701aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 702aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchterdef RunCode(env, code_node, output): 703aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter for atomic_code in code_node.atomic_code: 704aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter RunAtomicCode(env, atomic_code, output) 705aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 706aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 707aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchterdef IsSingleLineComment(cur_line): 708aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter return '//' in cur_line 709aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 710aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 711aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchterdef IsInPreprocessorDirective(prev_lines, cur_line): 712aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter if cur_line.lstrip().startswith('#'): 713aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter return True 714aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter return prev_lines and prev_lines[-1].endswith('\\') 715aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 716aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 717aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchterdef WrapComment(line, output): 718aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter loc = line.find('//') 719aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter before_comment = line[:loc].rstrip() 720aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter if before_comment == '': 721aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter indent = loc 722aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter else: 723aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter output.append(before_comment) 724aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter indent = len(before_comment) - len(before_comment.lstrip()) 725aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter prefix = indent*' ' + '// ' 726aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter max_len = 80 - len(prefix) 727aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter comment = line[loc + 2:].strip() 728aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter segs = [seg for seg in re.split(r'(\w+\W*)', comment) if seg != ''] 729aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter cur_line = '' 730aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter for seg in segs: 731aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter if len((cur_line + seg).rstrip()) < max_len: 732aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter cur_line += seg 733aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter else: 734aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter if cur_line.strip() != '': 735aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter output.append(prefix + cur_line.rstrip()) 736aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter cur_line = seg.lstrip() 737aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter if cur_line.strip() != '': 738aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter output.append(prefix + cur_line.strip()) 739aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 740aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 741aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchterdef WrapCode(line, line_concat, output): 742aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter indent = len(line) - len(line.lstrip()) 743aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter prefix = indent*' ' # Prefix of the current line 744aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter max_len = 80 - indent - len(line_concat) # Maximum length of the current line 745aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter new_prefix = prefix + 4*' ' # Prefix of a continuation line 746aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter new_max_len = max_len - 4 # Maximum length of a continuation line 747aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter # Prefers to wrap a line after a ',' or ';'. 748aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter segs = [seg for seg in re.split(r'([^,;]+[,;]?)', line.strip()) if seg != ''] 749aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter cur_line = '' # The current line without leading spaces. 750aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter for seg in segs: 751aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter # If the line is still too long, wrap at a space. 752aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter while cur_line == '' and len(seg.strip()) > max_len: 753aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter seg = seg.lstrip() 754aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter split_at = seg.rfind(' ', 0, max_len) 755aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter output.append(prefix + seg[:split_at].strip() + line_concat) 756aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter seg = seg[split_at + 1:] 757aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter prefix = new_prefix 758aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter max_len = new_max_len 759aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 760aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter if len((cur_line + seg).rstrip()) < max_len: 761aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter cur_line = (cur_line + seg).lstrip() 762aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter else: 763aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter output.append(prefix + cur_line.rstrip() + line_concat) 764aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter prefix = new_prefix 765aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter max_len = new_max_len 766aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter cur_line = seg.lstrip() 767aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter if cur_line.strip() != '': 768aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter output.append(prefix + cur_line.strip()) 769aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 770aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 771aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchterdef WrapPreprocessorDirective(line, output): 772aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter WrapCode(line, ' \\', output) 773aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 774aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 775aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchterdef WrapPlainCode(line, output): 776aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter WrapCode(line, '', output) 777aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 778aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 779aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchterdef IsMultiLineIWYUPragma(line): 780aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter return re.search(r'/\* IWYU pragma: ', line) 781aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 782aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 783aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchterdef IsHeaderGuardIncludeOrOneLineIWYUPragma(line): 784aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter return (re.match(r'^#(ifndef|define|endif\s*//)\s*[\w_]+\s*$', line) or 785aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter re.match(r'^#include\s', line) or 786aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter # Don't break IWYU pragmas, either; that causes iwyu.py problems. 787aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter re.search(r'// IWYU pragma: ', line)) 788aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 789aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 790aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchterdef WrapLongLine(line, output): 791aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter line = line.rstrip() 792aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter if len(line) <= 80: 793aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter output.append(line) 794aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter elif IsSingleLineComment(line): 795aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter if IsHeaderGuardIncludeOrOneLineIWYUPragma(line): 796aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter # The style guide made an exception to allow long header guard lines, 797aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter # includes and IWYU pragmas. 798aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter output.append(line) 799aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter else: 800aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter WrapComment(line, output) 801aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter elif IsInPreprocessorDirective(output, line): 802aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter if IsHeaderGuardIncludeOrOneLineIWYUPragma(line): 803aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter # The style guide made an exception to allow long header guard lines, 804aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter # includes and IWYU pragmas. 805aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter output.append(line) 806aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter else: 807aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter WrapPreprocessorDirective(line, output) 808aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter elif IsMultiLineIWYUPragma(line): 809aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter output.append(line) 810aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter else: 811aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter WrapPlainCode(line, output) 812aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 813aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 814aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchterdef BeautifyCode(string): 815aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter lines = string.splitlines() 816aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter output = [] 817aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter for line in lines: 818aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter WrapLongLine(line, output) 819aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter output2 = [line.rstrip() for line in output] 820aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter return '\n'.join(output2) + '\n' 821aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 822aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 823aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchterdef ConvertFromPumpSource(src_text): 824aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter """Return the text generated from the given Pump source text.""" 825aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter ast = ParseToAST(StripMetaComments(src_text)) 826aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter output = Output() 827aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter RunCode(Env(), ast, output) 828aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter return BeautifyCode(output.string) 829aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 830aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 831aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchterdef main(argv): 832aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter if len(argv) == 1: 833aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter print __doc__ 834aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter sys.exit(1) 835aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 836aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter file_path = argv[-1] 837aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter output_str = ConvertFromPumpSource(file(file_path, 'r').read()) 838aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter if file_path.endswith('.pump'): 839aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter output_file_path = file_path[:-5] 840aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter else: 841aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter output_file_path = '-' 842aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter if output_file_path == '-': 843aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter print output_str, 844aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter else: 845aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter output_file = file(output_file_path, 'w') 846aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter output_file.write('// This file was GENERATED by command:\n') 847aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter output_file.write('// %s %s\n' % 848aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter (os.path.basename(__file__), os.path.basename(file_path))) 849aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter output_file.write('// DO NOT EDIT BY HAND!!!\n\n') 850aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter output_file.write(output_str) 851aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter output_file.close() 852aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 853aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter 854aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchterif __name__ == '__main__': 855aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter main(sys.argv) 856