1c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch#!/usr/bin/env python
2c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch#
3c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch# Copyright 2008, Google Inc.
4c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch# All rights reserved.
5c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch#
6c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch# Redistribution and use in source and binary forms, with or without
7c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch# modification, are permitted provided that the following conditions are
8c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch# met:
9c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch#
10c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch#     * Redistributions of source code must retain the above copyright
11c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch# notice, this list of conditions and the following disclaimer.
12c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch#     * Redistributions in binary form must reproduce the above
13c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch# copyright notice, this list of conditions and the following disclaimer
14c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch# in the documentation and/or other materials provided with the
15c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch# distribution.
16c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch#     * Neither the name of Google Inc. nor the names of its
17c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch# contributors may be used to endorse or promote products derived from
18c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch# this software without specific prior written permission.
19c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch#
20c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
21c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
22c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
23c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
24c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
25c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
26c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
27c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
28c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
29c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
30c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
31c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch
32dc0f95d653279beabeb9817299e2902918ba123eKristian Monsen"""pump v0.2.0 - Pretty Useful for Meta Programming.
33c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch
34c407dc5cd9bdc5668497f21b26b09d988ab439deBen MurdochA tool for preprocessor meta programming.  Useful for generating
35c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdochrepetitive boilerplate code.  Especially useful for writing C++
36c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdochclasses, functions, macros, and templates that need to work with
37c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdochvarious number of arguments.
38c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch
39c407dc5cd9bdc5668497f21b26b09d988ab439deBen MurdochUSAGE:
40c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch       pump.py SOURCE_FILE
41c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch
42c407dc5cd9bdc5668497f21b26b09d988ab439deBen MurdochEXAMPLES:
43c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch       pump.py foo.cc.pump
44c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch         Converts foo.cc.pump to foo.cc.
45c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch
46c407dc5cd9bdc5668497f21b26b09d988ab439deBen MurdochGRAMMAR:
47c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch       CODE ::= ATOMIC_CODE*
48c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch       ATOMIC_CODE ::= $var ID = EXPRESSION
49c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch           | $var ID = [[ CODE ]]
50c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch           | $range ID EXPRESSION..EXPRESSION
51c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch           | $for ID SEPARATOR [[ CODE ]]
52c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch           | $($)
53c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch           | $ID
54c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch           | $(EXPRESSION)
55c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch           | $if EXPRESSION [[ CODE ]] ELSE_BRANCH
56c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch           | [[ CODE ]]
57c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch           | RAW_CODE
58c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch       SEPARATOR ::= RAW_CODE | EMPTY
59c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch       ELSE_BRANCH ::= $else [[ CODE ]]
60c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch           | $elif EXPRESSION [[ CODE ]] ELSE_BRANCH
61c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch           | EMPTY
62c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch       EXPRESSION has Python syntax.
63c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch"""
64c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch
65c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch__author__ = 'wan@google.com (Zhanyong Wan)'
66c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch
67c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdochimport os
68c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdochimport re
69c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdochimport sys
70c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch
71c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch
72c407dc5cd9bdc5668497f21b26b09d988ab439deBen MurdochTOKEN_TABLE = [
73c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    (re.compile(r'\$var\s+'), '$var'),
74c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    (re.compile(r'\$elif\s+'), '$elif'),
75c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    (re.compile(r'\$else\s+'), '$else'),
76c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    (re.compile(r'\$for\s+'), '$for'),
77c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    (re.compile(r'\$if\s+'), '$if'),
78c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    (re.compile(r'\$range\s+'), '$range'),
79c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    (re.compile(r'\$[_A-Za-z]\w*'), '$id'),
80c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    (re.compile(r'\$\(\$\)'), '$($)'),
81c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    (re.compile(r'\$'), '$'),
82c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    (re.compile(r'\[\[\n?'), '[['),
83c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    (re.compile(r'\]\]\n?'), ']]'),
84c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    ]
85c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch
86c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch
87c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdochclass Cursor:
88c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  """Represents a position (line and column) in a text file."""
89c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch
90c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  def __init__(self, line=-1, column=-1):
91c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    self.line = line
92c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    self.column = column
93c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch
94c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  def __eq__(self, rhs):
95c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    return self.line == rhs.line and self.column == rhs.column
96c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch
97c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  def __ne__(self, rhs):
98c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    return not self == rhs
99c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch
100c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  def __lt__(self, rhs):
101c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    return self.line < rhs.line or (
102c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch        self.line == rhs.line and self.column < rhs.column)
103c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch
104c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  def __le__(self, rhs):
105c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    return self < rhs or self == rhs
106c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch
107c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  def __gt__(self, rhs):
108c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    return rhs < self
109c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch
110c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  def __ge__(self, rhs):
111c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    return rhs <= self
112c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch
113c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  def __str__(self):
114c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    if self == Eof():
115c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch      return 'EOF'
116c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    else:
117c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch      return '%s(%s)' % (self.line + 1, self.column)
118c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch
119c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  def __add__(self, offset):
120c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    return Cursor(self.line, self.column + offset)
121c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch
122c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  def __sub__(self, offset):
123c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    return Cursor(self.line, self.column - offset)
124c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch
125c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  def Clone(self):
126c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    """Returns a copy of self."""
127c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch
128c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    return Cursor(self.line, self.column)
129c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch
130c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch
131c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch# Special cursor to indicate the end-of-file.
132c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdochdef Eof():
133c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  """Returns the special cursor to denote the end-of-file."""
134c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  return Cursor(-1, -1)
135c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch
136c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch
137c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdochclass Token:
138c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  """Represents a token in a Pump source file."""
139c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch
140c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  def __init__(self, start=None, end=None, value=None, token_type=None):
141c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    if start is None:
142c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch      self.start = Eof()
143c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    else:
144c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch      self.start = start
145c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    if end is None:
146c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch      self.end = Eof()
147c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    else:
148c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch      self.end = end
149c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    self.value = value
150c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    self.token_type = token_type
151c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch
152c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  def __str__(self):
153c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    return 'Token @%s: \'%s\' type=%s' % (
154c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch        self.start, self.value, self.token_type)
155c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch
156c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  def Clone(self):
157c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    """Returns a copy of self."""
158c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch
159c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    return Token(self.start.Clone(), self.end.Clone(), self.value,
160c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch                 self.token_type)
161c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch
162c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch
163c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdochdef StartsWith(lines, pos, string):
164c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  """Returns True iff the given position in lines starts with 'string'."""
165c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch
166c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  return lines[pos.line][pos.column:].startswith(string)
167c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch
168c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch
169c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdochdef FindFirstInLine(line, token_table):
170c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  best_match_start = -1
171c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  for (regex, token_type) in token_table:
172c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    m = regex.search(line)
173c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    if m:
174c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch      # We found regex in lines
175c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch      if best_match_start < 0 or m.start() < best_match_start:
176c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch        best_match_start = m.start()
177c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch        best_match_length = m.end() - m.start()
178c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch        best_match_token_type = token_type
179c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch
180c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  if best_match_start < 0:
181c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    return None
182c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch
183c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  return (best_match_start, best_match_length, best_match_token_type)
184c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch
185c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch
186c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdochdef FindFirst(lines, token_table, cursor):
187c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  """Finds the first occurrence of any string in strings in lines."""
188c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch
189c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  start = cursor.Clone()
190c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  cur_line_number = cursor.line
191c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  for line in lines[start.line:]:
192c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    if cur_line_number == start.line:
193c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch      line = line[start.column:]
194c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    m = FindFirstInLine(line, token_table)
195c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    if m:
196c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch      # We found a regex in line.
197c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch      (start_column, length, token_type) = m
198c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch      if cur_line_number == start.line:
199c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch        start_column += start.column
200c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch      found_start = Cursor(cur_line_number, start_column)
201c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch      found_end = found_start + length
202c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch      return MakeToken(lines, found_start, found_end, token_type)
203c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    cur_line_number += 1
204c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  # We failed to find str in lines
205c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  return None
206c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch
207c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch
208c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdochdef SubString(lines, start, end):
209c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  """Returns a substring in lines."""
210c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch
211c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  if end == Eof():
212c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    end = Cursor(len(lines) - 1, len(lines[-1]))
213c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch
214c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  if start >= end:
215c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    return ''
216c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch
217c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  if start.line == end.line:
218c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    return lines[start.line][start.column:end.column]
219c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch
220c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  result_lines = ([lines[start.line][start.column:]] +
221c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch                  lines[start.line + 1:end.line] +
222c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch                  [lines[end.line][:end.column]])
223c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  return ''.join(result_lines)
224c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch
225c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch
226dc0f95d653279beabeb9817299e2902918ba123eKristian Monsendef StripMetaComments(str):
227dc0f95d653279beabeb9817299e2902918ba123eKristian Monsen  """Strip meta comments from each line in the given string."""
228dc0f95d653279beabeb9817299e2902918ba123eKristian Monsen
229dc0f95d653279beabeb9817299e2902918ba123eKristian Monsen  # First, completely remove lines containing nothing but a meta
230dc0f95d653279beabeb9817299e2902918ba123eKristian Monsen  # comment, including the trailing \n.
231dc0f95d653279beabeb9817299e2902918ba123eKristian Monsen  str = re.sub(r'^\s*\$\$.*\n', '', str)
232dc0f95d653279beabeb9817299e2902918ba123eKristian Monsen
233dc0f95d653279beabeb9817299e2902918ba123eKristian Monsen  # Then, remove meta comments from contentful lines.
234dc0f95d653279beabeb9817299e2902918ba123eKristian Monsen  return re.sub(r'\s*\$\$.*', '', str)
235dc0f95d653279beabeb9817299e2902918ba123eKristian Monsen
236dc0f95d653279beabeb9817299e2902918ba123eKristian Monsen
237c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdochdef MakeToken(lines, start, end, token_type):
238c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  """Creates a new instance of Token."""
239c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch
240c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  return Token(start, end, SubString(lines, start, end), token_type)
241c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch
242c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch
243c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdochdef ParseToken(lines, pos, regex, token_type):
244c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  line = lines[pos.line][pos.column:]
245c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  m = regex.search(line)
246c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  if m and not m.start():
247c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    return MakeToken(lines, pos, pos + m.end(), token_type)
248c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  else:
249c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    print 'ERROR: %s expected at %s.' % (token_type, pos)
250c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    sys.exit(1)
251c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch
252c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch
253c407dc5cd9bdc5668497f21b26b09d988ab439deBen MurdochID_REGEX = re.compile(r'[_A-Za-z]\w*')
254c407dc5cd9bdc5668497f21b26b09d988ab439deBen MurdochEQ_REGEX = re.compile(r'=')
255c407dc5cd9bdc5668497f21b26b09d988ab439deBen MurdochREST_OF_LINE_REGEX = re.compile(r'.*?(?=$|\$\$)')
256c407dc5cd9bdc5668497f21b26b09d988ab439deBen MurdochOPTIONAL_WHITE_SPACES_REGEX = re.compile(r'\s*')
257c407dc5cd9bdc5668497f21b26b09d988ab439deBen MurdochWHITE_SPACE_REGEX = re.compile(r'\s')
258c407dc5cd9bdc5668497f21b26b09d988ab439deBen MurdochDOT_DOT_REGEX = re.compile(r'\.\.')
259c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch
260c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch
261c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdochdef Skip(lines, pos, regex):
262c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  line = lines[pos.line][pos.column:]
263c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  m = re.search(regex, line)
264c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  if m and not m.start():
265c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    return pos + m.end()
266c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  else:
267c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    return pos
268c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch
269c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch
270c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdochdef SkipUntil(lines, pos, regex, token_type):
271c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  line = lines[pos.line][pos.column:]
272c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  m = re.search(regex, line)
273c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  if m:
274c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    return pos + m.start()
275c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  else:
276c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    print ('ERROR: %s expected on line %s after column %s.' %
277c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch           (token_type, pos.line + 1, pos.column))
278c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    sys.exit(1)
279c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch
280c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch
281c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdochdef ParseExpTokenInParens(lines, pos):
282c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  def ParseInParens(pos):
283c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    pos = Skip(lines, pos, OPTIONAL_WHITE_SPACES_REGEX)
284c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    pos = Skip(lines, pos, r'\(')
285c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    pos = Parse(pos)
286c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    pos = Skip(lines, pos, r'\)')
287c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    return pos
288c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch
289c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  def Parse(pos):
290c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    pos = SkipUntil(lines, pos, r'\(|\)', ')')
291c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    if SubString(lines, pos, pos + 1) == '(':
292c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch      pos = Parse(pos + 1)
293c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch      pos = Skip(lines, pos, r'\)')
294c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch      return Parse(pos)
295c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    else:
296c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch      return pos
297c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch
298c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  start = pos.Clone()
299c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  pos = ParseInParens(pos)
300c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  return MakeToken(lines, start, pos, 'exp')
301c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch
302c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch
303c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdochdef RStripNewLineFromToken(token):
304c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  if token.value.endswith('\n'):
305c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    return Token(token.start, token.end, token.value[:-1], token.token_type)
306c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  else:
307c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    return token
308c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch
309c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch
310c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdochdef TokenizeLines(lines, pos):
311c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  while True:
312c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    found = FindFirst(lines, TOKEN_TABLE, pos)
313c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    if not found:
314c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch      yield MakeToken(lines, pos, Eof(), 'code')
315c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch      return
316c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch
317c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    if found.start == pos:
318c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch      prev_token = None
319c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch      prev_token_rstripped = None
320c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    else:
321c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch      prev_token = MakeToken(lines, pos, found.start, 'code')
322c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch      prev_token_rstripped = RStripNewLineFromToken(prev_token)
323c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch
324dc0f95d653279beabeb9817299e2902918ba123eKristian Monsen    if found.token_type == '$var':
325c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch      if prev_token_rstripped:
326c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch        yield prev_token_rstripped
327c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch      yield found
328c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch      id_token = ParseToken(lines, found.end, ID_REGEX, 'id')
329c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch      yield id_token
330c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch      pos = Skip(lines, id_token.end, OPTIONAL_WHITE_SPACES_REGEX)
331c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch
332c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch      eq_token = ParseToken(lines, pos, EQ_REGEX, '=')
333c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch      yield eq_token
334c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch      pos = Skip(lines, eq_token.end, r'\s*')
335c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch
336c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch      if SubString(lines, pos, pos + 2) != '[[':
337c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch        exp_token = ParseToken(lines, pos, REST_OF_LINE_REGEX, 'exp')
338c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch        yield exp_token
339c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch        pos = Cursor(exp_token.end.line + 1, 0)
340c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    elif found.token_type == '$for':
341c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch      if prev_token_rstripped:
342c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch        yield prev_token_rstripped
343c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch      yield found
344c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch      id_token = ParseToken(lines, found.end, ID_REGEX, 'id')
345c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch      yield id_token
346c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch      pos = Skip(lines, id_token.end, WHITE_SPACE_REGEX)
347c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    elif found.token_type == '$range':
348c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch      if prev_token_rstripped:
349c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch        yield prev_token_rstripped
350c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch      yield found
351c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch      id_token = ParseToken(lines, found.end, ID_REGEX, 'id')
352c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch      yield id_token
353c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch      pos = Skip(lines, id_token.end, OPTIONAL_WHITE_SPACES_REGEX)
354c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch
355c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch      dots_pos = SkipUntil(lines, pos, DOT_DOT_REGEX, '..')
356c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch      yield MakeToken(lines, pos, dots_pos, 'exp')
357c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch      yield MakeToken(lines, dots_pos, dots_pos + 2, '..')
358c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch      pos = dots_pos + 2
359c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch      new_pos = Cursor(pos.line + 1, 0)
360c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch      yield MakeToken(lines, pos, new_pos, 'exp')
361c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch      pos = new_pos
362c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    elif found.token_type == '$':
363c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch      if prev_token:
364c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch        yield prev_token
365c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch      yield found
366c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch      exp_token = ParseExpTokenInParens(lines, found.end)
367c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch      yield exp_token
368c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch      pos = exp_token.end
369c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    elif (found.token_type == ']]' or found.token_type == '$if' or
370c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch          found.token_type == '$elif' or found.token_type == '$else'):
371c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch      if prev_token_rstripped:
372c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch        yield prev_token_rstripped
373c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch      yield found
374c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch      pos = found.end
375c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    else:
376c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch      if prev_token:
377c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch        yield prev_token
378c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch      yield found
379c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch      pos = found.end
380c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch
381c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch
382c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdochdef Tokenize(s):
383dc0f95d653279beabeb9817299e2902918ba123eKristian Monsen  """A generator that yields the tokens in the given string."""
384dc0f95d653279beabeb9817299e2902918ba123eKristian Monsen  if s != '':
385dc0f95d653279beabeb9817299e2902918ba123eKristian Monsen    lines = s.splitlines(True)
386dc0f95d653279beabeb9817299e2902918ba123eKristian Monsen    for token in TokenizeLines(lines, Cursor(0, 0)):
387dc0f95d653279beabeb9817299e2902918ba123eKristian Monsen      yield token
388c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch
389c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch
390c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdochclass CodeNode:
391c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  def __init__(self, atomic_code_list=None):
392c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    self.atomic_code = atomic_code_list
393c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch
394c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch
395c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdochclass VarNode:
396c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  def __init__(self, identifier=None, atomic_code=None):
397c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    self.identifier = identifier
398c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    self.atomic_code = atomic_code
399c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch
400c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch
401c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdochclass RangeNode:
402c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  def __init__(self, identifier=None, exp1=None, exp2=None):
403c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    self.identifier = identifier
404c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    self.exp1 = exp1
405c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    self.exp2 = exp2
406c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch
407c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch
408c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdochclass ForNode:
409c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  def __init__(self, identifier=None, sep=None, code=None):
410c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    self.identifier = identifier
411c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    self.sep = sep
412c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    self.code = code
413c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch
414c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch
415c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdochclass ElseNode:
416c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  def __init__(self, else_branch=None):
417c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    self.else_branch = else_branch
418c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch
419c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch
420c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdochclass IfNode:
421c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  def __init__(self, exp=None, then_branch=None, else_branch=None):
422c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    self.exp = exp
423c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    self.then_branch = then_branch
424c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    self.else_branch = else_branch
425c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch
426c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch
427c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdochclass RawCodeNode:
428c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  def __init__(self, token=None):
429c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    self.raw_code = token
430c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch
431c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch
432c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdochclass LiteralDollarNode:
433c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  def __init__(self, token):
434c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    self.token = token
435c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch
436c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch
437c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdochclass ExpNode:
438c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  def __init__(self, token, python_exp):
439c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    self.token = token
440c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    self.python_exp = python_exp
441c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch
442c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch
443c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdochdef PopFront(a_list):
444c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  head = a_list[0]
445c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  a_list[:1] = []
446c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  return head
447c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch
448c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch
449c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdochdef PushFront(a_list, elem):
450c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  a_list[:0] = [elem]
451c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch
452c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch
453c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdochdef PopToken(a_list, token_type=None):
454c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  token = PopFront(a_list)
455c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  if token_type is not None and token.token_type != token_type:
456c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    print 'ERROR: %s expected at %s' % (token_type, token.start)
457c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    print 'ERROR: %s found instead' % (token,)
458c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    sys.exit(1)
459c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch
460c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  return token
461c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch
462c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch
463c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdochdef PeekToken(a_list):
464c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  if not a_list:
465c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    return None
466c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch
467c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  return a_list[0]
468c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch
469c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch
470c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdochdef ParseExpNode(token):
471c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  python_exp = re.sub(r'([_A-Za-z]\w*)', r'self.GetValue("\1")', token.value)
472c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  return ExpNode(token, python_exp)
473c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch
474c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch
475c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdochdef ParseElseNode(tokens):
476c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  def Pop(token_type=None):
477c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    return PopToken(tokens, token_type)
478c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch
479c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  next = PeekToken(tokens)
480c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  if not next:
481c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    return None
482c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  if next.token_type == '$else':
483c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    Pop('$else')
484c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    Pop('[[')
485c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    code_node = ParseCodeNode(tokens)
486c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    Pop(']]')
487c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    return code_node
488c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  elif next.token_type == '$elif':
489c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    Pop('$elif')
490c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    exp = Pop('code')
491c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    Pop('[[')
492c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    code_node = ParseCodeNode(tokens)
493c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    Pop(']]')
494c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    inner_else_node = ParseElseNode(tokens)
495c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    return CodeNode([IfNode(ParseExpNode(exp), code_node, inner_else_node)])
496c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  elif not next.value.strip():
497c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    Pop('code')
498c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    return ParseElseNode(tokens)
499c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  else:
500c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    return None
501c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch
502c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch
503c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdochdef ParseAtomicCodeNode(tokens):
504c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  def Pop(token_type=None):
505c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    return PopToken(tokens, token_type)
506c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch
507c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  head = PopFront(tokens)
508c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  t = head.token_type
509c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  if t == 'code':
510c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    return RawCodeNode(head)
511c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  elif t == '$var':
512c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    id_token = Pop('id')
513c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    Pop('=')
514c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    next = PeekToken(tokens)
515c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    if next.token_type == 'exp':
516c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch      exp_token = Pop()
517c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch      return VarNode(id_token, ParseExpNode(exp_token))
518c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    Pop('[[')
519c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    code_node = ParseCodeNode(tokens)
520c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    Pop(']]')
521c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    return VarNode(id_token, code_node)
522c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  elif t == '$for':
523c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    id_token = Pop('id')
524c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    next_token = PeekToken(tokens)
525c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    if next_token.token_type == 'code':
526c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch      sep_token = next_token
527c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch      Pop('code')
528c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    else:
529c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch      sep_token = None
530c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    Pop('[[')
531c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    code_node = ParseCodeNode(tokens)
532c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    Pop(']]')
533c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    return ForNode(id_token, sep_token, code_node)
534c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  elif t == '$if':
535c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    exp_token = Pop('code')
536c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    Pop('[[')
537c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    code_node = ParseCodeNode(tokens)
538c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    Pop(']]')
539c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    else_node = ParseElseNode(tokens)
540c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    return IfNode(ParseExpNode(exp_token), code_node, else_node)
541c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  elif t == '$range':
542c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    id_token = Pop('id')
543c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    exp1_token = Pop('exp')
544c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    Pop('..')
545c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    exp2_token = Pop('exp')
546c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    return RangeNode(id_token, ParseExpNode(exp1_token),
547c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch                     ParseExpNode(exp2_token))
548c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  elif t == '$id':
549c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    return ParseExpNode(Token(head.start + 1, head.end, head.value[1:], 'id'))
550c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  elif t == '$($)':
551c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    return LiteralDollarNode(head)
552c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  elif t == '$':
553c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    exp_token = Pop('exp')
554c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    return ParseExpNode(exp_token)
555c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  elif t == '[[':
556c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    code_node = ParseCodeNode(tokens)
557c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    Pop(']]')
558c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    return code_node
559c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  else:
560c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    PushFront(tokens, head)
561c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    return None
562c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch
563c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch
564c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdochdef ParseCodeNode(tokens):
565c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  atomic_code_list = []
566c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  while True:
567c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    if not tokens:
568c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch      break
569c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    atomic_code_node = ParseAtomicCodeNode(tokens)
570c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    if atomic_code_node:
571c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch      atomic_code_list.append(atomic_code_node)
572c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    else:
573c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch      break
574c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  return CodeNode(atomic_code_list)
575c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch
576c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch
577dc0f95d653279beabeb9817299e2902918ba123eKristian Monsendef ParseToAST(pump_src_text):
578dc0f95d653279beabeb9817299e2902918ba123eKristian Monsen  """Convert the given Pump source text into an AST."""
579dc0f95d653279beabeb9817299e2902918ba123eKristian Monsen  tokens = list(Tokenize(pump_src_text))
580c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  code_node = ParseCodeNode(tokens)
581c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  return code_node
582c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch
583c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch
584c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdochclass Env:
585c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  def __init__(self):
586c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    self.variables = []
587c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    self.ranges = []
588c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch
589c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  def Clone(self):
590c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    clone = Env()
591c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    clone.variables = self.variables[:]
592c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    clone.ranges = self.ranges[:]
593c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    return clone
594c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch
595c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  def PushVariable(self, var, value):
596c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    # If value looks like an int, store it as an int.
597c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    try:
598c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch      int_value = int(value)
599c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch      if ('%s' % int_value) == value:
600c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch        value = int_value
601c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    except Exception:
602c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch      pass
603c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    self.variables[:0] = [(var, value)]
604c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch
605c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  def PopVariable(self):
606c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    self.variables[:1] = []
607c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch
608c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  def PushRange(self, var, lower, upper):
609c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    self.ranges[:0] = [(var, lower, upper)]
610c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch
611c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  def PopRange(self):
612c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    self.ranges[:1] = []
613c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch
614c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  def GetValue(self, identifier):
615c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    for (var, value) in self.variables:
616c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch      if identifier == var:
617c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch        return value
618c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch
619c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    print 'ERROR: meta variable %s is undefined.' % (identifier,)
620c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    sys.exit(1)
621c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch
622c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  def EvalExp(self, exp):
623c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    try:
624c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch      result = eval(exp.python_exp)
625c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    except Exception, e:
626c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch      print 'ERROR: caught exception %s: %s' % (e.__class__.__name__, e)
627c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch      print ('ERROR: failed to evaluate meta expression %s at %s' %
628c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch             (exp.python_exp, exp.token.start))
629c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch      sys.exit(1)
630c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    return result
631c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch
632c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  def GetRange(self, identifier):
633c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    for (var, lower, upper) in self.ranges:
634c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch      if identifier == var:
635c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch        return (lower, upper)
636c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch
637c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    print 'ERROR: range %s is undefined.' % (identifier,)
638c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    sys.exit(1)
639c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch
640c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch
641c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdochclass Output:
642c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  def __init__(self):
643c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    self.string = ''
644c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch
645c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  def GetLastLine(self):
646c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    index = self.string.rfind('\n')
647c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    if index < 0:
648c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch      return ''
649c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch
650c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    return self.string[index + 1:]
651c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch
652c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  def Append(self, s):
653c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    self.string += s
654c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch
655c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch
656c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdochdef RunAtomicCode(env, node, output):
657c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  if isinstance(node, VarNode):
658c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    identifier = node.identifier.value.strip()
659c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    result = Output()
660c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    RunAtomicCode(env.Clone(), node.atomic_code, result)
661c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    value = result.string
662c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    env.PushVariable(identifier, value)
663c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  elif isinstance(node, RangeNode):
664c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    identifier = node.identifier.value.strip()
665c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    lower = int(env.EvalExp(node.exp1))
666c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    upper = int(env.EvalExp(node.exp2))
667c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    env.PushRange(identifier, lower, upper)
668c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  elif isinstance(node, ForNode):
669c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    identifier = node.identifier.value.strip()
670c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    if node.sep is None:
671c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch      sep = ''
672c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    else:
673c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch      sep = node.sep.value
674c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    (lower, upper) = env.GetRange(identifier)
675c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    for i in range(lower, upper + 1):
676c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch      new_env = env.Clone()
677c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch      new_env.PushVariable(identifier, i)
678c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch      RunCode(new_env, node.code, output)
679c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch      if i != upper:
680c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch        output.Append(sep)
681c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  elif isinstance(node, RawCodeNode):
682c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    output.Append(node.raw_code.value)
683c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  elif isinstance(node, IfNode):
684c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    cond = env.EvalExp(node.exp)
685c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    if cond:
686c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch      RunCode(env.Clone(), node.then_branch, output)
687c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    elif node.else_branch is not None:
688c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch      RunCode(env.Clone(), node.else_branch, output)
689c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  elif isinstance(node, ExpNode):
690c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    value = env.EvalExp(node)
691c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    output.Append('%s' % (value,))
692c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  elif isinstance(node, LiteralDollarNode):
693c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    output.Append('$')
694c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  elif isinstance(node, CodeNode):
695c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    RunCode(env.Clone(), node, output)
696c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  else:
697c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    print 'BAD'
698c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    print node
699c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    sys.exit(1)
700c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch
701c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch
702c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdochdef RunCode(env, code_node, output):
703c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  for atomic_code in code_node.atomic_code:
704c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    RunAtomicCode(env, atomic_code, output)
705c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch
706c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch
707c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdochdef IsComment(cur_line):
708c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  return '//' in cur_line
709c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch
710c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch
711c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdochdef IsInPreprocessorDirevative(prev_lines, cur_line):
712c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  if cur_line.lstrip().startswith('#'):
713c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    return True
714c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  return prev_lines != [] and prev_lines[-1].endswith('\\')
715c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch
716c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch
717c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdochdef WrapComment(line, output):
718c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  loc = line.find('//')
719c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  before_comment = line[:loc].rstrip()
720c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  if before_comment == '':
721c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    indent = loc
722c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  else:
723c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    output.append(before_comment)
724c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    indent = len(before_comment) - len(before_comment.lstrip())
725c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  prefix = indent*' ' + '// '
726c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  max_len = 80 - len(prefix)
727c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  comment = line[loc + 2:].strip()
728c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  segs = [seg for seg in re.split(r'(\w+\W*)', comment) if seg != '']
729c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  cur_line = ''
730c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  for seg in segs:
731c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    if len((cur_line + seg).rstrip()) < max_len:
732c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch      cur_line += seg
733c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    else:
734c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch      if cur_line.strip() != '':
735c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch        output.append(prefix + cur_line.rstrip())
736c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch      cur_line = seg.lstrip()
737c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  if cur_line.strip() != '':
738c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    output.append(prefix + cur_line.strip())
739c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch
740c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch
741c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdochdef WrapCode(line, line_concat, output):
742c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  indent = len(line) - len(line.lstrip())
743c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  prefix = indent*' '  # Prefix of the current line
744c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  max_len = 80 - indent - len(line_concat)  # Maximum length of the current line
745c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  new_prefix = prefix + 4*' '  # Prefix of a continuation line
746c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  new_max_len = max_len - 4  # Maximum length of a continuation line
747c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  # Prefers to wrap a line after a ',' or ';'.
748c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  segs = [seg for seg in re.split(r'([^,;]+[,;]?)', line.strip()) if seg != '']
749c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  cur_line = ''  # The current line without leading spaces.
750c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  for seg in segs:
751c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    # If the line is still too long, wrap at a space.
752c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    while cur_line == '' and len(seg.strip()) > max_len:
753c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch      seg = seg.lstrip()
754c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch      split_at = seg.rfind(' ', 0, max_len)
755c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch      output.append(prefix + seg[:split_at].strip() + line_concat)
756c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch      seg = seg[split_at + 1:]
757c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch      prefix = new_prefix
758c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch      max_len = new_max_len
759c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch
760c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    if len((cur_line + seg).rstrip()) < max_len:
761c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch      cur_line = (cur_line + seg).lstrip()
762c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    else:
763c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch      output.append(prefix + cur_line.rstrip() + line_concat)
764c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch      prefix = new_prefix
765c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch      max_len = new_max_len
766c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch      cur_line = seg.lstrip()
767c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  if cur_line.strip() != '':
768c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    output.append(prefix + cur_line.strip())
769c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch
770c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch
771c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdochdef WrapPreprocessorDirevative(line, output):
772c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  WrapCode(line, ' \\', output)
773c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch
774c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch
775c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdochdef WrapPlainCode(line, output):
776c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  WrapCode(line, '', output)
777c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch
778c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch
779c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdochdef IsHeaderGuardOrInclude(line):
780c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  return (re.match(r'^#(ifndef|define|endif\s*//)\s*[\w_]+\s*$', line) or
781c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch          re.match(r'^#include\s', line))
782c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch
783c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch
784c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdochdef WrapLongLine(line, output):
785c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  line = line.rstrip()
786c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  if len(line) <= 80:
787c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    output.append(line)
788c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  elif IsComment(line):
789c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    if IsHeaderGuardOrInclude(line):
790c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch      # The style guide made an exception to allow long header guard lines
791c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch      # and includes.
792c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch      output.append(line)
793c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    else:
794c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch      WrapComment(line, output)
795c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  elif IsInPreprocessorDirevative(output, line):
796c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    if IsHeaderGuardOrInclude(line):
797c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch      # The style guide made an exception to allow long header guard lines
798c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch      # and includes.
799c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch      output.append(line)
800c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    else:
801c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch      WrapPreprocessorDirevative(line, output)
802c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  else:
803c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    WrapPlainCode(line, output)
804c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch
805c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch
806c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdochdef BeautifyCode(string):
807c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  lines = string.splitlines()
808c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  output = []
809c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  for line in lines:
810c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    WrapLongLine(line, output)
811c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  output2 = [line.rstrip() for line in output]
812c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  return '\n'.join(output2) + '\n'
813c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch
814c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch
815dc0f95d653279beabeb9817299e2902918ba123eKristian Monsendef ConvertFromPumpSource(src_text):
816dc0f95d653279beabeb9817299e2902918ba123eKristian Monsen  """Return the text generated from the given Pump source text."""
817dc0f95d653279beabeb9817299e2902918ba123eKristian Monsen  ast = ParseToAST(StripMetaComments(src_text))
818dc0f95d653279beabeb9817299e2902918ba123eKristian Monsen  output = Output()
819dc0f95d653279beabeb9817299e2902918ba123eKristian Monsen  RunCode(Env(), ast, output)
820dc0f95d653279beabeb9817299e2902918ba123eKristian Monsen  return BeautifyCode(output.string)
821dc0f95d653279beabeb9817299e2902918ba123eKristian Monsen
822dc0f95d653279beabeb9817299e2902918ba123eKristian Monsen
823c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdochdef main(argv):
824c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  if len(argv) == 1:
825c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    print __doc__
826c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    sys.exit(1)
827c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch
828c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  file_path = argv[-1]
829dc0f95d653279beabeb9817299e2902918ba123eKristian Monsen  output_str = ConvertFromPumpSource(file(file_path, 'r').read())
830c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  if file_path.endswith('.pump'):
831c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    output_file_path = file_path[:-5]
832c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  else:
833c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    output_file_path = '-'
834c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  if output_file_path == '-':
835c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    print output_str,
836c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  else:
837c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    output_file = file(output_file_path, 'w')
838c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    output_file.write('// This file was GENERATED by command:\n')
839c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    output_file.write('//     %s %s\n' %
840c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch                      (os.path.basename(__file__), os.path.basename(file_path)))
841c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    output_file.write('// DO NOT EDIT BY HAND!!!\n\n')
842c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    output_file.write(output_str)
843c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch    output_file.close()
844c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch
845c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch
846c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdochif __name__ == '__main__':
847c407dc5cd9bdc5668497f21b26b09d988ab439deBen Murdoch  main(sys.argv)
848