141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot#!/usr/bin/env python
241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot#
341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot# Copyright 2008, Google Inc.
441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot# All rights reserved.
541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot#
641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot# Redistribution and use in source and binary forms, with or without
741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot# modification, are permitted provided that the following conditions are
841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot# met:
941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot#
1041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot#     * Redistributions of source code must retain the above copyright
1141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot# notice, this list of conditions and the following disclaimer.
1241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot#     * Redistributions in binary form must reproduce the above
1341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot# copyright notice, this list of conditions and the following disclaimer
1441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot# in the documentation and/or other materials provided with the
1541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot# distribution.
1641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot#     * Neither the name of Google Inc. nor the names of its
1741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot# contributors may be used to endorse or promote products derived from
1841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot# this software without specific prior written permission.
1941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot#
2041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
2141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
2241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
2341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
2441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
2541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
2641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
2741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
2841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
2941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
3041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
3141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot
3241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot"""pump v0.2.0 - Pretty Useful for Meta Programming.
3341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot
3441d0579e8de9ef4ff178fc4991043c61a19943f7Brett ChabotA tool for preprocessor meta programming.  Useful for generating
3541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabotrepetitive boilerplate code.  Especially useful for writing C++
3641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabotclasses, functions, macros, and templates that need to work with
3741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabotvarious number of arguments.
3841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot
3941d0579e8de9ef4ff178fc4991043c61a19943f7Brett ChabotUSAGE:
4041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot       pump.py SOURCE_FILE
4141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot
4241d0579e8de9ef4ff178fc4991043c61a19943f7Brett ChabotEXAMPLES:
4341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot       pump.py foo.cc.pump
4441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot         Converts foo.cc.pump to foo.cc.
4541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot
4641d0579e8de9ef4ff178fc4991043c61a19943f7Brett ChabotGRAMMAR:
4741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot       CODE ::= ATOMIC_CODE*
4841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot       ATOMIC_CODE ::= $var ID = EXPRESSION
4941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot           | $var ID = [[ CODE ]]
5041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot           | $range ID EXPRESSION..EXPRESSION
5141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot           | $for ID SEPARATOR [[ CODE ]]
5241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot           | $($)
5341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot           | $ID
5441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot           | $(EXPRESSION)
5541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot           | $if EXPRESSION [[ CODE ]] ELSE_BRANCH
5641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot           | [[ CODE ]]
5741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot           | RAW_CODE
5841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot       SEPARATOR ::= RAW_CODE | EMPTY
5941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot       ELSE_BRANCH ::= $else [[ CODE ]]
6041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot           | $elif EXPRESSION [[ CODE ]] ELSE_BRANCH
6141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot           | EMPTY
6241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot       EXPRESSION has Python syntax.
6341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot"""
6441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot
6541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot__author__ = 'wan@google.com (Zhanyong Wan)'
6641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot
6741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabotimport os
6841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabotimport re
6941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabotimport sys
7041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot
7141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot
7241d0579e8de9ef4ff178fc4991043c61a19943f7Brett ChabotTOKEN_TABLE = [
7341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    (re.compile(r'\$var\s+'), '$var'),
7441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    (re.compile(r'\$elif\s+'), '$elif'),
7541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    (re.compile(r'\$else\s+'), '$else'),
7641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    (re.compile(r'\$for\s+'), '$for'),
7741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    (re.compile(r'\$if\s+'), '$if'),
7841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    (re.compile(r'\$range\s+'), '$range'),
7941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    (re.compile(r'\$[_A-Za-z]\w*'), '$id'),
8041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    (re.compile(r'\$\(\$\)'), '$($)'),
8141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    (re.compile(r'\$'), '$'),
8241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    (re.compile(r'\[\[\n?'), '[['),
8341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    (re.compile(r'\]\]\n?'), ']]'),
8441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    ]
8541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot
8641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot
8741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabotclass Cursor:
8841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  """Represents a position (line and column) in a text file."""
8941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot
9041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  def __init__(self, line=-1, column=-1):
9141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    self.line = line
9241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    self.column = column
9341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot
9441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  def __eq__(self, rhs):
9541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    return self.line == rhs.line and self.column == rhs.column
9641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot
9741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  def __ne__(self, rhs):
9841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    return not self == rhs
9941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot
10041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  def __lt__(self, rhs):
10141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    return self.line < rhs.line or (
10241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot        self.line == rhs.line and self.column < rhs.column)
10341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot
10441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  def __le__(self, rhs):
10541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    return self < rhs or self == rhs
10641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot
10741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  def __gt__(self, rhs):
10841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    return rhs < self
10941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot
11041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  def __ge__(self, rhs):
11141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    return rhs <= self
11241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot
11341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  def __str__(self):
11441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    if self == Eof():
11541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot      return 'EOF'
11641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    else:
11741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot      return '%s(%s)' % (self.line + 1, self.column)
11841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot
11941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  def __add__(self, offset):
12041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    return Cursor(self.line, self.column + offset)
12141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot
12241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  def __sub__(self, offset):
12341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    return Cursor(self.line, self.column - offset)
12441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot
12541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  def Clone(self):
12641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    """Returns a copy of self."""
12741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot
12841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    return Cursor(self.line, self.column)
12941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot
13041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot
13141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot# Special cursor to indicate the end-of-file.
13241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabotdef Eof():
13341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  """Returns the special cursor to denote the end-of-file."""
13441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  return Cursor(-1, -1)
13541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot
13641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot
13741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabotclass Token:
13841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  """Represents a token in a Pump source file."""
13941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot
14041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  def __init__(self, start=None, end=None, value=None, token_type=None):
14141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    if start is None:
14241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot      self.start = Eof()
14341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    else:
14441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot      self.start = start
14541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    if end is None:
14641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot      self.end = Eof()
14741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    else:
14841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot      self.end = end
14941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    self.value = value
15041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    self.token_type = token_type
15141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot
15241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  def __str__(self):
15341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    return 'Token @%s: \'%s\' type=%s' % (
15441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot        self.start, self.value, self.token_type)
15541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot
15641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  def Clone(self):
15741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    """Returns a copy of self."""
15841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot
15941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    return Token(self.start.Clone(), self.end.Clone(), self.value,
16041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot                 self.token_type)
16141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot
16241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot
16341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabotdef StartsWith(lines, pos, string):
16441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  """Returns True iff the given position in lines starts with 'string'."""
16541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot
16641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  return lines[pos.line][pos.column:].startswith(string)
16741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot
16841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot
16941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabotdef FindFirstInLine(line, token_table):
17041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  best_match_start = -1
17141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  for (regex, token_type) in token_table:
17241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    m = regex.search(line)
17341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    if m:
17441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot      # We found regex in lines
17541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot      if best_match_start < 0 or m.start() < best_match_start:
17641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot        best_match_start = m.start()
17741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot        best_match_length = m.end() - m.start()
17841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot        best_match_token_type = token_type
17941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot
18041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  if best_match_start < 0:
18141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    return None
18241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot
18341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  return (best_match_start, best_match_length, best_match_token_type)
18441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot
18541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot
18641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabotdef FindFirst(lines, token_table, cursor):
18741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  """Finds the first occurrence of any string in strings in lines."""
18841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot
18941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  start = cursor.Clone()
19041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  cur_line_number = cursor.line
19141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  for line in lines[start.line:]:
19241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    if cur_line_number == start.line:
19341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot      line = line[start.column:]
19441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    m = FindFirstInLine(line, token_table)
19541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    if m:
19641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot      # We found a regex in line.
19741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot      (start_column, length, token_type) = m
19841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot      if cur_line_number == start.line:
19941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot        start_column += start.column
20041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot      found_start = Cursor(cur_line_number, start_column)
20141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot      found_end = found_start + length
20241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot      return MakeToken(lines, found_start, found_end, token_type)
20341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    cur_line_number += 1
20441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  # We failed to find str in lines
20541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  return None
20641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot
20741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot
20841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabotdef SubString(lines, start, end):
20941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  """Returns a substring in lines."""
21041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot
21141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  if end == Eof():
21241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    end = Cursor(len(lines) - 1, len(lines[-1]))
21341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot
21441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  if start >= end:
21541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    return ''
21641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot
21741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  if start.line == end.line:
21841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    return lines[start.line][start.column:end.column]
21941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot
22041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  result_lines = ([lines[start.line][start.column:]] +
22141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot                  lines[start.line + 1:end.line] +
22241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot                  [lines[end.line][:end.column]])
22341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  return ''.join(result_lines)
22441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot
22541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot
22641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabotdef StripMetaComments(str):
22741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  """Strip meta comments from each line in the given string."""
22841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot
22941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  # First, completely remove lines containing nothing but a meta
23041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  # comment, including the trailing \n.
23141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  str = re.sub(r'^\s*\$\$.*\n', '', str)
23241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot
23341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  # Then, remove meta comments from contentful lines.
23441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  return re.sub(r'\s*\$\$.*', '', str)
23541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot
23641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot
23741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabotdef MakeToken(lines, start, end, token_type):
23841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  """Creates a new instance of Token."""
23941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot
24041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  return Token(start, end, SubString(lines, start, end), token_type)
24141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot
24241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot
24341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabotdef ParseToken(lines, pos, regex, token_type):
24441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  line = lines[pos.line][pos.column:]
24541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  m = regex.search(line)
24641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  if m and not m.start():
24741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    return MakeToken(lines, pos, pos + m.end(), token_type)
24841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  else:
24941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    print 'ERROR: %s expected at %s.' % (token_type, pos)
25041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    sys.exit(1)
25141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot
25241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot
25341d0579e8de9ef4ff178fc4991043c61a19943f7Brett ChabotID_REGEX = re.compile(r'[_A-Za-z]\w*')
25441d0579e8de9ef4ff178fc4991043c61a19943f7Brett ChabotEQ_REGEX = re.compile(r'=')
25541d0579e8de9ef4ff178fc4991043c61a19943f7Brett ChabotREST_OF_LINE_REGEX = re.compile(r'.*?(?=$|\$\$)')
25641d0579e8de9ef4ff178fc4991043c61a19943f7Brett ChabotOPTIONAL_WHITE_SPACES_REGEX = re.compile(r'\s*')
25741d0579e8de9ef4ff178fc4991043c61a19943f7Brett ChabotWHITE_SPACE_REGEX = re.compile(r'\s')
25841d0579e8de9ef4ff178fc4991043c61a19943f7Brett ChabotDOT_DOT_REGEX = re.compile(r'\.\.')
25941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot
26041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot
26141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabotdef Skip(lines, pos, regex):
26241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  line = lines[pos.line][pos.column:]
26341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  m = re.search(regex, line)
26441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  if m and not m.start():
26541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    return pos + m.end()
26641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  else:
26741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    return pos
26841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot
26941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot
27041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabotdef SkipUntil(lines, pos, regex, token_type):
27141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  line = lines[pos.line][pos.column:]
27241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  m = re.search(regex, line)
27341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  if m:
27441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    return pos + m.start()
27541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  else:
27641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    print ('ERROR: %s expected on line %s after column %s.' %
27741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot           (token_type, pos.line + 1, pos.column))
27841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    sys.exit(1)
27941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot
28041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot
28141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabotdef ParseExpTokenInParens(lines, pos):
28241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  def ParseInParens(pos):
28341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    pos = Skip(lines, pos, OPTIONAL_WHITE_SPACES_REGEX)
28441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    pos = Skip(lines, pos, r'\(')
28541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    pos = Parse(pos)
28641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    pos = Skip(lines, pos, r'\)')
28741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    return pos
28841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot
28941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  def Parse(pos):
29041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    pos = SkipUntil(lines, pos, r'\(|\)', ')')
29141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    if SubString(lines, pos, pos + 1) == '(':
29241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot      pos = Parse(pos + 1)
29341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot      pos = Skip(lines, pos, r'\)')
29441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot      return Parse(pos)
29541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    else:
29641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot      return pos
29741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot
29841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  start = pos.Clone()
29941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  pos = ParseInParens(pos)
30041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  return MakeToken(lines, start, pos, 'exp')
30141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot
30241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot
30341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabotdef RStripNewLineFromToken(token):
30441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  if token.value.endswith('\n'):
30541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    return Token(token.start, token.end, token.value[:-1], token.token_type)
30641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  else:
30741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    return token
30841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot
30941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot
31041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabotdef TokenizeLines(lines, pos):
31141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  while True:
31241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    found = FindFirst(lines, TOKEN_TABLE, pos)
31341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    if not found:
31441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot      yield MakeToken(lines, pos, Eof(), 'code')
31541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot      return
31641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot
31741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    if found.start == pos:
31841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot      prev_token = None
31941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot      prev_token_rstripped = None
32041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    else:
32141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot      prev_token = MakeToken(lines, pos, found.start, 'code')
32241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot      prev_token_rstripped = RStripNewLineFromToken(prev_token)
32341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot
32441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    if found.token_type == '$var':
32541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot      if prev_token_rstripped:
32641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot        yield prev_token_rstripped
32741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot      yield found
32841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot      id_token = ParseToken(lines, found.end, ID_REGEX, 'id')
32941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot      yield id_token
33041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot      pos = Skip(lines, id_token.end, OPTIONAL_WHITE_SPACES_REGEX)
33141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot
33241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot      eq_token = ParseToken(lines, pos, EQ_REGEX, '=')
33341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot      yield eq_token
33441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot      pos = Skip(lines, eq_token.end, r'\s*')
33541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot
33641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot      if SubString(lines, pos, pos + 2) != '[[':
33741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot        exp_token = ParseToken(lines, pos, REST_OF_LINE_REGEX, 'exp')
33841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot        yield exp_token
33941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot        pos = Cursor(exp_token.end.line + 1, 0)
34041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    elif found.token_type == '$for':
34141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot      if prev_token_rstripped:
34241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot        yield prev_token_rstripped
34341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot      yield found
34441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot      id_token = ParseToken(lines, found.end, ID_REGEX, 'id')
34541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot      yield id_token
34641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot      pos = Skip(lines, id_token.end, WHITE_SPACE_REGEX)
34741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    elif found.token_type == '$range':
34841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot      if prev_token_rstripped:
34941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot        yield prev_token_rstripped
35041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot      yield found
35141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot      id_token = ParseToken(lines, found.end, ID_REGEX, 'id')
35241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot      yield id_token
35341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot      pos = Skip(lines, id_token.end, OPTIONAL_WHITE_SPACES_REGEX)
35441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot
35541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot      dots_pos = SkipUntil(lines, pos, DOT_DOT_REGEX, '..')
35641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot      yield MakeToken(lines, pos, dots_pos, 'exp')
35741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot      yield MakeToken(lines, dots_pos, dots_pos + 2, '..')
35841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot      pos = dots_pos + 2
35941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot      new_pos = Cursor(pos.line + 1, 0)
36041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot      yield MakeToken(lines, pos, new_pos, 'exp')
36141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot      pos = new_pos
36241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    elif found.token_type == '$':
36341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot      if prev_token:
36441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot        yield prev_token
36541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot      yield found
36641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot      exp_token = ParseExpTokenInParens(lines, found.end)
36741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot      yield exp_token
36841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot      pos = exp_token.end
36941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    elif (found.token_type == ']]' or found.token_type == '$if' or
37041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot          found.token_type == '$elif' or found.token_type == '$else'):
37141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot      if prev_token_rstripped:
37241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot        yield prev_token_rstripped
37341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot      yield found
37441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot      pos = found.end
37541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    else:
37641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot      if prev_token:
37741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot        yield prev_token
37841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot      yield found
37941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot      pos = found.end
38041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot
38141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot
38241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabotdef Tokenize(s):
38341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  """A generator that yields the tokens in the given string."""
38441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  if s != '':
38541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    lines = s.splitlines(True)
38641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    for token in TokenizeLines(lines, Cursor(0, 0)):
38741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot      yield token
38841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot
38941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot
39041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabotclass CodeNode:
39141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  def __init__(self, atomic_code_list=None):
39241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    self.atomic_code = atomic_code_list
39341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot
39441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot
39541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabotclass VarNode:
39641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  def __init__(self, identifier=None, atomic_code=None):
39741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    self.identifier = identifier
39841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    self.atomic_code = atomic_code
39941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot
40041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot
40141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabotclass RangeNode:
40241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  def __init__(self, identifier=None, exp1=None, exp2=None):
40341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    self.identifier = identifier
40441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    self.exp1 = exp1
40541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    self.exp2 = exp2
40641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot
40741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot
40841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabotclass ForNode:
40941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  def __init__(self, identifier=None, sep=None, code=None):
41041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    self.identifier = identifier
41141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    self.sep = sep
41241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    self.code = code
41341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot
41441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot
41541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabotclass ElseNode:
41641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  def __init__(self, else_branch=None):
41741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    self.else_branch = else_branch
41841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot
41941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot
42041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabotclass IfNode:
42141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  def __init__(self, exp=None, then_branch=None, else_branch=None):
42241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    self.exp = exp
42341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    self.then_branch = then_branch
42441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    self.else_branch = else_branch
42541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot
42641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot
42741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabotclass RawCodeNode:
42841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  def __init__(self, token=None):
42941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    self.raw_code = token
43041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot
43141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot
43241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabotclass LiteralDollarNode:
43341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  def __init__(self, token):
43441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    self.token = token
43541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot
43641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot
43741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabotclass ExpNode:
43841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  def __init__(self, token, python_exp):
43941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    self.token = token
44041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    self.python_exp = python_exp
44141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot
44241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot
44341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabotdef PopFront(a_list):
44441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  head = a_list[0]
44541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  a_list[:1] = []
44641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  return head
44741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot
44841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot
44941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabotdef PushFront(a_list, elem):
45041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  a_list[:0] = [elem]
45141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot
45241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot
45341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabotdef PopToken(a_list, token_type=None):
45441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  token = PopFront(a_list)
45541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  if token_type is not None and token.token_type != token_type:
45641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    print 'ERROR: %s expected at %s' % (token_type, token.start)
45741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    print 'ERROR: %s found instead' % (token,)
45841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    sys.exit(1)
45941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot
46041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  return token
46141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot
46241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot
46341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabotdef PeekToken(a_list):
46441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  if not a_list:
46541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    return None
46641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot
46741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  return a_list[0]
46841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot
46941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot
47041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabotdef ParseExpNode(token):
47141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  python_exp = re.sub(r'([_A-Za-z]\w*)', r'self.GetValue("\1")', token.value)
47241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  return ExpNode(token, python_exp)
47341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot
47441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot
47541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabotdef ParseElseNode(tokens):
47641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  def Pop(token_type=None):
47741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    return PopToken(tokens, token_type)
47841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot
47941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  next = PeekToken(tokens)
48041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  if not next:
48141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    return None
48241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  if next.token_type == '$else':
48341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    Pop('$else')
48441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    Pop('[[')
48541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    code_node = ParseCodeNode(tokens)
48641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    Pop(']]')
48741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    return code_node
48841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  elif next.token_type == '$elif':
48941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    Pop('$elif')
49041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    exp = Pop('code')
49141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    Pop('[[')
49241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    code_node = ParseCodeNode(tokens)
49341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    Pop(']]')
49441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    inner_else_node = ParseElseNode(tokens)
49541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    return CodeNode([IfNode(ParseExpNode(exp), code_node, inner_else_node)])
49641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  elif not next.value.strip():
49741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    Pop('code')
49841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    return ParseElseNode(tokens)
49941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  else:
50041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    return None
50141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot
50241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot
50341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabotdef ParseAtomicCodeNode(tokens):
50441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  def Pop(token_type=None):
50541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    return PopToken(tokens, token_type)
50641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot
50741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  head = PopFront(tokens)
50841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  t = head.token_type
50941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  if t == 'code':
51041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    return RawCodeNode(head)
51141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  elif t == '$var':
51241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    id_token = Pop('id')
51341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    Pop('=')
51441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    next = PeekToken(tokens)
51541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    if next.token_type == 'exp':
51641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot      exp_token = Pop()
51741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot      return VarNode(id_token, ParseExpNode(exp_token))
51841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    Pop('[[')
51941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    code_node = ParseCodeNode(tokens)
52041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    Pop(']]')
52141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    return VarNode(id_token, code_node)
52241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  elif t == '$for':
52341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    id_token = Pop('id')
52441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    next_token = PeekToken(tokens)
52541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    if next_token.token_type == 'code':
52641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot      sep_token = next_token
52741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot      Pop('code')
52841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    else:
52941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot      sep_token = None
53041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    Pop('[[')
53141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    code_node = ParseCodeNode(tokens)
53241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    Pop(']]')
53341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    return ForNode(id_token, sep_token, code_node)
53441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  elif t == '$if':
53541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    exp_token = Pop('code')
53641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    Pop('[[')
53741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    code_node = ParseCodeNode(tokens)
53841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    Pop(']]')
53941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    else_node = ParseElseNode(tokens)
54041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    return IfNode(ParseExpNode(exp_token), code_node, else_node)
54141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  elif t == '$range':
54241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    id_token = Pop('id')
54341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    exp1_token = Pop('exp')
54441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    Pop('..')
54541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    exp2_token = Pop('exp')
54641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    return RangeNode(id_token, ParseExpNode(exp1_token),
54741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot                     ParseExpNode(exp2_token))
54841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  elif t == '$id':
54941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    return ParseExpNode(Token(head.start + 1, head.end, head.value[1:], 'id'))
55041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  elif t == '$($)':
55141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    return LiteralDollarNode(head)
55241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  elif t == '$':
55341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    exp_token = Pop('exp')
55441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    return ParseExpNode(exp_token)
55541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  elif t == '[[':
55641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    code_node = ParseCodeNode(tokens)
55741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    Pop(']]')
55841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    return code_node
55941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  else:
56041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    PushFront(tokens, head)
56141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    return None
56241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot
56341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot
56441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabotdef ParseCodeNode(tokens):
56541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  atomic_code_list = []
56641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  while True:
56741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    if not tokens:
56841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot      break
56941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    atomic_code_node = ParseAtomicCodeNode(tokens)
57041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    if atomic_code_node:
57141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot      atomic_code_list.append(atomic_code_node)
57241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    else:
57341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot      break
57441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  return CodeNode(atomic_code_list)
57541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot
57641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot
57741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabotdef ParseToAST(pump_src_text):
57841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  """Convert the given Pump source text into an AST."""
57941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  tokens = list(Tokenize(pump_src_text))
58041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  code_node = ParseCodeNode(tokens)
58141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  return code_node
58241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot
58341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot
58441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabotclass Env:
58541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  def __init__(self):
58641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    self.variables = []
58741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    self.ranges = []
58841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot
58941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  def Clone(self):
59041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    clone = Env()
59141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    clone.variables = self.variables[:]
59241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    clone.ranges = self.ranges[:]
59341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    return clone
59441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot
59541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  def PushVariable(self, var, value):
59641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    # If value looks like an int, store it as an int.
59741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    try:
59841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot      int_value = int(value)
59941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot      if ('%s' % int_value) == value:
60041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot        value = int_value
60141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    except Exception:
60241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot      pass
60341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    self.variables[:0] = [(var, value)]
60441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot
60541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  def PopVariable(self):
60641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    self.variables[:1] = []
60741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot
60841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  def PushRange(self, var, lower, upper):
60941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    self.ranges[:0] = [(var, lower, upper)]
61041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot
61141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  def PopRange(self):
61241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    self.ranges[:1] = []
61341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot
61441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  def GetValue(self, identifier):
61541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    for (var, value) in self.variables:
61641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot      if identifier == var:
61741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot        return value
61841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot
61941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    print 'ERROR: meta variable %s is undefined.' % (identifier,)
62041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    sys.exit(1)
62141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot
62241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  def EvalExp(self, exp):
62341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    try:
62441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot      result = eval(exp.python_exp)
62541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    except Exception, e:
62641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot      print 'ERROR: caught exception %s: %s' % (e.__class__.__name__, e)
62741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot      print ('ERROR: failed to evaluate meta expression %s at %s' %
62841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot             (exp.python_exp, exp.token.start))
62941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot      sys.exit(1)
63041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    return result
63141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot
63241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  def GetRange(self, identifier):
63341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    for (var, lower, upper) in self.ranges:
63441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot      if identifier == var:
63541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot        return (lower, upper)
63641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot
63741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    print 'ERROR: range %s is undefined.' % (identifier,)
63841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    sys.exit(1)
63941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot
64041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot
64141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabotclass Output:
64241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  def __init__(self):
64341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    self.string = ''
64441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot
64541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  def GetLastLine(self):
64641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    index = self.string.rfind('\n')
64741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    if index < 0:
64841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot      return ''
64941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot
65041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    return self.string[index + 1:]
65141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot
65241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  def Append(self, s):
65341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    self.string += s
65441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot
65541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot
65641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabotdef RunAtomicCode(env, node, output):
65741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  if isinstance(node, VarNode):
65841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    identifier = node.identifier.value.strip()
65941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    result = Output()
66041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    RunAtomicCode(env.Clone(), node.atomic_code, result)
66141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    value = result.string
66241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    env.PushVariable(identifier, value)
66341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  elif isinstance(node, RangeNode):
66441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    identifier = node.identifier.value.strip()
66541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    lower = int(env.EvalExp(node.exp1))
66641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    upper = int(env.EvalExp(node.exp2))
66741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    env.PushRange(identifier, lower, upper)
66841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  elif isinstance(node, ForNode):
66941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    identifier = node.identifier.value.strip()
67041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    if node.sep is None:
67141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot      sep = ''
67241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    else:
67341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot      sep = node.sep.value
67441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    (lower, upper) = env.GetRange(identifier)
67541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    for i in range(lower, upper + 1):
67641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot      new_env = env.Clone()
67741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot      new_env.PushVariable(identifier, i)
67841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot      RunCode(new_env, node.code, output)
67941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot      if i != upper:
68041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot        output.Append(sep)
68141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  elif isinstance(node, RawCodeNode):
68241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    output.Append(node.raw_code.value)
68341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  elif isinstance(node, IfNode):
68441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    cond = env.EvalExp(node.exp)
68541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    if cond:
68641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot      RunCode(env.Clone(), node.then_branch, output)
68741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    elif node.else_branch is not None:
68841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot      RunCode(env.Clone(), node.else_branch, output)
68941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  elif isinstance(node, ExpNode):
69041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    value = env.EvalExp(node)
69141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    output.Append('%s' % (value,))
69241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  elif isinstance(node, LiteralDollarNode):
69341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    output.Append('$')
69441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  elif isinstance(node, CodeNode):
69541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    RunCode(env.Clone(), node, output)
69641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  else:
69741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    print 'BAD'
69841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    print node
69941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    sys.exit(1)
70041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot
70141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot
70241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabotdef RunCode(env, code_node, output):
70341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  for atomic_code in code_node.atomic_code:
70441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    RunAtomicCode(env, atomic_code, output)
70541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot
70641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot
70741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabotdef IsComment(cur_line):
70841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  return '//' in cur_line
70941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot
71041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot
71141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabotdef IsInPreprocessorDirevative(prev_lines, cur_line):
71241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  if cur_line.lstrip().startswith('#'):
71341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    return True
71441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  return prev_lines != [] and prev_lines[-1].endswith('\\')
71541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot
71641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot
71741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabotdef WrapComment(line, output):
71841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  loc = line.find('//')
71941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  before_comment = line[:loc].rstrip()
72041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  if before_comment == '':
72141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    indent = loc
72241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  else:
72341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    output.append(before_comment)
72441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    indent = len(before_comment) - len(before_comment.lstrip())
72541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  prefix = indent*' ' + '// '
72641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  max_len = 80 - len(prefix)
72741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  comment = line[loc + 2:].strip()
72841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  segs = [seg for seg in re.split(r'(\w+\W*)', comment) if seg != '']
72941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  cur_line = ''
73041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  for seg in segs:
73141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    if len((cur_line + seg).rstrip()) < max_len:
73241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot      cur_line += seg
73341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    else:
73441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot      if cur_line.strip() != '':
73541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot        output.append(prefix + cur_line.rstrip())
73641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot      cur_line = seg.lstrip()
73741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  if cur_line.strip() != '':
73841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    output.append(prefix + cur_line.strip())
73941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot
74041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot
74141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabotdef WrapCode(line, line_concat, output):
74241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  indent = len(line) - len(line.lstrip())
74341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  prefix = indent*' '  # Prefix of the current line
74441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  max_len = 80 - indent - len(line_concat)  # Maximum length of the current line
74541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  new_prefix = prefix + 4*' '  # Prefix of a continuation line
74641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  new_max_len = max_len - 4  # Maximum length of a continuation line
74741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  # Prefers to wrap a line after a ',' or ';'.
74841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  segs = [seg for seg in re.split(r'([^,;]+[,;]?)', line.strip()) if seg != '']
74941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  cur_line = ''  # The current line without leading spaces.
75041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  for seg in segs:
75141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    # If the line is still too long, wrap at a space.
75241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    while cur_line == '' and len(seg.strip()) > max_len:
75341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot      seg = seg.lstrip()
75441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot      split_at = seg.rfind(' ', 0, max_len)
75541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot      output.append(prefix + seg[:split_at].strip() + line_concat)
75641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot      seg = seg[split_at + 1:]
75741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot      prefix = new_prefix
75841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot      max_len = new_max_len
75941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot
76041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    if len((cur_line + seg).rstrip()) < max_len:
76141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot      cur_line = (cur_line + seg).lstrip()
76241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    else:
76341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot      output.append(prefix + cur_line.rstrip() + line_concat)
76441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot      prefix = new_prefix
76541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot      max_len = new_max_len
76641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot      cur_line = seg.lstrip()
76741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  if cur_line.strip() != '':
76841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    output.append(prefix + cur_line.strip())
76941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot
77041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot
77141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabotdef WrapPreprocessorDirevative(line, output):
77241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  WrapCode(line, ' \\', output)
77341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot
77441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot
77541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabotdef WrapPlainCode(line, output):
77641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  WrapCode(line, '', output)
77741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot
77841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot
77941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabotdef IsHeaderGuardOrInclude(line):
78041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  return (re.match(r'^#(ifndef|define|endif\s*//)\s*[\w_]+\s*$', line) or
78141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot          re.match(r'^#include\s', line))
78241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot
78341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot
78441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabotdef WrapLongLine(line, output):
78541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  line = line.rstrip()
78641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  if len(line) <= 80:
78741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    output.append(line)
78841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  elif IsComment(line):
78941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    if IsHeaderGuardOrInclude(line):
79041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot      # The style guide made an exception to allow long header guard lines
79141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot      # and includes.
79241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot      output.append(line)
79341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    else:
79441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot      WrapComment(line, output)
79541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  elif IsInPreprocessorDirevative(output, line):
79641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    if IsHeaderGuardOrInclude(line):
79741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot      # The style guide made an exception to allow long header guard lines
79841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot      # and includes.
79941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot      output.append(line)
80041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    else:
80141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot      WrapPreprocessorDirevative(line, output)
80241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  else:
80341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    WrapPlainCode(line, output)
80441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot
80541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot
80641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabotdef BeautifyCode(string):
80741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  lines = string.splitlines()
80841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  output = []
80941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  for line in lines:
81041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    WrapLongLine(line, output)
81141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  output2 = [line.rstrip() for line in output]
81241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  return '\n'.join(output2) + '\n'
81341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot
81441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot
81541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabotdef ConvertFromPumpSource(src_text):
81641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  """Return the text generated from the given Pump source text."""
81741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  ast = ParseToAST(StripMetaComments(src_text))
81841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  output = Output()
81941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  RunCode(Env(), ast, output)
82041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  return BeautifyCode(output.string)
82141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot
82241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot
82341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabotdef main(argv):
82441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  if len(argv) == 1:
82541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    print __doc__
82641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    sys.exit(1)
82741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot
82841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  file_path = argv[-1]
82941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  output_str = ConvertFromPumpSource(file(file_path, 'r').read())
83041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  if file_path.endswith('.pump'):
83141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    output_file_path = file_path[:-5]
83241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  else:
83341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    output_file_path = '-'
83441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  if output_file_path == '-':
83541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    print output_str,
83641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  else:
83741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    output_file = file(output_file_path, 'w')
83841d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    output_file.write('// This file was GENERATED by command:\n')
83941d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    output_file.write('//     %s %s\n' %
84041d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot                      (os.path.basename(__file__), os.path.basename(file_path)))
84141d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    output_file.write('// DO NOT EDIT BY HAND!!!\n\n')
84241d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    output_file.write(output_str)
84341d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot    output_file.close()
84441d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot
84541d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot
84641d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabotif __name__ == '__main__':
84741d0579e8de9ef4ff178fc4991043c61a19943f7Brett Chabot  main(sys.argv)
848