146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan#!/usr/bin/env python
246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan#
346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan# Copyright 2008, Google Inc.
446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan# All rights reserved.
546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan#
646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan# Redistribution and use in source and binary forms, with or without
746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan# modification, are permitted provided that the following conditions are
846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan# met:
946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan#
1046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan#     * Redistributions of source code must retain the above copyright
1146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan# notice, this list of conditions and the following disclaimer.
1246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan#     * Redistributions in binary form must reproduce the above
1346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan# copyright notice, this list of conditions and the following disclaimer
1446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan# in the documentation and/or other materials provided with the
1546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan# distribution.
1646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan#     * Neither the name of Google Inc. nor the names of its
1746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan# contributors may be used to endorse or promote products derived from
1846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan# this software without specific prior written permission.
1946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan#
2046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
2146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
2246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
2346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
2446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
2546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
2646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
2746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
2846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
2946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
3046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
3146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan
3246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan"""pump v0.2.0 - Pretty Useful for Meta Programming.
3346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan
3446108a219a4b812dd8f36fee479a0340ea5963f5Ben ChanA tool for preprocessor meta programming.  Useful for generating
3546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chanrepetitive boilerplate code.  Especially useful for writing C++
3646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chanclasses, functions, macros, and templates that need to work with
3746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chanvarious number of arguments.
3846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan
3946108a219a4b812dd8f36fee479a0340ea5963f5Ben ChanUSAGE:
4046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan       pump.py SOURCE_FILE
4146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan
4246108a219a4b812dd8f36fee479a0340ea5963f5Ben ChanEXAMPLES:
4346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan       pump.py foo.cc.pump
4446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan         Converts foo.cc.pump to foo.cc.
4546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan
4646108a219a4b812dd8f36fee479a0340ea5963f5Ben ChanGRAMMAR:
4746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan       CODE ::= ATOMIC_CODE*
4846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan       ATOMIC_CODE ::= $var ID = EXPRESSION
4946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan           | $var ID = [[ CODE ]]
5046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan           | $range ID EXPRESSION..EXPRESSION
5146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan           | $for ID SEPARATOR [[ CODE ]]
5246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan           | $($)
5346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan           | $ID
5446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan           | $(EXPRESSION)
5546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan           | $if EXPRESSION [[ CODE ]] ELSE_BRANCH
5646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan           | [[ CODE ]]
5746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan           | RAW_CODE
5846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan       SEPARATOR ::= RAW_CODE | EMPTY
5946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan       ELSE_BRANCH ::= $else [[ CODE ]]
6046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan           | $elif EXPRESSION [[ CODE ]] ELSE_BRANCH
6146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan           | EMPTY
6246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan       EXPRESSION has Python syntax.
6346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan"""
6446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan
6546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan__author__ = 'wan@google.com (Zhanyong Wan)'
6646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan
6746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chanimport os
6846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chanimport re
6946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chanimport sys
7046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan
7146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan
7246108a219a4b812dd8f36fee479a0340ea5963f5Ben ChanTOKEN_TABLE = [
7346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    (re.compile(r'\$var\s+'), '$var'),
7446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    (re.compile(r'\$elif\s+'), '$elif'),
7546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    (re.compile(r'\$else\s+'), '$else'),
7646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    (re.compile(r'\$for\s+'), '$for'),
7746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    (re.compile(r'\$if\s+'), '$if'),
7846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    (re.compile(r'\$range\s+'), '$range'),
7946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    (re.compile(r'\$[_A-Za-z]\w*'), '$id'),
8046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    (re.compile(r'\$\(\$\)'), '$($)'),
8146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    (re.compile(r'\$'), '$'),
8246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    (re.compile(r'\[\[\n?'), '[['),
8346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    (re.compile(r'\]\]\n?'), ']]'),
8446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    ]
8546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan
8646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan
8746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chanclass Cursor:
8846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  """Represents a position (line and column) in a text file."""
8946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan
9046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  def __init__(self, line=-1, column=-1):
9146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    self.line = line
9246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    self.column = column
9346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan
9446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  def __eq__(self, rhs):
9546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    return self.line == rhs.line and self.column == rhs.column
9646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan
9746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  def __ne__(self, rhs):
9846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    return not self == rhs
9946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan
10046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  def __lt__(self, rhs):
10146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    return self.line < rhs.line or (
10246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan        self.line == rhs.line and self.column < rhs.column)
10346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan
10446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  def __le__(self, rhs):
10546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    return self < rhs or self == rhs
10646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan
10746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  def __gt__(self, rhs):
10846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    return rhs < self
10946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan
11046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  def __ge__(self, rhs):
11146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    return rhs <= self
11246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan
11346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  def __str__(self):
11446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    if self == Eof():
11546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan      return 'EOF'
11646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    else:
11746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan      return '%s(%s)' % (self.line + 1, self.column)
11846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan
11946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  def __add__(self, offset):
12046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    return Cursor(self.line, self.column + offset)
12146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan
12246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  def __sub__(self, offset):
12346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    return Cursor(self.line, self.column - offset)
12446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan
12546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  def Clone(self):
12646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    """Returns a copy of self."""
12746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan
12846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    return Cursor(self.line, self.column)
12946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan
13046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan
13146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan# Special cursor to indicate the end-of-file.
13246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chandef Eof():
13346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  """Returns the special cursor to denote the end-of-file."""
13446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  return Cursor(-1, -1)
13546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan
13646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan
13746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chanclass Token:
13846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  """Represents a token in a Pump source file."""
13946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan
14046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  def __init__(self, start=None, end=None, value=None, token_type=None):
14146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    if start is None:
14246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan      self.start = Eof()
14346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    else:
14446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan      self.start = start
14546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    if end is None:
14646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan      self.end = Eof()
14746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    else:
14846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan      self.end = end
14946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    self.value = value
15046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    self.token_type = token_type
15146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan
15246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  def __str__(self):
15346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    return 'Token @%s: \'%s\' type=%s' % (
15446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan        self.start, self.value, self.token_type)
15546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan
15646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  def Clone(self):
15746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    """Returns a copy of self."""
15846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan
15946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    return Token(self.start.Clone(), self.end.Clone(), self.value,
16046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan                 self.token_type)
16146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan
16246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan
16346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chandef StartsWith(lines, pos, string):
16446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  """Returns True iff the given position in lines starts with 'string'."""
16546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan
16646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  return lines[pos.line][pos.column:].startswith(string)
16746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan
16846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan
16946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chandef FindFirstInLine(line, token_table):
17046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  best_match_start = -1
17146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  for (regex, token_type) in token_table:
17246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    m = regex.search(line)
17346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    if m:
17446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan      # We found regex in lines
17546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan      if best_match_start < 0 or m.start() < best_match_start:
17646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan        best_match_start = m.start()
17746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan        best_match_length = m.end() - m.start()
17846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan        best_match_token_type = token_type
17946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan
18046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  if best_match_start < 0:
18146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    return None
18246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan
18346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  return (best_match_start, best_match_length, best_match_token_type)
18446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan
18546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan
18646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chandef FindFirst(lines, token_table, cursor):
18746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  """Finds the first occurrence of any string in strings in lines."""
18846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan
18946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  start = cursor.Clone()
19046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  cur_line_number = cursor.line
19146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  for line in lines[start.line:]:
19246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    if cur_line_number == start.line:
19346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan      line = line[start.column:]
19446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    m = FindFirstInLine(line, token_table)
19546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    if m:
19646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan      # We found a regex in line.
19746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan      (start_column, length, token_type) = m
19846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan      if cur_line_number == start.line:
19946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan        start_column += start.column
20046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan      found_start = Cursor(cur_line_number, start_column)
20146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan      found_end = found_start + length
20246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan      return MakeToken(lines, found_start, found_end, token_type)
20346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    cur_line_number += 1
20446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  # We failed to find str in lines
20546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  return None
20646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan
20746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan
20846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chandef SubString(lines, start, end):
20946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  """Returns a substring in lines."""
21046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan
21146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  if end == Eof():
21246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    end = Cursor(len(lines) - 1, len(lines[-1]))
21346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan
21446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  if start >= end:
21546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    return ''
21646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan
21746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  if start.line == end.line:
21846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    return lines[start.line][start.column:end.column]
21946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan
22046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  result_lines = ([lines[start.line][start.column:]] +
22146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan                  lines[start.line + 1:end.line] +
22246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan                  [lines[end.line][:end.column]])
22346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  return ''.join(result_lines)
22446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan
22546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan
22646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chandef StripMetaComments(str):
22746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  """Strip meta comments from each line in the given string."""
22846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan
22946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  # First, completely remove lines containing nothing but a meta
23046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  # comment, including the trailing \n.
23146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  str = re.sub(r'^\s*\$\$.*\n', '', str)
23246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan
23346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  # Then, remove meta comments from contentful lines.
23446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  return re.sub(r'\s*\$\$.*', '', str)
23546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan
23646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan
23746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chandef MakeToken(lines, start, end, token_type):
23846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  """Creates a new instance of Token."""
23946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan
24046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  return Token(start, end, SubString(lines, start, end), token_type)
24146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan
24246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan
24346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chandef ParseToken(lines, pos, regex, token_type):
24446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  line = lines[pos.line][pos.column:]
24546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  m = regex.search(line)
24646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  if m and not m.start():
24746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    return MakeToken(lines, pos, pos + m.end(), token_type)
24846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  else:
24946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    print 'ERROR: %s expected at %s.' % (token_type, pos)
25046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    sys.exit(1)
25146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan
25246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan
25346108a219a4b812dd8f36fee479a0340ea5963f5Ben ChanID_REGEX = re.compile(r'[_A-Za-z]\w*')
25446108a219a4b812dd8f36fee479a0340ea5963f5Ben ChanEQ_REGEX = re.compile(r'=')
25546108a219a4b812dd8f36fee479a0340ea5963f5Ben ChanREST_OF_LINE_REGEX = re.compile(r'.*?(?=$|\$\$)')
25646108a219a4b812dd8f36fee479a0340ea5963f5Ben ChanOPTIONAL_WHITE_SPACES_REGEX = re.compile(r'\s*')
25746108a219a4b812dd8f36fee479a0340ea5963f5Ben ChanWHITE_SPACE_REGEX = re.compile(r'\s')
25846108a219a4b812dd8f36fee479a0340ea5963f5Ben ChanDOT_DOT_REGEX = re.compile(r'\.\.')
25946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan
26046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan
26146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chandef Skip(lines, pos, regex):
26246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  line = lines[pos.line][pos.column:]
26346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  m = re.search(regex, line)
26446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  if m and not m.start():
26546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    return pos + m.end()
26646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  else:
26746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    return pos
26846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan
26946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan
27046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chandef SkipUntil(lines, pos, regex, token_type):
27146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  line = lines[pos.line][pos.column:]
27246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  m = re.search(regex, line)
27346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  if m:
27446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    return pos + m.start()
27546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  else:
27646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    print ('ERROR: %s expected on line %s after column %s.' %
27746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan           (token_type, pos.line + 1, pos.column))
27846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    sys.exit(1)
27946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan
28046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan
28146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chandef ParseExpTokenInParens(lines, pos):
28246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  def ParseInParens(pos):
28346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    pos = Skip(lines, pos, OPTIONAL_WHITE_SPACES_REGEX)
28446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    pos = Skip(lines, pos, r'\(')
28546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    pos = Parse(pos)
28646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    pos = Skip(lines, pos, r'\)')
28746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    return pos
28846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan
28946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  def Parse(pos):
29046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    pos = SkipUntil(lines, pos, r'\(|\)', ')')
29146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    if SubString(lines, pos, pos + 1) == '(':
29246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan      pos = Parse(pos + 1)
29346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan      pos = Skip(lines, pos, r'\)')
29446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan      return Parse(pos)
29546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    else:
29646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan      return pos
29746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan
29846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  start = pos.Clone()
29946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  pos = ParseInParens(pos)
30046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  return MakeToken(lines, start, pos, 'exp')
30146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan
30246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan
30346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chandef RStripNewLineFromToken(token):
30446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  if token.value.endswith('\n'):
30546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    return Token(token.start, token.end, token.value[:-1], token.token_type)
30646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  else:
30746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    return token
30846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan
30946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan
31046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chandef TokenizeLines(lines, pos):
31146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  while True:
31246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    found = FindFirst(lines, TOKEN_TABLE, pos)
31346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    if not found:
31446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan      yield MakeToken(lines, pos, Eof(), 'code')
31546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan      return
31646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan
31746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    if found.start == pos:
31846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan      prev_token = None
31946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan      prev_token_rstripped = None
32046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    else:
32146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan      prev_token = MakeToken(lines, pos, found.start, 'code')
32246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan      prev_token_rstripped = RStripNewLineFromToken(prev_token)
32346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan
32446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    if found.token_type == '$var':
32546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan      if prev_token_rstripped:
32646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan        yield prev_token_rstripped
32746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan      yield found
32846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan      id_token = ParseToken(lines, found.end, ID_REGEX, 'id')
32946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan      yield id_token
33046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan      pos = Skip(lines, id_token.end, OPTIONAL_WHITE_SPACES_REGEX)
33146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan
33246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan      eq_token = ParseToken(lines, pos, EQ_REGEX, '=')
33346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan      yield eq_token
33446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan      pos = Skip(lines, eq_token.end, r'\s*')
33546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan
33646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan      if SubString(lines, pos, pos + 2) != '[[':
33746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan        exp_token = ParseToken(lines, pos, REST_OF_LINE_REGEX, 'exp')
33846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan        yield exp_token
33946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan        pos = Cursor(exp_token.end.line + 1, 0)
34046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    elif found.token_type == '$for':
34146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan      if prev_token_rstripped:
34246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan        yield prev_token_rstripped
34346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan      yield found
34446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan      id_token = ParseToken(lines, found.end, ID_REGEX, 'id')
34546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan      yield id_token
34646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan      pos = Skip(lines, id_token.end, WHITE_SPACE_REGEX)
34746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    elif found.token_type == '$range':
34846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan      if prev_token_rstripped:
34946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan        yield prev_token_rstripped
35046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan      yield found
35146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan      id_token = ParseToken(lines, found.end, ID_REGEX, 'id')
35246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan      yield id_token
35346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan      pos = Skip(lines, id_token.end, OPTIONAL_WHITE_SPACES_REGEX)
35446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan
35546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan      dots_pos = SkipUntil(lines, pos, DOT_DOT_REGEX, '..')
35646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan      yield MakeToken(lines, pos, dots_pos, 'exp')
35746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan      yield MakeToken(lines, dots_pos, dots_pos + 2, '..')
35846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan      pos = dots_pos + 2
35946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan      new_pos = Cursor(pos.line + 1, 0)
36046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan      yield MakeToken(lines, pos, new_pos, 'exp')
36146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan      pos = new_pos
36246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    elif found.token_type == '$':
36346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan      if prev_token:
36446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan        yield prev_token
36546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan      yield found
36646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan      exp_token = ParseExpTokenInParens(lines, found.end)
36746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan      yield exp_token
36846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan      pos = exp_token.end
36946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    elif (found.token_type == ']]' or found.token_type == '$if' or
37046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan          found.token_type == '$elif' or found.token_type == '$else'):
37146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan      if prev_token_rstripped:
37246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan        yield prev_token_rstripped
37346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan      yield found
37446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan      pos = found.end
37546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    else:
37646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan      if prev_token:
37746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan        yield prev_token
37846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan      yield found
37946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan      pos = found.end
38046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan
38146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan
38246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chandef Tokenize(s):
38346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  """A generator that yields the tokens in the given string."""
38446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  if s != '':
38546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    lines = s.splitlines(True)
38646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    for token in TokenizeLines(lines, Cursor(0, 0)):
38746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan      yield token
38846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan
38946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan
39046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chanclass CodeNode:
39146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  def __init__(self, atomic_code_list=None):
39246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    self.atomic_code = atomic_code_list
39346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan
39446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan
39546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chanclass VarNode:
39646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  def __init__(self, identifier=None, atomic_code=None):
39746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    self.identifier = identifier
39846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    self.atomic_code = atomic_code
39946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan
40046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan
40146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chanclass RangeNode:
40246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  def __init__(self, identifier=None, exp1=None, exp2=None):
40346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    self.identifier = identifier
40446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    self.exp1 = exp1
40546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    self.exp2 = exp2
40646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan
40746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan
40846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chanclass ForNode:
40946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  def __init__(self, identifier=None, sep=None, code=None):
41046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    self.identifier = identifier
41146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    self.sep = sep
41246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    self.code = code
41346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan
41446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan
41546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chanclass ElseNode:
41646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  def __init__(self, else_branch=None):
41746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    self.else_branch = else_branch
41846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan
41946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan
42046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chanclass IfNode:
42146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  def __init__(self, exp=None, then_branch=None, else_branch=None):
42246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    self.exp = exp
42346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    self.then_branch = then_branch
42446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    self.else_branch = else_branch
42546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan
42646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan
42746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chanclass RawCodeNode:
42846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  def __init__(self, token=None):
42946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    self.raw_code = token
43046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan
43146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan
43246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chanclass LiteralDollarNode:
43346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  def __init__(self, token):
43446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    self.token = token
43546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan
43646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan
43746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chanclass ExpNode:
43846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  def __init__(self, token, python_exp):
43946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    self.token = token
44046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    self.python_exp = python_exp
44146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan
44246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan
44346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chandef PopFront(a_list):
44446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  head = a_list[0]
44546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  a_list[:1] = []
44646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  return head
44746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan
44846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan
44946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chandef PushFront(a_list, elem):
45046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  a_list[:0] = [elem]
45146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan
45246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan
45346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chandef PopToken(a_list, token_type=None):
45446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  token = PopFront(a_list)
45546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  if token_type is not None and token.token_type != token_type:
45646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    print 'ERROR: %s expected at %s' % (token_type, token.start)
45746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    print 'ERROR: %s found instead' % (token,)
45846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    sys.exit(1)
45946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan
46046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  return token
46146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan
46246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan
46346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chandef PeekToken(a_list):
46446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  if not a_list:
46546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    return None
46646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan
46746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  return a_list[0]
46846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan
46946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan
47046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chandef ParseExpNode(token):
47146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  python_exp = re.sub(r'([_A-Za-z]\w*)', r'self.GetValue("\1")', token.value)
47246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  return ExpNode(token, python_exp)
47346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan
47446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan
47546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chandef ParseElseNode(tokens):
47646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  def Pop(token_type=None):
47746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    return PopToken(tokens, token_type)
47846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan
47946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  next = PeekToken(tokens)
48046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  if not next:
48146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    return None
48246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  if next.token_type == '$else':
48346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    Pop('$else')
48446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    Pop('[[')
48546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    code_node = ParseCodeNode(tokens)
48646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    Pop(']]')
48746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    return code_node
48846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  elif next.token_type == '$elif':
48946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    Pop('$elif')
49046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    exp = Pop('code')
49146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    Pop('[[')
49246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    code_node = ParseCodeNode(tokens)
49346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    Pop(']]')
49446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    inner_else_node = ParseElseNode(tokens)
49546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    return CodeNode([IfNode(ParseExpNode(exp), code_node, inner_else_node)])
49646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  elif not next.value.strip():
49746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    Pop('code')
49846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    return ParseElseNode(tokens)
49946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  else:
50046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    return None
50146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan
50246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan
50346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chandef ParseAtomicCodeNode(tokens):
50446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  def Pop(token_type=None):
50546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    return PopToken(tokens, token_type)
50646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan
50746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  head = PopFront(tokens)
50846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  t = head.token_type
50946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  if t == 'code':
51046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    return RawCodeNode(head)
51146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  elif t == '$var':
51246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    id_token = Pop('id')
51346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    Pop('=')
51446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    next = PeekToken(tokens)
51546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    if next.token_type == 'exp':
51646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan      exp_token = Pop()
51746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan      return VarNode(id_token, ParseExpNode(exp_token))
51846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    Pop('[[')
51946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    code_node = ParseCodeNode(tokens)
52046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    Pop(']]')
52146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    return VarNode(id_token, code_node)
52246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  elif t == '$for':
52346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    id_token = Pop('id')
52446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    next_token = PeekToken(tokens)
52546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    if next_token.token_type == 'code':
52646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan      sep_token = next_token
52746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan      Pop('code')
52846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    else:
52946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan      sep_token = None
53046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    Pop('[[')
53146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    code_node = ParseCodeNode(tokens)
53246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    Pop(']]')
53346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    return ForNode(id_token, sep_token, code_node)
53446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  elif t == '$if':
53546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    exp_token = Pop('code')
53646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    Pop('[[')
53746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    code_node = ParseCodeNode(tokens)
53846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    Pop(']]')
53946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    else_node = ParseElseNode(tokens)
54046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    return IfNode(ParseExpNode(exp_token), code_node, else_node)
54146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  elif t == '$range':
54246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    id_token = Pop('id')
54346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    exp1_token = Pop('exp')
54446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    Pop('..')
54546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    exp2_token = Pop('exp')
54646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    return RangeNode(id_token, ParseExpNode(exp1_token),
54746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan                     ParseExpNode(exp2_token))
54846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  elif t == '$id':
54946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    return ParseExpNode(Token(head.start + 1, head.end, head.value[1:], 'id'))
55046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  elif t == '$($)':
55146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    return LiteralDollarNode(head)
55246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  elif t == '$':
55346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    exp_token = Pop('exp')
55446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    return ParseExpNode(exp_token)
55546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  elif t == '[[':
55646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    code_node = ParseCodeNode(tokens)
55746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    Pop(']]')
55846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    return code_node
55946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  else:
56046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    PushFront(tokens, head)
56146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    return None
56246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan
56346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan
56446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chandef ParseCodeNode(tokens):
56546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  atomic_code_list = []
56646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  while True:
56746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    if not tokens:
56846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan      break
56946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    atomic_code_node = ParseAtomicCodeNode(tokens)
57046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    if atomic_code_node:
57146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan      atomic_code_list.append(atomic_code_node)
57246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    else:
57346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan      break
57446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  return CodeNode(atomic_code_list)
57546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan
57646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan
57746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chandef ParseToAST(pump_src_text):
57846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  """Convert the given Pump source text into an AST."""
57946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  tokens = list(Tokenize(pump_src_text))
58046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  code_node = ParseCodeNode(tokens)
58146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  return code_node
58246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan
58346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan
58446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chanclass Env:
58546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  def __init__(self):
58646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    self.variables = []
58746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    self.ranges = []
58846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan
58946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  def Clone(self):
59046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    clone = Env()
59146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    clone.variables = self.variables[:]
59246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    clone.ranges = self.ranges[:]
59346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    return clone
59446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan
59546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  def PushVariable(self, var, value):
59646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    # If value looks like an int, store it as an int.
59746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    try:
59846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan      int_value = int(value)
59946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan      if ('%s' % int_value) == value:
60046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan        value = int_value
60146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    except Exception:
60246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan      pass
60346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    self.variables[:0] = [(var, value)]
60446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan
60546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  def PopVariable(self):
60646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    self.variables[:1] = []
60746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan
60846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  def PushRange(self, var, lower, upper):
60946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    self.ranges[:0] = [(var, lower, upper)]
61046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan
61146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  def PopRange(self):
61246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    self.ranges[:1] = []
61346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan
61446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  def GetValue(self, identifier):
61546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    for (var, value) in self.variables:
61646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan      if identifier == var:
61746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan        return value
61846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan
61946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    print 'ERROR: meta variable %s is undefined.' % (identifier,)
62046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    sys.exit(1)
62146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan
62246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  def EvalExp(self, exp):
62346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    try:
62446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan      result = eval(exp.python_exp)
62546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    except Exception, e:
62646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan      print 'ERROR: caught exception %s: %s' % (e.__class__.__name__, e)
62746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan      print ('ERROR: failed to evaluate meta expression %s at %s' %
62846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan             (exp.python_exp, exp.token.start))
62946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan      sys.exit(1)
63046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    return result
63146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan
63246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  def GetRange(self, identifier):
63346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    for (var, lower, upper) in self.ranges:
63446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan      if identifier == var:
63546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan        return (lower, upper)
63646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan
63746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    print 'ERROR: range %s is undefined.' % (identifier,)
63846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    sys.exit(1)
63946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan
64046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan
64146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chanclass Output:
64246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  def __init__(self):
64346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    self.string = ''
64446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan
64546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  def GetLastLine(self):
64646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    index = self.string.rfind('\n')
64746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    if index < 0:
64846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan      return ''
64946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan
65046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    return self.string[index + 1:]
65146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan
65246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  def Append(self, s):
65346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    self.string += s
65446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan
65546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan
65646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chandef RunAtomicCode(env, node, output):
65746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  if isinstance(node, VarNode):
65846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    identifier = node.identifier.value.strip()
65946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    result = Output()
66046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    RunAtomicCode(env.Clone(), node.atomic_code, result)
66146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    value = result.string
66246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    env.PushVariable(identifier, value)
66346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  elif isinstance(node, RangeNode):
66446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    identifier = node.identifier.value.strip()
66546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    lower = int(env.EvalExp(node.exp1))
66646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    upper = int(env.EvalExp(node.exp2))
66746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    env.PushRange(identifier, lower, upper)
66846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  elif isinstance(node, ForNode):
66946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    identifier = node.identifier.value.strip()
67046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    if node.sep is None:
67146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan      sep = ''
67246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    else:
67346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan      sep = node.sep.value
67446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    (lower, upper) = env.GetRange(identifier)
67546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    for i in range(lower, upper + 1):
67646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan      new_env = env.Clone()
67746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan      new_env.PushVariable(identifier, i)
67846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan      RunCode(new_env, node.code, output)
67946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan      if i != upper:
68046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan        output.Append(sep)
68146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  elif isinstance(node, RawCodeNode):
68246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    output.Append(node.raw_code.value)
68346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  elif isinstance(node, IfNode):
68446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    cond = env.EvalExp(node.exp)
68546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    if cond:
68646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan      RunCode(env.Clone(), node.then_branch, output)
68746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    elif node.else_branch is not None:
68846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan      RunCode(env.Clone(), node.else_branch, output)
68946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  elif isinstance(node, ExpNode):
69046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    value = env.EvalExp(node)
69146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    output.Append('%s' % (value,))
69246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  elif isinstance(node, LiteralDollarNode):
69346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    output.Append('$')
69446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  elif isinstance(node, CodeNode):
69546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    RunCode(env.Clone(), node, output)
69646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  else:
69746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    print 'BAD'
69846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    print node
69946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    sys.exit(1)
70046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan
70146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan
70246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chandef RunCode(env, code_node, output):
70346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  for atomic_code in code_node.atomic_code:
70446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    RunAtomicCode(env, atomic_code, output)
70546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan
70646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan
70746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chandef IsComment(cur_line):
70846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  return '//' in cur_line
70946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan
71046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan
71146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chandef IsInPreprocessorDirevative(prev_lines, cur_line):
71246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  if cur_line.lstrip().startswith('#'):
71346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    return True
71446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  return prev_lines != [] and prev_lines[-1].endswith('\\')
71546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan
71646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan
71746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chandef WrapComment(line, output):
71846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  loc = line.find('//')
71946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  before_comment = line[:loc].rstrip()
72046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  if before_comment == '':
72146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    indent = loc
72246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  else:
72346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    output.append(before_comment)
72446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    indent = len(before_comment) - len(before_comment.lstrip())
72546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  prefix = indent*' ' + '// '
72646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  max_len = 80 - len(prefix)
72746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  comment = line[loc + 2:].strip()
72846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  segs = [seg for seg in re.split(r'(\w+\W*)', comment) if seg != '']
72946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  cur_line = ''
73046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  for seg in segs:
73146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    if len((cur_line + seg).rstrip()) < max_len:
73246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan      cur_line += seg
73346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    else:
73446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan      if cur_line.strip() != '':
73546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan        output.append(prefix + cur_line.rstrip())
73646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan      cur_line = seg.lstrip()
73746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  if cur_line.strip() != '':
73846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    output.append(prefix + cur_line.strip())
73946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan
74046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan
74146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chandef WrapCode(line, line_concat, output):
74246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  indent = len(line) - len(line.lstrip())
74346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  prefix = indent*' '  # Prefix of the current line
74446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  max_len = 80 - indent - len(line_concat)  # Maximum length of the current line
74546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  new_prefix = prefix + 4*' '  # Prefix of a continuation line
74646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  new_max_len = max_len - 4  # Maximum length of a continuation line
74746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  # Prefers to wrap a line after a ',' or ';'.
74846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  segs = [seg for seg in re.split(r'([^,;]+[,;]?)', line.strip()) if seg != '']
74946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  cur_line = ''  # The current line without leading spaces.
75046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  for seg in segs:
75146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    # If the line is still too long, wrap at a space.
75246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    while cur_line == '' and len(seg.strip()) > max_len:
75346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan      seg = seg.lstrip()
75446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan      split_at = seg.rfind(' ', 0, max_len)
75546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan      output.append(prefix + seg[:split_at].strip() + line_concat)
75646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan      seg = seg[split_at + 1:]
75746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan      prefix = new_prefix
75846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan      max_len = new_max_len
75946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan
76046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    if len((cur_line + seg).rstrip()) < max_len:
76146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan      cur_line = (cur_line + seg).lstrip()
76246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    else:
76346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan      output.append(prefix + cur_line.rstrip() + line_concat)
76446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan      prefix = new_prefix
76546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan      max_len = new_max_len
76646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan      cur_line = seg.lstrip()
76746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  if cur_line.strip() != '':
76846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    output.append(prefix + cur_line.strip())
76946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan
77046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan
77146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chandef WrapPreprocessorDirevative(line, output):
77246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  WrapCode(line, ' \\', output)
77346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan
77446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan
77546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chandef WrapPlainCode(line, output):
77646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  WrapCode(line, '', output)
77746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan
77846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan
77946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chandef IsHeaderGuardOrInclude(line):
78046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  return (re.match(r'^#(ifndef|define|endif\s*//)\s*[\w_]+\s*$', line) or
78146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan          re.match(r'^#include\s', line))
78246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan
78346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan
78446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chandef WrapLongLine(line, output):
78546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  line = line.rstrip()
78646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  if len(line) <= 80:
78746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    output.append(line)
78846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  elif IsComment(line):
78946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    if IsHeaderGuardOrInclude(line):
79046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan      # The style guide made an exception to allow long header guard lines
79146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan      # and includes.
79246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan      output.append(line)
79346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    else:
79446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan      WrapComment(line, output)
79546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  elif IsInPreprocessorDirevative(output, line):
79646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    if IsHeaderGuardOrInclude(line):
79746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan      # The style guide made an exception to allow long header guard lines
79846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan      # and includes.
79946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan      output.append(line)
80046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    else:
80146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan      WrapPreprocessorDirevative(line, output)
80246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  else:
80346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    WrapPlainCode(line, output)
80446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan
80546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan
80646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chandef BeautifyCode(string):
80746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  lines = string.splitlines()
80846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  output = []
80946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  for line in lines:
81046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    WrapLongLine(line, output)
81146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  output2 = [line.rstrip() for line in output]
81246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  return '\n'.join(output2) + '\n'
81346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan
81446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan
81546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chandef ConvertFromPumpSource(src_text):
81646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  """Return the text generated from the given Pump source text."""
81746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  ast = ParseToAST(StripMetaComments(src_text))
81846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  output = Output()
81946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  RunCode(Env(), ast, output)
82046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  return BeautifyCode(output.string)
82146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan
82246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan
82346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chandef main(argv):
82446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  if len(argv) == 1:
82546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    print __doc__
82646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    sys.exit(1)
82746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan
82846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  file_path = argv[-1]
82946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  output_str = ConvertFromPumpSource(file(file_path, 'r').read())
83046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  if file_path.endswith('.pump'):
83146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    output_file_path = file_path[:-5]
83246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  else:
83346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    output_file_path = '-'
83446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  if output_file_path == '-':
83546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    print output_str,
83646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  else:
83746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    output_file = file(output_file_path, 'w')
83846108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    output_file.write('// This file was GENERATED by command:\n')
83946108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    output_file.write('//     %s %s\n' %
84046108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan                      (os.path.basename(__file__), os.path.basename(file_path)))
84146108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    output_file.write('// DO NOT EDIT BY HAND!!!\n\n')
84246108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    output_file.write(output_str)
84346108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan    output_file.close()
84446108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan
84546108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan
84646108a219a4b812dd8f36fee479a0340ea5963f5Ben Chanif __name__ == '__main__':
84746108a219a4b812dd8f36fee479a0340ea5963f5Ben Chan  main(sys.argv)
848