1aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter#!/usr/bin/env python
2aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter#
3aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter# Copyright 2008, Google Inc.
4aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter# All rights reserved.
5aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter#
6aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter# Redistribution and use in source and binary forms, with or without
7aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter# modification, are permitted provided that the following conditions are
8aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter# met:
9aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter#
10aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter#     * Redistributions of source code must retain the above copyright
11aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter# notice, this list of conditions and the following disclaimer.
12aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter#     * Redistributions in binary form must reproduce the above
13aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter# copyright notice, this list of conditions and the following disclaimer
14aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter# in the documentation and/or other materials provided with the
15aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter# distribution.
16aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter#     * Neither the name of Google Inc. nor the names of its
17aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter# contributors may be used to endorse or promote products derived from
18aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter# this software without specific prior written permission.
19aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter#
20aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
21aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
22aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
23aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
24aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
25aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
26aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
27aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
28aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
29aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
30aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
31aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
32aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter"""pump v0.2.0 - Pretty Useful for Meta Programming.
33aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
34aa46da279e2426caf1e103eb079dfec8124c5feeCourtney GoeltzenleuchterA tool for preprocessor meta programming.  Useful for generating
35aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchterrepetitive boilerplate code.  Especially useful for writing C++
36aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchterclasses, functions, macros, and templates that need to work with
37aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchtervarious number of arguments.
38aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
39aa46da279e2426caf1e103eb079dfec8124c5feeCourtney GoeltzenleuchterUSAGE:
40aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter       pump.py SOURCE_FILE
41aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
42aa46da279e2426caf1e103eb079dfec8124c5feeCourtney GoeltzenleuchterEXAMPLES:
43aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter       pump.py foo.cc.pump
44aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter         Converts foo.cc.pump to foo.cc.
45aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
46aa46da279e2426caf1e103eb079dfec8124c5feeCourtney GoeltzenleuchterGRAMMAR:
47aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter       CODE ::= ATOMIC_CODE*
48aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter       ATOMIC_CODE ::= $var ID = EXPRESSION
49aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter           | $var ID = [[ CODE ]]
50aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter           | $range ID EXPRESSION..EXPRESSION
51aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter           | $for ID SEPARATOR [[ CODE ]]
52aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter           | $($)
53aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter           | $ID
54aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter           | $(EXPRESSION)
55aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter           | $if EXPRESSION [[ CODE ]] ELSE_BRANCH
56aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter           | [[ CODE ]]
57aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter           | RAW_CODE
58aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter       SEPARATOR ::= RAW_CODE | EMPTY
59aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter       ELSE_BRANCH ::= $else [[ CODE ]]
60aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter           | $elif EXPRESSION [[ CODE ]] ELSE_BRANCH
61aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter           | EMPTY
62aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter       EXPRESSION has Python syntax.
63aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter"""
64aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
65aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter__author__ = 'wan@google.com (Zhanyong Wan)'
66aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
67aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchterimport os
68aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchterimport re
69aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchterimport sys
70aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
71aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
72aa46da279e2426caf1e103eb079dfec8124c5feeCourtney GoeltzenleuchterTOKEN_TABLE = [
73aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    (re.compile(r'\$var\s+'), '$var'),
74aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    (re.compile(r'\$elif\s+'), '$elif'),
75aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    (re.compile(r'\$else\s+'), '$else'),
76aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    (re.compile(r'\$for\s+'), '$for'),
77aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    (re.compile(r'\$if\s+'), '$if'),
78aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    (re.compile(r'\$range\s+'), '$range'),
79aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    (re.compile(r'\$[_A-Za-z]\w*'), '$id'),
80aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    (re.compile(r'\$\(\$\)'), '$($)'),
81aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    (re.compile(r'\$'), '$'),
82aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    (re.compile(r'\[\[\n?'), '[['),
83aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    (re.compile(r'\]\]\n?'), ']]'),
84aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    ]
85aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
86aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
87aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchterclass Cursor:
88aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  """Represents a position (line and column) in a text file."""
89aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
90aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  def __init__(self, line=-1, column=-1):
91aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    self.line = line
92aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    self.column = column
93aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
94aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  def __eq__(self, rhs):
95aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    return self.line == rhs.line and self.column == rhs.column
96aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
97aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  def __ne__(self, rhs):
98aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    return not self == rhs
99aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
100aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  def __lt__(self, rhs):
101aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    return self.line < rhs.line or (
102aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter        self.line == rhs.line and self.column < rhs.column)
103aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
104aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  def __le__(self, rhs):
105aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    return self < rhs or self == rhs
106aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
107aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  def __gt__(self, rhs):
108aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    return rhs < self
109aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
110aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  def __ge__(self, rhs):
111aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    return rhs <= self
112aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
113aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  def __str__(self):
114aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    if self == Eof():
115aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter      return 'EOF'
116aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    else:
117aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter      return '%s(%s)' % (self.line + 1, self.column)
118aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
119aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  def __add__(self, offset):
120aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    return Cursor(self.line, self.column + offset)
121aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
122aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  def __sub__(self, offset):
123aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    return Cursor(self.line, self.column - offset)
124aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
125aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  def Clone(self):
126aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    """Returns a copy of self."""
127aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
128aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    return Cursor(self.line, self.column)
129aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
130aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
131aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter# Special cursor to indicate the end-of-file.
132aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchterdef Eof():
133aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  """Returns the special cursor to denote the end-of-file."""
134aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  return Cursor(-1, -1)
135aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
136aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
137aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchterclass Token:
138aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  """Represents a token in a Pump source file."""
139aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
140aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  def __init__(self, start=None, end=None, value=None, token_type=None):
141aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    if start is None:
142aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter      self.start = Eof()
143aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    else:
144aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter      self.start = start
145aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    if end is None:
146aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter      self.end = Eof()
147aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    else:
148aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter      self.end = end
149aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    self.value = value
150aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    self.token_type = token_type
151aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
152aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  def __str__(self):
153aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    return 'Token @%s: \'%s\' type=%s' % (
154aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter        self.start, self.value, self.token_type)
155aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
156aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  def Clone(self):
157aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    """Returns a copy of self."""
158aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
159aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    return Token(self.start.Clone(), self.end.Clone(), self.value,
160aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter                 self.token_type)
161aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
162aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
163aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchterdef StartsWith(lines, pos, string):
164aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  """Returns True iff the given position in lines starts with 'string'."""
165aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
166aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  return lines[pos.line][pos.column:].startswith(string)
167aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
168aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
169aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchterdef FindFirstInLine(line, token_table):
170aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  best_match_start = -1
171aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  for (regex, token_type) in token_table:
172aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    m = regex.search(line)
173aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    if m:
174aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter      # We found regex in lines
175aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter      if best_match_start < 0 or m.start() < best_match_start:
176aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter        best_match_start = m.start()
177aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter        best_match_length = m.end() - m.start()
178aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter        best_match_token_type = token_type
179aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
180aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  if best_match_start < 0:
181aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    return None
182aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
183aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  return (best_match_start, best_match_length, best_match_token_type)
184aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
185aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
186aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchterdef FindFirst(lines, token_table, cursor):
187aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  """Finds the first occurrence of any string in strings in lines."""
188aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
189aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  start = cursor.Clone()
190aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  cur_line_number = cursor.line
191aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  for line in lines[start.line:]:
192aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    if cur_line_number == start.line:
193aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter      line = line[start.column:]
194aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    m = FindFirstInLine(line, token_table)
195aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    if m:
196aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter      # We found a regex in line.
197aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter      (start_column, length, token_type) = m
198aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter      if cur_line_number == start.line:
199aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter        start_column += start.column
200aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter      found_start = Cursor(cur_line_number, start_column)
201aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter      found_end = found_start + length
202aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter      return MakeToken(lines, found_start, found_end, token_type)
203aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    cur_line_number += 1
204aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  # We failed to find str in lines
205aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  return None
206aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
207aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
208aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchterdef SubString(lines, start, end):
209aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  """Returns a substring in lines."""
210aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
211aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  if end == Eof():
212aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    end = Cursor(len(lines) - 1, len(lines[-1]))
213aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
214aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  if start >= end:
215aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    return ''
216aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
217aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  if start.line == end.line:
218aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    return lines[start.line][start.column:end.column]
219aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
220aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  result_lines = ([lines[start.line][start.column:]] +
221aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter                  lines[start.line + 1:end.line] +
222aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter                  [lines[end.line][:end.column]])
223aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  return ''.join(result_lines)
224aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
225aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
226aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchterdef StripMetaComments(str):
227aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  """Strip meta comments from each line in the given string."""
228aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
229aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  # First, completely remove lines containing nothing but a meta
230aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  # comment, including the trailing \n.
231aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  str = re.sub(r'^\s*\$\$.*\n', '', str)
232aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
233aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  # Then, remove meta comments from contentful lines.
234aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  return re.sub(r'\s*\$\$.*', '', str)
235aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
236aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
237aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchterdef MakeToken(lines, start, end, token_type):
238aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  """Creates a new instance of Token."""
239aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
240aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  return Token(start, end, SubString(lines, start, end), token_type)
241aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
242aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
243aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchterdef ParseToken(lines, pos, regex, token_type):
244aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  line = lines[pos.line][pos.column:]
245aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  m = regex.search(line)
246aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  if m and not m.start():
247aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    return MakeToken(lines, pos, pos + m.end(), token_type)
248aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  else:
249aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    print 'ERROR: %s expected at %s.' % (token_type, pos)
250aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    sys.exit(1)
251aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
252aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
253aa46da279e2426caf1e103eb079dfec8124c5feeCourtney GoeltzenleuchterID_REGEX = re.compile(r'[_A-Za-z]\w*')
254aa46da279e2426caf1e103eb079dfec8124c5feeCourtney GoeltzenleuchterEQ_REGEX = re.compile(r'=')
255aa46da279e2426caf1e103eb079dfec8124c5feeCourtney GoeltzenleuchterREST_OF_LINE_REGEX = re.compile(r'.*?(?=$|\$\$)')
256aa46da279e2426caf1e103eb079dfec8124c5feeCourtney GoeltzenleuchterOPTIONAL_WHITE_SPACES_REGEX = re.compile(r'\s*')
257aa46da279e2426caf1e103eb079dfec8124c5feeCourtney GoeltzenleuchterWHITE_SPACE_REGEX = re.compile(r'\s')
258aa46da279e2426caf1e103eb079dfec8124c5feeCourtney GoeltzenleuchterDOT_DOT_REGEX = re.compile(r'\.\.')
259aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
260aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
261aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchterdef Skip(lines, pos, regex):
262aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  line = lines[pos.line][pos.column:]
263aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  m = re.search(regex, line)
264aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  if m and not m.start():
265aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    return pos + m.end()
266aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  else:
267aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    return pos
268aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
269aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
270aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchterdef SkipUntil(lines, pos, regex, token_type):
271aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  line = lines[pos.line][pos.column:]
272aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  m = re.search(regex, line)
273aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  if m:
274aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    return pos + m.start()
275aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  else:
276aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    print ('ERROR: %s expected on line %s after column %s.' %
277aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter           (token_type, pos.line + 1, pos.column))
278aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    sys.exit(1)
279aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
280aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
281aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchterdef ParseExpTokenInParens(lines, pos):
282aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  def ParseInParens(pos):
283aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    pos = Skip(lines, pos, OPTIONAL_WHITE_SPACES_REGEX)
284aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    pos = Skip(lines, pos, r'\(')
285aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    pos = Parse(pos)
286aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    pos = Skip(lines, pos, r'\)')
287aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    return pos
288aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
289aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  def Parse(pos):
290aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    pos = SkipUntil(lines, pos, r'\(|\)', ')')
291aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    if SubString(lines, pos, pos + 1) == '(':
292aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter      pos = Parse(pos + 1)
293aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter      pos = Skip(lines, pos, r'\)')
294aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter      return Parse(pos)
295aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    else:
296aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter      return pos
297aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
298aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  start = pos.Clone()
299aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  pos = ParseInParens(pos)
300aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  return MakeToken(lines, start, pos, 'exp')
301aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
302aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
303aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchterdef RStripNewLineFromToken(token):
304aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  if token.value.endswith('\n'):
305aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    return Token(token.start, token.end, token.value[:-1], token.token_type)
306aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  else:
307aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    return token
308aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
309aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
310aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchterdef TokenizeLines(lines, pos):
311aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  while True:
312aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    found = FindFirst(lines, TOKEN_TABLE, pos)
313aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    if not found:
314aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter      yield MakeToken(lines, pos, Eof(), 'code')
315aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter      return
316aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
317aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    if found.start == pos:
318aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter      prev_token = None
319aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter      prev_token_rstripped = None
320aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    else:
321aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter      prev_token = MakeToken(lines, pos, found.start, 'code')
322aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter      prev_token_rstripped = RStripNewLineFromToken(prev_token)
323aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
324aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    if found.token_type == '$var':
325aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter      if prev_token_rstripped:
326aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter        yield prev_token_rstripped
327aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter      yield found
328aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter      id_token = ParseToken(lines, found.end, ID_REGEX, 'id')
329aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter      yield id_token
330aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter      pos = Skip(lines, id_token.end, OPTIONAL_WHITE_SPACES_REGEX)
331aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
332aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter      eq_token = ParseToken(lines, pos, EQ_REGEX, '=')
333aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter      yield eq_token
334aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter      pos = Skip(lines, eq_token.end, r'\s*')
335aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
336aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter      if SubString(lines, pos, pos + 2) != '[[':
337aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter        exp_token = ParseToken(lines, pos, REST_OF_LINE_REGEX, 'exp')
338aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter        yield exp_token
339aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter        pos = Cursor(exp_token.end.line + 1, 0)
340aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    elif found.token_type == '$for':
341aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter      if prev_token_rstripped:
342aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter        yield prev_token_rstripped
343aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter      yield found
344aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter      id_token = ParseToken(lines, found.end, ID_REGEX, 'id')
345aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter      yield id_token
346aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter      pos = Skip(lines, id_token.end, WHITE_SPACE_REGEX)
347aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    elif found.token_type == '$range':
348aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter      if prev_token_rstripped:
349aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter        yield prev_token_rstripped
350aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter      yield found
351aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter      id_token = ParseToken(lines, found.end, ID_REGEX, 'id')
352aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter      yield id_token
353aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter      pos = Skip(lines, id_token.end, OPTIONAL_WHITE_SPACES_REGEX)
354aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
355aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter      dots_pos = SkipUntil(lines, pos, DOT_DOT_REGEX, '..')
356aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter      yield MakeToken(lines, pos, dots_pos, 'exp')
357aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter      yield MakeToken(lines, dots_pos, dots_pos + 2, '..')
358aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter      pos = dots_pos + 2
359aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter      new_pos = Cursor(pos.line + 1, 0)
360aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter      yield MakeToken(lines, pos, new_pos, 'exp')
361aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter      pos = new_pos
362aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    elif found.token_type == '$':
363aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter      if prev_token:
364aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter        yield prev_token
365aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter      yield found
366aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter      exp_token = ParseExpTokenInParens(lines, found.end)
367aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter      yield exp_token
368aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter      pos = exp_token.end
369aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    elif (found.token_type == ']]' or found.token_type == '$if' or
370aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter          found.token_type == '$elif' or found.token_type == '$else'):
371aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter      if prev_token_rstripped:
372aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter        yield prev_token_rstripped
373aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter      yield found
374aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter      pos = found.end
375aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    else:
376aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter      if prev_token:
377aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter        yield prev_token
378aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter      yield found
379aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter      pos = found.end
380aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
381aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
382aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchterdef Tokenize(s):
383aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  """A generator that yields the tokens in the given string."""
384aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  if s != '':
385aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    lines = s.splitlines(True)
386aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    for token in TokenizeLines(lines, Cursor(0, 0)):
387aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter      yield token
388aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
389aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
390aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchterclass CodeNode:
391aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  def __init__(self, atomic_code_list=None):
392aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    self.atomic_code = atomic_code_list
393aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
394aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
395aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchterclass VarNode:
396aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  def __init__(self, identifier=None, atomic_code=None):
397aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    self.identifier = identifier
398aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    self.atomic_code = atomic_code
399aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
400aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
401aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchterclass RangeNode:
402aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  def __init__(self, identifier=None, exp1=None, exp2=None):
403aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    self.identifier = identifier
404aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    self.exp1 = exp1
405aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    self.exp2 = exp2
406aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
407aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
408aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchterclass ForNode:
409aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  def __init__(self, identifier=None, sep=None, code=None):
410aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    self.identifier = identifier
411aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    self.sep = sep
412aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    self.code = code
413aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
414aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
415aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchterclass ElseNode:
416aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  def __init__(self, else_branch=None):
417aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    self.else_branch = else_branch
418aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
419aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
420aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchterclass IfNode:
421aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  def __init__(self, exp=None, then_branch=None, else_branch=None):
422aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    self.exp = exp
423aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    self.then_branch = then_branch
424aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    self.else_branch = else_branch
425aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
426aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
427aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchterclass RawCodeNode:
428aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  def __init__(self, token=None):
429aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    self.raw_code = token
430aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
431aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
432aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchterclass LiteralDollarNode:
433aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  def __init__(self, token):
434aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    self.token = token
435aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
436aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
437aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchterclass ExpNode:
438aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  def __init__(self, token, python_exp):
439aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    self.token = token
440aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    self.python_exp = python_exp
441aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
442aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
443aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchterdef PopFront(a_list):
444aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  head = a_list[0]
445aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  a_list[:1] = []
446aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  return head
447aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
448aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
449aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchterdef PushFront(a_list, elem):
450aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  a_list[:0] = [elem]
451aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
452aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
453aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchterdef PopToken(a_list, token_type=None):
454aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  token = PopFront(a_list)
455aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  if token_type is not None and token.token_type != token_type:
456aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    print 'ERROR: %s expected at %s' % (token_type, token.start)
457aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    print 'ERROR: %s found instead' % (token,)
458aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    sys.exit(1)
459aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
460aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  return token
461aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
462aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
463aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchterdef PeekToken(a_list):
464aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  if not a_list:
465aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    return None
466aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
467aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  return a_list[0]
468aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
469aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
470aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchterdef ParseExpNode(token):
471aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  python_exp = re.sub(r'([_A-Za-z]\w*)', r'self.GetValue("\1")', token.value)
472aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  return ExpNode(token, python_exp)
473aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
474aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
475aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchterdef ParseElseNode(tokens):
476aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  def Pop(token_type=None):
477aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    return PopToken(tokens, token_type)
478aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
479aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  next = PeekToken(tokens)
480aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  if not next:
481aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    return None
482aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  if next.token_type == '$else':
483aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    Pop('$else')
484aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    Pop('[[')
485aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    code_node = ParseCodeNode(tokens)
486aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    Pop(']]')
487aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    return code_node
488aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  elif next.token_type == '$elif':
489aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    Pop('$elif')
490aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    exp = Pop('code')
491aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    Pop('[[')
492aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    code_node = ParseCodeNode(tokens)
493aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    Pop(']]')
494aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    inner_else_node = ParseElseNode(tokens)
495aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    return CodeNode([IfNode(ParseExpNode(exp), code_node, inner_else_node)])
496aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  elif not next.value.strip():
497aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    Pop('code')
498aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    return ParseElseNode(tokens)
499aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  else:
500aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    return None
501aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
502aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
503aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchterdef ParseAtomicCodeNode(tokens):
504aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  def Pop(token_type=None):
505aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    return PopToken(tokens, token_type)
506aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
507aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  head = PopFront(tokens)
508aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  t = head.token_type
509aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  if t == 'code':
510aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    return RawCodeNode(head)
511aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  elif t == '$var':
512aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    id_token = Pop('id')
513aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    Pop('=')
514aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    next = PeekToken(tokens)
515aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    if next.token_type == 'exp':
516aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter      exp_token = Pop()
517aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter      return VarNode(id_token, ParseExpNode(exp_token))
518aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    Pop('[[')
519aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    code_node = ParseCodeNode(tokens)
520aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    Pop(']]')
521aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    return VarNode(id_token, code_node)
522aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  elif t == '$for':
523aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    id_token = Pop('id')
524aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    next_token = PeekToken(tokens)
525aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    if next_token.token_type == 'code':
526aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter      sep_token = next_token
527aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter      Pop('code')
528aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    else:
529aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter      sep_token = None
530aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    Pop('[[')
531aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    code_node = ParseCodeNode(tokens)
532aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    Pop(']]')
533aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    return ForNode(id_token, sep_token, code_node)
534aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  elif t == '$if':
535aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    exp_token = Pop('code')
536aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    Pop('[[')
537aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    code_node = ParseCodeNode(tokens)
538aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    Pop(']]')
539aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    else_node = ParseElseNode(tokens)
540aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    return IfNode(ParseExpNode(exp_token), code_node, else_node)
541aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  elif t == '$range':
542aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    id_token = Pop('id')
543aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    exp1_token = Pop('exp')
544aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    Pop('..')
545aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    exp2_token = Pop('exp')
546aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    return RangeNode(id_token, ParseExpNode(exp1_token),
547aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter                     ParseExpNode(exp2_token))
548aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  elif t == '$id':
549aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    return ParseExpNode(Token(head.start + 1, head.end, head.value[1:], 'id'))
550aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  elif t == '$($)':
551aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    return LiteralDollarNode(head)
552aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  elif t == '$':
553aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    exp_token = Pop('exp')
554aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    return ParseExpNode(exp_token)
555aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  elif t == '[[':
556aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    code_node = ParseCodeNode(tokens)
557aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    Pop(']]')
558aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    return code_node
559aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  else:
560aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    PushFront(tokens, head)
561aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    return None
562aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
563aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
564aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchterdef ParseCodeNode(tokens):
565aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  atomic_code_list = []
566aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  while True:
567aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    if not tokens:
568aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter      break
569aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    atomic_code_node = ParseAtomicCodeNode(tokens)
570aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    if atomic_code_node:
571aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter      atomic_code_list.append(atomic_code_node)
572aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    else:
573aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter      break
574aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  return CodeNode(atomic_code_list)
575aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
576aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
577aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchterdef ParseToAST(pump_src_text):
578aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  """Convert the given Pump source text into an AST."""
579aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  tokens = list(Tokenize(pump_src_text))
580aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  code_node = ParseCodeNode(tokens)
581aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  return code_node
582aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
583aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
584aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchterclass Env:
585aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  def __init__(self):
586aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    self.variables = []
587aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    self.ranges = []
588aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
589aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  def Clone(self):
590aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    clone = Env()
591aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    clone.variables = self.variables[:]
592aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    clone.ranges = self.ranges[:]
593aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    return clone
594aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
595aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  def PushVariable(self, var, value):
596aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    # If value looks like an int, store it as an int.
597aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    try:
598aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter      int_value = int(value)
599aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter      if ('%s' % int_value) == value:
600aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter        value = int_value
601aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    except Exception:
602aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter      pass
603aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    self.variables[:0] = [(var, value)]
604aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
605aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  def PopVariable(self):
606aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    self.variables[:1] = []
607aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
608aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  def PushRange(self, var, lower, upper):
609aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    self.ranges[:0] = [(var, lower, upper)]
610aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
611aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  def PopRange(self):
612aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    self.ranges[:1] = []
613aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
614aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  def GetValue(self, identifier):
615aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    for (var, value) in self.variables:
616aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter      if identifier == var:
617aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter        return value
618aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
619aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    print 'ERROR: meta variable %s is undefined.' % (identifier,)
620aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    sys.exit(1)
621aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
622aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  def EvalExp(self, exp):
623aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    try:
624aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter      result = eval(exp.python_exp)
625aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    except Exception, e:
626aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter      print 'ERROR: caught exception %s: %s' % (e.__class__.__name__, e)
627aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter      print ('ERROR: failed to evaluate meta expression %s at %s' %
628aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter             (exp.python_exp, exp.token.start))
629aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter      sys.exit(1)
630aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    return result
631aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
632aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  def GetRange(self, identifier):
633aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    for (var, lower, upper) in self.ranges:
634aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter      if identifier == var:
635aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter        return (lower, upper)
636aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
637aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    print 'ERROR: range %s is undefined.' % (identifier,)
638aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    sys.exit(1)
639aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
640aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
641aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchterclass Output:
642aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  def __init__(self):
643aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    self.string = ''
644aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
645aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  def GetLastLine(self):
646aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    index = self.string.rfind('\n')
647aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    if index < 0:
648aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter      return ''
649aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
650aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    return self.string[index + 1:]
651aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
652aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  def Append(self, s):
653aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    self.string += s
654aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
655aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
656aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchterdef RunAtomicCode(env, node, output):
657aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  if isinstance(node, VarNode):
658aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    identifier = node.identifier.value.strip()
659aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    result = Output()
660aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    RunAtomicCode(env.Clone(), node.atomic_code, result)
661aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    value = result.string
662aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    env.PushVariable(identifier, value)
663aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  elif isinstance(node, RangeNode):
664aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    identifier = node.identifier.value.strip()
665aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    lower = int(env.EvalExp(node.exp1))
666aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    upper = int(env.EvalExp(node.exp2))
667aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    env.PushRange(identifier, lower, upper)
668aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  elif isinstance(node, ForNode):
669aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    identifier = node.identifier.value.strip()
670aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    if node.sep is None:
671aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter      sep = ''
672aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    else:
673aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter      sep = node.sep.value
674aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    (lower, upper) = env.GetRange(identifier)
675aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    for i in range(lower, upper + 1):
676aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter      new_env = env.Clone()
677aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter      new_env.PushVariable(identifier, i)
678aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter      RunCode(new_env, node.code, output)
679aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter      if i != upper:
680aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter        output.Append(sep)
681aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  elif isinstance(node, RawCodeNode):
682aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    output.Append(node.raw_code.value)
683aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  elif isinstance(node, IfNode):
684aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    cond = env.EvalExp(node.exp)
685aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    if cond:
686aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter      RunCode(env.Clone(), node.then_branch, output)
687aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    elif node.else_branch is not None:
688aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter      RunCode(env.Clone(), node.else_branch, output)
689aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  elif isinstance(node, ExpNode):
690aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    value = env.EvalExp(node)
691aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    output.Append('%s' % (value,))
692aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  elif isinstance(node, LiteralDollarNode):
693aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    output.Append('$')
694aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  elif isinstance(node, CodeNode):
695aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    RunCode(env.Clone(), node, output)
696aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  else:
697aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    print 'BAD'
698aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    print node
699aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    sys.exit(1)
700aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
701aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
702aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchterdef RunCode(env, code_node, output):
703aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  for atomic_code in code_node.atomic_code:
704aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    RunAtomicCode(env, atomic_code, output)
705aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
706aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
707aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchterdef IsSingleLineComment(cur_line):
708aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  return '//' in cur_line
709aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
710aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
711aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchterdef IsInPreprocessorDirective(prev_lines, cur_line):
712aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  if cur_line.lstrip().startswith('#'):
713aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    return True
714aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  return prev_lines and prev_lines[-1].endswith('\\')
715aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
716aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
717aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchterdef WrapComment(line, output):
718aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  loc = line.find('//')
719aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  before_comment = line[:loc].rstrip()
720aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  if before_comment == '':
721aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    indent = loc
722aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  else:
723aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    output.append(before_comment)
724aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    indent = len(before_comment) - len(before_comment.lstrip())
725aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  prefix = indent*' ' + '// '
726aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  max_len = 80 - len(prefix)
727aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  comment = line[loc + 2:].strip()
728aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  segs = [seg for seg in re.split(r'(\w+\W*)', comment) if seg != '']
729aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  cur_line = ''
730aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  for seg in segs:
731aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    if len((cur_line + seg).rstrip()) < max_len:
732aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter      cur_line += seg
733aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    else:
734aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter      if cur_line.strip() != '':
735aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter        output.append(prefix + cur_line.rstrip())
736aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter      cur_line = seg.lstrip()
737aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  if cur_line.strip() != '':
738aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    output.append(prefix + cur_line.strip())
739aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
740aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
741aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchterdef WrapCode(line, line_concat, output):
742aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  indent = len(line) - len(line.lstrip())
743aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  prefix = indent*' '  # Prefix of the current line
744aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  max_len = 80 - indent - len(line_concat)  # Maximum length of the current line
745aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  new_prefix = prefix + 4*' '  # Prefix of a continuation line
746aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  new_max_len = max_len - 4  # Maximum length of a continuation line
747aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  # Prefers to wrap a line after a ',' or ';'.
748aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  segs = [seg for seg in re.split(r'([^,;]+[,;]?)', line.strip()) if seg != '']
749aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  cur_line = ''  # The current line without leading spaces.
750aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  for seg in segs:
751aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    # If the line is still too long, wrap at a space.
752aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    while cur_line == '' and len(seg.strip()) > max_len:
753aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter      seg = seg.lstrip()
754aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter      split_at = seg.rfind(' ', 0, max_len)
755aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter      output.append(prefix + seg[:split_at].strip() + line_concat)
756aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter      seg = seg[split_at + 1:]
757aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter      prefix = new_prefix
758aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter      max_len = new_max_len
759aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
760aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    if len((cur_line + seg).rstrip()) < max_len:
761aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter      cur_line = (cur_line + seg).lstrip()
762aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    else:
763aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter      output.append(prefix + cur_line.rstrip() + line_concat)
764aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter      prefix = new_prefix
765aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter      max_len = new_max_len
766aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter      cur_line = seg.lstrip()
767aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  if cur_line.strip() != '':
768aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    output.append(prefix + cur_line.strip())
769aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
770aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
771aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchterdef WrapPreprocessorDirective(line, output):
772aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  WrapCode(line, ' \\', output)
773aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
774aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
775aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchterdef WrapPlainCode(line, output):
776aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  WrapCode(line, '', output)
777aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
778aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
779aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchterdef IsMultiLineIWYUPragma(line):
780aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  return re.search(r'/\* IWYU pragma: ', line)
781aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
782aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
783aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchterdef IsHeaderGuardIncludeOrOneLineIWYUPragma(line):
784aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  return (re.match(r'^#(ifndef|define|endif\s*//)\s*[\w_]+\s*$', line) or
785aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter          re.match(r'^#include\s', line) or
786aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter          # Don't break IWYU pragmas, either; that causes iwyu.py problems.
787aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter          re.search(r'// IWYU pragma: ', line))
788aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
789aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
790aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchterdef WrapLongLine(line, output):
791aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  line = line.rstrip()
792aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  if len(line) <= 80:
793aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    output.append(line)
794aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  elif IsSingleLineComment(line):
795aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    if IsHeaderGuardIncludeOrOneLineIWYUPragma(line):
796aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter      # The style guide made an exception to allow long header guard lines,
797aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter      # includes and IWYU pragmas.
798aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter      output.append(line)
799aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    else:
800aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter      WrapComment(line, output)
801aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  elif IsInPreprocessorDirective(output, line):
802aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    if IsHeaderGuardIncludeOrOneLineIWYUPragma(line):
803aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter      # The style guide made an exception to allow long header guard lines,
804aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter      # includes and IWYU pragmas.
805aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter      output.append(line)
806aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    else:
807aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter      WrapPreprocessorDirective(line, output)
808aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  elif IsMultiLineIWYUPragma(line):
809aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    output.append(line)
810aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  else:
811aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    WrapPlainCode(line, output)
812aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
813aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
814aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchterdef BeautifyCode(string):
815aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  lines = string.splitlines()
816aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  output = []
817aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  for line in lines:
818aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    WrapLongLine(line, output)
819aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  output2 = [line.rstrip() for line in output]
820aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  return '\n'.join(output2) + '\n'
821aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
822aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
823aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchterdef ConvertFromPumpSource(src_text):
824aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  """Return the text generated from the given Pump source text."""
825aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  ast = ParseToAST(StripMetaComments(src_text))
826aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  output = Output()
827aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  RunCode(Env(), ast, output)
828aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  return BeautifyCode(output.string)
829aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
830aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
831aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchterdef main(argv):
832aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  if len(argv) == 1:
833aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    print __doc__
834aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    sys.exit(1)
835aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
836aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  file_path = argv[-1]
837aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  output_str = ConvertFromPumpSource(file(file_path, 'r').read())
838aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  if file_path.endswith('.pump'):
839aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    output_file_path = file_path[:-5]
840aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  else:
841aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    output_file_path = '-'
842aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  if output_file_path == '-':
843aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    print output_str,
844aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  else:
845aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    output_file = file(output_file_path, 'w')
846aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    output_file.write('// This file was GENERATED by command:\n')
847aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    output_file.write('//     %s %s\n' %
848aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter                      (os.path.basename(__file__), os.path.basename(file_path)))
849aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    output_file.write('// DO NOT EDIT BY HAND!!!\n\n')
850aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    output_file.write(output_str)
851aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter    output_file.close()
852aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
853aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter
854aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchterif __name__ == '__main__':
855aa46da279e2426caf1e103eb079dfec8124c5feeCourtney Goeltzenleuchter  main(sys.argv)
856