1324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver"""ANTLR3 runtime package""" 2324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 3324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver# begin[licence] 4324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver# 5324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver# [The "BSD licence"] 6324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver# Copyright (c) 2005-2008 Terence Parr 7324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver# All rights reserved. 8324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver# 9324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver# Redistribution and use in source and binary forms, with or without 10324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver# modification, are permitted provided that the following conditions 11324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver# are met: 12324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver# 1. Redistributions of source code must retain the above copyright 13324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver# notice, this list of conditions and the following disclaimer. 14324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver# 2. Redistributions in binary form must reproduce the above copyright 15324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver# notice, this list of conditions and the following disclaimer in the 16324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver# documentation and/or other materials provided with the distribution. 17324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver# 3. The name of the author may not be used to endorse or promote products 18324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver# derived from this software without specific prior written permission. 19324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver# 20324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver# THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 21324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 22324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver# OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 23324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver# IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 24324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 25324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver# NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 26324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 27324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 28324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 29324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver# 31324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver# end[licence] 32324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 33324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruverimport codecs 34324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruverfrom StringIO import StringIO 35324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 36324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruverfrom antlr3.constants import DEFAULT_CHANNEL, EOF 37324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruverfrom antlr3.tokens import Token, CommonToken 38324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 39324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 40324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver############################################################################ 41324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver# 42324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver# basic interfaces 43324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver# IntStream 44324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver# +- CharStream 45324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver# \- TokenStream 46324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver# 47324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver# subclasses must implemented all methods 48324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver# 49324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver############################################################################ 50324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 51324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruverclass IntStream(object): 52324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver """ 53324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver @brief Base interface for streams of integer values. 54324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 55324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver A simple stream of integers used when all I care about is the char 56324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver or token type sequence (such as interpretation). 57324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver """ 58324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 59324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver def consume(self): 60324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver raise NotImplementedError 61324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 62324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 63324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver def LA(self, i): 64324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver """Get int at current input pointer + i ahead where i=1 is next int. 65324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 66324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver Negative indexes are allowed. LA(-1) is previous token (token 67324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver just matched). LA(-i) where i is before first token should 68324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver yield -1, invalid char / EOF. 69324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver """ 70324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 71324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver raise NotImplementedError 72324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 73324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 74324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver def mark(self): 75324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver """ 76324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver Tell the stream to start buffering if it hasn't already. Return 77324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver current input position, index(), or some other marker so that 78324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver when passed to rewind() you get back to the same spot. 79324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver rewind(mark()) should not affect the input cursor. The Lexer 80324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver track line/col info as well as input index so its markers are 81324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver not pure input indexes. Same for tree node streams. 82324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver """ 83324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 84324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver raise NotImplementedError 85324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 86324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 87324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver def index(self): 88324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver """ 89324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver Return the current input symbol index 0..n where n indicates the 90324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver last symbol has been read. The index is the symbol about to be 91324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver read not the most recently read symbol. 92324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver """ 93324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 94324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver raise NotImplementedError 95324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 96324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 97324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver def rewind(self, marker=None): 98324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver """ 99324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver Reset the stream so that next call to index would return marker. 100324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver The marker will usually be index() but it doesn't have to be. It's 101324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver just a marker to indicate what state the stream was in. This is 102324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver essentially calling release() and seek(). If there are markers 103324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver created after this marker argument, this routine must unroll them 104324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver like a stack. Assume the state the stream was in when this marker 105324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver was created. 106324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 107324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver If marker is None: 108324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver Rewind to the input position of the last marker. 109324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver Used currently only after a cyclic DFA and just 110324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver before starting a sem/syn predicate to get the 111324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver input position back to the start of the decision. 112324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver Do not "pop" the marker off the state. mark(i) 113324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver and rewind(i) should balance still. It is 114324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver like invoking rewind(last marker) but it should not "pop" 115324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver the marker off. It's like seek(last marker's input position). 116324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver """ 117324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 118324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver raise NotImplementedError 119324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 120324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 121324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver def release(self, marker=None): 122324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver """ 123324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver You may want to commit to a backtrack but don't want to force the 124324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver stream to keep bookkeeping objects around for a marker that is 125324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver no longer necessary. This will have the same behavior as 126324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver rewind() except it releases resources without the backward seek. 127324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver This must throw away resources for all markers back to the marker 128324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver argument. So if you're nested 5 levels of mark(), and then release(2) 129324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver you have to release resources for depths 2..5. 130324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver """ 131324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 132324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver raise NotImplementedError 133324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 134324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 135324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver def seek(self, index): 136324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver """ 137324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver Set the input cursor to the position indicated by index. This is 138324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver normally used to seek ahead in the input stream. No buffering is 139324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver required to do this unless you know your stream will use seek to 140324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver move backwards such as when backtracking. 141324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 142324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver This is different from rewind in its multi-directional 143324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver requirement and in that its argument is strictly an input cursor 144324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver (index). 145324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 146324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver For char streams, seeking forward must update the stream state such 147324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver as line number. For seeking backwards, you will be presumably 148324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver backtracking using the mark/rewind mechanism that restores state and 149324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver so this method does not need to update state when seeking backwards. 150324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 151324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver Currently, this method is only used for efficient backtracking using 152324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver memoization, but in the future it may be used for incremental parsing. 153324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 154324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver The index is 0..n-1. A seek to position i means that LA(1) will 155324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return the ith symbol. So, seeking to 0 means LA(1) will return the 156324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver first element in the stream. 157324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver """ 158324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 159324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver raise NotImplementedError 160324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 161324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 162324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver def size(self): 163324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver """ 164324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver Only makes sense for streams that buffer everything up probably, but 165324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver might be useful to display the entire stream or for testing. This 166324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver value includes a single EOF. 167324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver """ 168324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 169324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver raise NotImplementedError 170324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 171324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 172324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver def getSourceName(self): 173324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver """ 174324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver Where are you getting symbols from? Normally, implementations will 175324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver pass the buck all the way to the lexer who can ask its input stream 176324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver for the file name or whatever. 177324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver """ 178324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 179324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver raise NotImplementedError 180324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 181324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 182324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruverclass CharStream(IntStream): 183324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver """ 184324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver @brief A source of characters for an ANTLR lexer. 185324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 186324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver This is an abstract class that must be implemented by a subclass. 187324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 188324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver """ 189324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 190324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver # pylint does not realize that this is an interface, too 191324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver #pylint: disable-msg=W0223 192324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 193324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver EOF = -1 194324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 195324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 196324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver def substring(self, start, stop): 197324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver """ 198324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver For infinite streams, you don't need this; primarily I'm providing 199324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver a useful interface for action code. Just make sure actions don't 200324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver use this on streams that don't support it. 201324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver """ 202324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 203324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver raise NotImplementedError 204324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 205324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 206324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver def LT(self, i): 207324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver """ 208324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver Get the ith character of lookahead. This is the same usually as 209324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver LA(i). This will be used for labels in the generated 210324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver lexer code. I'd prefer to return a char here type-wise, but it's 211324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver probably better to be 32-bit clean and be consistent with LA. 212324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver """ 213324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 214324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver raise NotImplementedError 215324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 216324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 217324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver def getLine(self): 218324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver """ANTLR tracks the line information automatically""" 219324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 220324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver raise NotImplementedError 221324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 222324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 223324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver def setLine(self, line): 224324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver """ 225324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver Because this stream can rewind, we need to be able to reset the line 226324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver """ 227324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 228324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver raise NotImplementedError 229324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 230324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 231324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver def getCharPositionInLine(self): 232324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver """ 233324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver The index of the character relative to the beginning of the line 0..n-1 234324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver """ 235324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 236324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver raise NotImplementedError 237324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 238324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 239324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver def setCharPositionInLine(self, pos): 240324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver raise NotImplementedError 241324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 242324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 243324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruverclass TokenStream(IntStream): 244324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver """ 245324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 246324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver @brief A stream of tokens accessing tokens from a TokenSource 247324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 248324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver This is an abstract class that must be implemented by a subclass. 249324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 250324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver """ 251324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 252324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver # pylint does not realize that this is an interface, too 253324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver #pylint: disable-msg=W0223 254324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 255324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver def LT(self, k): 256324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver """ 257324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver Get Token at current input pointer + i ahead where i=1 is next Token. 258324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver i<0 indicates tokens in the past. So -1 is previous token and -2 is 259324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver two tokens ago. LT(0) is undefined. For i>=n, return Token.EOFToken. 260324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver Return null for LT(0) and any index that results in an absolute address 261324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver that is negative. 262324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver """ 263324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 264324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver raise NotImplementedError 265324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 266324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 267324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver def range(self): 268324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver """ 269324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver How far ahead has the stream been asked to look? The return 270324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver value is a valid index from 0..n-1. 271324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver """ 272324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 273324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver raise NotImplementedError 274324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 275324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 276324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver def get(self, i): 277324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver """ 278324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver Get a token at an absolute index i; 0..n-1. This is really only 279324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver needed for profiling and debugging and token stream rewriting. 280324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver If you don't want to buffer up tokens, then this method makes no 281324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver sense for you. Naturally you can't use the rewrite stream feature. 282324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver I believe DebugTokenStream can easily be altered to not use 283324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver this method, removing the dependency. 284324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver """ 285324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 286324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver raise NotImplementedError 287324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 288324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 289324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver def getTokenSource(self): 290324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver """ 291324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver Where is this stream pulling tokens from? This is not the name, but 292324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver the object that provides Token objects. 293324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver """ 294324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 295324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver raise NotImplementedError 296324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 297324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 298324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver def toString(self, start=None, stop=None): 299324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver """ 300324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver Return the text of all tokens from start to stop, inclusive. 301324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver If the stream does not buffer all the tokens then it can just 302324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return "" or null; Users should not access $ruleLabel.text in 303324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver an action of course in that case. 304324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 305324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver Because the user is not required to use a token with an index stored 306324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver in it, we must provide a means for two token objects themselves to 307324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver indicate the start/end location. Most often this will just delegate 308324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver to the other toString(int,int). This is also parallel with 309324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver the TreeNodeStream.toString(Object,Object). 310324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver """ 311324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 312324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver raise NotImplementedError 313324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 314324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 315324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver############################################################################ 316324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver# 317324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver# character streams for use in lexers 318324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver# CharStream 319324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver# \- ANTLRStringStream 320324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver# 321324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver############################################################################ 322324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 323324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 324324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruverclass ANTLRStringStream(CharStream): 325324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver """ 326324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver @brief CharStream that pull data from a unicode string. 327324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 328324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver A pretty quick CharStream that pulls all data from an array 329324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver directly. Every method call counts in the lexer. 330324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 331324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver """ 332324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 333324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 334324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver def __init__(self, data): 335324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver """ 336324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver @param data This should be a unicode string holding the data you want 337324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver to parse. If you pass in a byte string, the Lexer will choke on 338324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver non-ascii data. 339324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 340324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver """ 341324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 342324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver CharStream.__init__(self) 343324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 344324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver # The data being scanned 345324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver self.strdata = unicode(data) 346324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver self.data = [ord(c) for c in self.strdata] 347324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 348324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver # How many characters are actually in the buffer 349324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver self.n = len(data) 350324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 351324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver # 0..n-1 index into string of next char 352324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver self.p = 0 353324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 354324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver # line number 1..n within the input 355324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver self.line = 1 356324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 357324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver # The index of the character relative to the beginning of the 358324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver # line 0..n-1 359324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver self.charPositionInLine = 0 360324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 361324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver # A list of CharStreamState objects that tracks the stream state 362324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver # values line, charPositionInLine, and p that can change as you 363324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver # move through the input stream. Indexed from 0..markDepth-1. 364324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver self._markers = [ ] 365324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver self.lastMarker = None 366324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver self.markDepth = 0 367324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 368324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver # What is name or source of this char stream? 369324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver self.name = None 370324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 371324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 372324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver def reset(self): 373324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver """ 374324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver Reset the stream so that it's in the same state it was 375324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver when the object was created *except* the data array is not 376324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver touched. 377324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver """ 378324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 379324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver self.p = 0 380324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver self.line = 1 381324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver self.charPositionInLine = 0 382324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver self._markers = [ ] 383324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 384324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 385324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver def consume(self): 386324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver try: 387324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if self.data[self.p] == 10: # \n 388324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver self.line += 1 389324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver self.charPositionInLine = 0 390324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver else: 391324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver self.charPositionInLine += 1 392324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 393324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver self.p += 1 394324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 395324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver except IndexError: 396324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver # happend when we reached EOF and self.data[self.p] fails 397324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver # just do nothing 398324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver pass 399324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 400324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 401324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 402324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver def LA(self, i): 403324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if i == 0: 404324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return 0 # undefined 405324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 406324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if i < 0: 407324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver i += 1 # e.g., translate LA(-1) to use offset i=0; then data[p+0-1] 408324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 409324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver try: 410324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return self.data[self.p+i-1] 411324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver except IndexError: 412324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return EOF 413324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 414324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 415324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 416324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver def LT(self, i): 417324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if i == 0: 418324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return 0 # undefined 419324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 420324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if i < 0: 421324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver i += 1 # e.g., translate LA(-1) to use offset i=0; then data[p+0-1] 422324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 423324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver try: 424324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return self.strdata[self.p+i-1] 425324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver except IndexError: 426324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return EOF 427324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 428324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 429324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver def index(self): 430324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver """ 431324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver Return the current input symbol index 0..n where n indicates the 432324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver last symbol has been read. The index is the index of char to 433324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver be returned from LA(1). 434324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver """ 435324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 436324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return self.p 437324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 438324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 439324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver def size(self): 440324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return self.n 441324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 442324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 443324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver def mark(self): 444324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver state = (self.p, self.line, self.charPositionInLine) 445324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver try: 446324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver self._markers[self.markDepth] = state 447324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver except IndexError: 448324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver self._markers.append(state) 449324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver self.markDepth += 1 450324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 451324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver self.lastMarker = self.markDepth 452324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 453324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return self.lastMarker 454324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 455324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 456324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver def rewind(self, marker=None): 457324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if marker is None: 458324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver marker = self.lastMarker 459324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 460324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver p, line, charPositionInLine = self._markers[marker-1] 461324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 462324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver self.seek(p) 463324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver self.line = line 464324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver self.charPositionInLine = charPositionInLine 465324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver self.release(marker) 466324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 467324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 468324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver def release(self, marker=None): 469324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if marker is None: 470324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver marker = self.lastMarker 471324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 472324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver self.markDepth = marker-1 473324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 474324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 475324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver def seek(self, index): 476324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver """ 477324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver consume() ahead until p==index; can't just set p=index as we must 478324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver update line and charPositionInLine. 479324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver """ 480324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 481324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if index <= self.p: 482324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver self.p = index # just jump; don't update stream state (line, ...) 483324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return 484324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 485324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver # seek forward, consume until p hits index 486324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver while self.p < index: 487324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver self.consume() 488324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 489324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 490324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver def substring(self, start, stop): 491324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return self.strdata[start:stop+1] 492324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 493324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 494324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver def getLine(self): 495324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver """Using setter/getter methods is deprecated. Use o.line instead.""" 496324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return self.line 497324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 498324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 499324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver def getCharPositionInLine(self): 500324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver """ 501324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver Using setter/getter methods is deprecated. Use o.charPositionInLine 502324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver instead. 503324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver """ 504324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return self.charPositionInLine 505324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 506324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 507324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver def setLine(self, line): 508324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver """Using setter/getter methods is deprecated. Use o.line instead.""" 509324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver self.line = line 510324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 511324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 512324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver def setCharPositionInLine(self, pos): 513324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver """ 514324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver Using setter/getter methods is deprecated. Use o.charPositionInLine 515324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver instead. 516324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver """ 517324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver self.charPositionInLine = pos 518324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 519324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 520324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver def getSourceName(self): 521324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return self.name 522324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 523324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 524324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruverclass ANTLRFileStream(ANTLRStringStream): 525324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver """ 526324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver @brief CharStream that opens a file to read the data. 527324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 528324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver This is a char buffer stream that is loaded from a file 529324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver all at once when you construct the object. 530324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver """ 531324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 532324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver def __init__(self, fileName, encoding=None): 533324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver """ 534324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver @param fileName The path to the file to be opened. The file will be 535324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver opened with mode 'rb'. 536324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 537324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver @param encoding If you set the optional encoding argument, then the 538324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver data will be decoded on the fly. 539324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 540324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver """ 541324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 542324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver self.fileName = fileName 543324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 544324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver fp = codecs.open(fileName, 'rb', encoding) 545324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver try: 546324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver data = fp.read() 547324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver finally: 548324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver fp.close() 549324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 550324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver ANTLRStringStream.__init__(self, data) 551324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 552324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 553324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver def getSourceName(self): 554324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver """Deprecated, access o.fileName directly.""" 555324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 556324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return self.fileName 557324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 558324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 559324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruverclass ANTLRInputStream(ANTLRStringStream): 560324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver """ 561324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver @brief CharStream that reads data from a file-like object. 562324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 563324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver This is a char buffer stream that is loaded from a file like object 564324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver all at once when you construct the object. 565324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 566324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver All input is consumed from the file, but it is not closed. 567324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver """ 568324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 569324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver def __init__(self, file, encoding=None): 570324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver """ 571324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver @param file A file-like object holding your input. Only the read() 572324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver method must be implemented. 573324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 574324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver @param encoding If you set the optional encoding argument, then the 575324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver data will be decoded on the fly. 576324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 577324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver """ 578324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 579324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if encoding is not None: 580324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver # wrap input in a decoding reader 581324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver reader = codecs.lookup(encoding)[2] 582324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver file = reader(file) 583324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 584324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver data = file.read() 585324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 586324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver ANTLRStringStream.__init__(self, data) 587324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 588324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 589324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver# I guess the ANTLR prefix exists only to avoid a name clash with some Java 590324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver# mumbojumbo. A plain "StringStream" looks better to me, which should be 591324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver# the preferred name in Python. 592324c4644fee44b9898524c09511bd33c3f12e2dfBen GruverStringStream = ANTLRStringStream 593324c4644fee44b9898524c09511bd33c3f12e2dfBen GruverFileStream = ANTLRFileStream 594324c4644fee44b9898524c09511bd33c3f12e2dfBen GruverInputStream = ANTLRInputStream 595324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 596324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 597324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver############################################################################ 598324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver# 599324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver# Token streams 600324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver# TokenStream 601324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver# +- CommonTokenStream 602324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver# \- TokenRewriteStream 603324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver# 604324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver############################################################################ 605324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 606324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 607324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruverclass CommonTokenStream(TokenStream): 608324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver """ 609324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver @brief The most common stream of tokens 610324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 611324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver The most common stream of tokens is one where every token is buffered up 612324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver and tokens are prefiltered for a certain channel (the parser will only 613324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver see these tokens and cannot change the filter channel number during the 614324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver parse). 615324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver """ 616324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 617324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver def __init__(self, tokenSource=None, channel=DEFAULT_CHANNEL): 618324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver """ 619324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver @param tokenSource A TokenSource instance (usually a Lexer) to pull 620324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver the tokens from. 621324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 622324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver @param channel Skip tokens on any channel but this one; this is how we 623324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver skip whitespace... 624324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 625324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver """ 626324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 627324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver TokenStream.__init__(self) 628324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 629324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver self.tokenSource = tokenSource 630324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 631324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver # Record every single token pulled from the source so we can reproduce 632324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver # chunks of it later. 633324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver self.tokens = [] 634324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 635324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver # Map<tokentype, channel> to override some Tokens' channel numbers 636324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver self.channelOverrideMap = {} 637324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 638324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver # Set<tokentype>; discard any tokens with this type 639324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver self.discardSet = set() 640324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 641324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver # Skip tokens on any channel but this one; this is how we skip 642324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver # whitespace... 643324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver self.channel = channel 644324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 645324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver # By default, track all incoming tokens 646324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver self.discardOffChannelTokens = False 647324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 648324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver # The index into the tokens list of the current token (next token 649324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver # to consume). p==-1 indicates that the tokens list is empty 650324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver self.p = -1 651324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 652324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver # Remember last marked position 653324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver self.lastMarker = None 654324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 655324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver # how deep have we gone? 656324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver self._range = -1 657324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 658324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 659324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver def makeEOFToken(self): 660324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return self.tokenSource.makeEOFToken() 661324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 662324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 663324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver def setTokenSource(self, tokenSource): 664324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver """Reset this token stream by setting its token source.""" 665324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 666324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver self.tokenSource = tokenSource 667324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver self.tokens = [] 668324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver self.p = -1 669324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver self.channel = DEFAULT_CHANNEL 670324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 671324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 672324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver def reset(self): 673324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver self.p = 0 674324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver self.lastMarker = None 675324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 676324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 677324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver def fillBuffer(self): 678324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver """ 679324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver Load all tokens from the token source and put in tokens. 680324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver This is done upon first LT request because you might want to 681324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver set some token type / channel overrides before filling buffer. 682324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver """ 683324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 684324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 685324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver index = 0 686324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver t = self.tokenSource.nextToken() 687324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver while t is not None and t.type != EOF: 688324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver discard = False 689324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 690324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if self.discardSet is not None and t.type in self.discardSet: 691324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver discard = True 692324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 693324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver elif self.discardOffChannelTokens and t.channel != self.channel: 694324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver discard = True 695324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 696324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver # is there a channel override for token type? 697324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver try: 698324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver overrideChannel = self.channelOverrideMap[t.type] 699324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 700324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver except KeyError: 701324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver # no override for this type 702324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver pass 703324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 704324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver else: 705324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if overrideChannel == self.channel: 706324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver t.channel = overrideChannel 707324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver else: 708324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver discard = True 709324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 710324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if not discard: 711324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver t.index = index 712324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver self.tokens.append(t) 713324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver index += 1 714324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 715324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver t = self.tokenSource.nextToken() 716324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 717324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver # leave p pointing at first token on channel 718324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver self.p = 0 719324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver self.p = self.skipOffTokenChannels(self.p) 720324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 721324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 722324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver def consume(self): 723324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver """ 724324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver Move the input pointer to the next incoming token. The stream 725324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver must become active with LT(1) available. consume() simply 726324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver moves the input pointer so that LT(1) points at the next 727324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver input symbol. Consume at least one token. 728324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 729324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver Walk past any token not on the channel the parser is listening to. 730324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver """ 731324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 732324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if self.p < len(self.tokens): 733324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver self.p += 1 734324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 735324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver self.p = self.skipOffTokenChannels(self.p) # leave p on valid token 736324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 737324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 738324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver def skipOffTokenChannels(self, i): 739324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver """ 740324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver Given a starting index, return the index of the first on-channel 741324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token. 742324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver """ 743324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 744324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver try: 745324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver while self.tokens[i].channel != self.channel: 746324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver i += 1 747324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver except IndexError: 748324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver # hit the end of token stream 749324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver pass 750324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 751324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return i 752324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 753324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 754324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver def skipOffTokenChannelsReverse(self, i): 755324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver while i >= 0 and self.tokens[i].channel != self.channel: 756324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver i -= 1 757324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 758324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return i 759324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 760324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 761324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver def setTokenTypeChannel(self, ttype, channel): 762324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver """ 763324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver A simple filter mechanism whereby you can tell this token stream 764324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver to force all tokens of type ttype to be on channel. For example, 765324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver when interpreting, we cannot exec actions so we need to tell 766324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver the stream to force all WS and NEWLINE to be a different, ignored 767324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver channel. 768324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver """ 769324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 770324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver self.channelOverrideMap[ttype] = channel 771324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 772324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 773324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver def discardTokenType(self, ttype): 774324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver self.discardSet.add(ttype) 775324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 776324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 777324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver def getTokens(self, start=None, stop=None, types=None): 778324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver """ 779324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver Given a start and stop index, return a list of all tokens in 780324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver the token type set. Return None if no tokens were found. This 781324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver method looks at both on and off channel tokens. 782324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver """ 783324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 784324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if self.p == -1: 785324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver self.fillBuffer() 786324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 787324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if stop is None or stop >= len(self.tokens): 788324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver stop = len(self.tokens) - 1 789324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 790324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if start is None or stop < 0: 791324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver start = 0 792324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 793324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if start > stop: 794324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return None 795324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 796324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if isinstance(types, (int, long)): 797324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver # called with a single type, wrap into set 798324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver types = set([types]) 799324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 800324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver filteredTokens = [ 801324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token for token in self.tokens[start:stop] 802324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if types is None or token.type in types 803324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver ] 804324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 805324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if len(filteredTokens) == 0: 806324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return None 807324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 808324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return filteredTokens 809324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 810324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 811324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver def LT(self, k): 812324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver """ 813324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver Get the ith token from the current position 1..n where k=1 is the 814324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver first symbol of lookahead. 815324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver """ 816324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 817324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if self.p == -1: 818324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver self.fillBuffer() 819324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 820324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if k == 0: 821324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return None 822324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 823324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if k < 0: 824324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return self.LB(-k) 825324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 826324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver i = self.p 827324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver n = 1 828324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver # find k good tokens 829324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver while n < k: 830324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver # skip off-channel tokens 831324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver i = self.skipOffTokenChannels(i+1) # leave p on valid token 832324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver n += 1 833324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 834324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if i > self._range: 835324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver self._range = i 836324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 837324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver try: 838324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return self.tokens[i] 839324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver except IndexError: 840324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return self.makeEOFToken() 841324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 842324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 843324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver def LB(self, k): 844324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver """Look backwards k tokens on-channel tokens""" 845324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 846324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if self.p == -1: 847324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver self.fillBuffer() 848324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 849324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if k == 0: 850324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return None 851324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 852324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if self.p - k < 0: 853324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return None 854324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 855324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver i = self.p 856324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver n = 1 857324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver # find k good tokens looking backwards 858324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver while n <= k: 859324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver # skip off-channel tokens 860324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver i = self.skipOffTokenChannelsReverse(i-1) # leave p on valid token 861324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver n += 1 862324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 863324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if i < 0: 864324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return None 865324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 866324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return self.tokens[i] 867324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 868324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 869324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver def get(self, i): 870324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver """ 871324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver Return absolute token i; ignore which channel the tokens are on; 872324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver that is, count all tokens not just on-channel tokens. 873324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver """ 874324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 875324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return self.tokens[i] 876324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 877324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 878324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver def slice(self, start, stop): 879324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if self.p == -1: 880324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver self.fillBuffer() 881324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 882324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if start < 0 or stop < 0: 883324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return None 884324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 885324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return self.tokens[start:stop+1] 886324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 887324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 888324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver def LA(self, i): 889324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return self.LT(i).type 890324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 891324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 892324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver def mark(self): 893324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver self.lastMarker = self.index() 894324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return self.lastMarker 895324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 896324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 897324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver def release(self, marker=None): 898324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver # no resources to release 899324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver pass 900324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 901324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 902324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver def size(self): 903324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return len(self.tokens) 904324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 905324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 906324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver def range(self): 907324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return self._range 908324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 909324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 910324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver def index(self): 911324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return self.p 912324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 913324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 914324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver def rewind(self, marker=None): 915324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if marker is None: 916324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver marker = self.lastMarker 917324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 918324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver self.seek(marker) 919324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 920324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 921324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver def seek(self, index): 922324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver self.p = index 923324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 924324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 925324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver def getTokenSource(self): 926324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return self.tokenSource 927324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 928324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 929324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver def getSourceName(self): 930324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return self.tokenSource.getSourceName() 931324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 932324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 933324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver def toString(self, start=None, stop=None): 934324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if self.p == -1: 935324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver self.fillBuffer() 936324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 937324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if start is None: 938324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver start = 0 939324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver elif not isinstance(start, int): 940324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver start = start.index 941324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 942324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if stop is None: 943324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver stop = len(self.tokens) - 1 944324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver elif not isinstance(stop, int): 945324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver stop = stop.index 946324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 947324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if stop >= len(self.tokens): 948324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver stop = len(self.tokens) - 1 949324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 950324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return ''.join([t.text for t in self.tokens[start:stop+1]]) 951324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 952324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 953324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruverclass RewriteOperation(object): 954324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver """@brief Internal helper class.""" 955324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 956324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver def __init__(self, stream, index, text): 957324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver self.stream = stream 958324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 959324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver # What index into rewrites List are we? 960324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver self.instructionIndex = None 961324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 962324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver # Token buffer index. 963324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver self.index = index 964324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver self.text = text 965324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 966324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver def execute(self, buf): 967324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver """Execute the rewrite operation by possibly adding to the buffer. 968324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver Return the index of the next token to operate on. 969324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver """ 970324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 971324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return self.index 972324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 973324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver def toString(self): 974324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver opName = self.__class__.__name__ 975324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return '<%s@%d:"%s">' % ( 976324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver opName, self.index, self.text) 977324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 978324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver __str__ = toString 979324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver __repr__ = toString 980324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 981324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 982324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruverclass InsertBeforeOp(RewriteOperation): 983324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver """@brief Internal helper class.""" 984324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 985324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver def execute(self, buf): 986324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver buf.write(self.text) 987324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if self.stream.tokens[self.index].type != EOF: 988324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver buf.write(self.stream.tokens[self.index].text) 989324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return self.index + 1 990324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 991324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 992324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruverclass ReplaceOp(RewriteOperation): 993324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver """ 994324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver @brief Internal helper class. 995324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 996324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver I'm going to try replacing range from x..y with (y-x)+1 ReplaceOp 997324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver instructions. 998324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver """ 999324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 1000324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver def __init__(self, stream, first, last, text): 1001324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver RewriteOperation.__init__(self, stream, first, text) 1002324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver self.lastIndex = last 1003324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 1004324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 1005324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver def execute(self, buf): 1006324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if self.text is not None: 1007324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver buf.write(self.text) 1008324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 1009324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return self.lastIndex + 1 1010324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 1011324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 1012324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver def toString(self): 1013324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if self.text is None: 1014324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return '<DeleteOp@%d..%d>' % (self.index, self.lastIndex) 1015324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 1016324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return '<ReplaceOp@%d..%d:"%s">' % ( 1017324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver self.index, self.lastIndex, self.text) 1018324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 1019324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver __str__ = toString 1020324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver __repr__ = toString 1021324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 1022324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 1023324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruverclass TokenRewriteStream(CommonTokenStream): 1024324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver """@brief CommonTokenStream that can be modified. 1025324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 1026324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver Useful for dumping out the input stream after doing some 1027324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver augmentation or other manipulations. 1028324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 1029324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver You can insert stuff, replace, and delete chunks. Note that the 1030324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver operations are done lazily--only if you convert the buffer to a 1031324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver String. This is very efficient because you are not moving data around 1032324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver all the time. As the buffer of tokens is converted to strings, the 1033324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver toString() method(s) check to see if there is an operation at the 1034324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver current index. If so, the operation is done and then normal String 1035324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver rendering continues on the buffer. This is like having multiple Turing 1036324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver machine instruction streams (programs) operating on a single input tape. :) 1037324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 1038324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver Since the operations are done lazily at toString-time, operations do not 1039324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver screw up the token index values. That is, an insert operation at token 1040324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver index i does not change the index values for tokens i+1..n-1. 1041324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 1042324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver Because operations never actually alter the buffer, you may always get 1043324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver the original token stream back without undoing anything. Since 1044324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver the instructions are queued up, you can easily simulate transactions and 1045324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver roll back any changes if there is an error just by removing instructions. 1046324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver For example, 1047324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 1048324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver CharStream input = new ANTLRFileStream("input"); 1049324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver TLexer lex = new TLexer(input); 1050324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver TokenRewriteStream tokens = new TokenRewriteStream(lex); 1051324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver T parser = new T(tokens); 1052324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver parser.startRule(); 1053324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 1054324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver Then in the rules, you can execute 1055324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver Token t,u; 1056324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver ... 1057324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver input.insertAfter(t, "text to put after t");} 1058324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver input.insertAfter(u, "text after u");} 1059324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver System.out.println(tokens.toString()); 1060324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 1061324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver Actually, you have to cast the 'input' to a TokenRewriteStream. :( 1062324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 1063324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver You can also have multiple "instruction streams" and get multiple 1064324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver rewrites from a single pass over the input. Just name the instruction 1065324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver streams and use that name again when printing the buffer. This could be 1066324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver useful for generating a C file and also its header file--all from the 1067324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver same buffer: 1068324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 1069324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver tokens.insertAfter("pass1", t, "text to put after t");} 1070324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver tokens.insertAfter("pass2", u, "text after u");} 1071324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver System.out.println(tokens.toString("pass1")); 1072324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver System.out.println(tokens.toString("pass2")); 1073324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 1074324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver If you don't use named rewrite streams, a "default" stream is used as 1075324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver the first example shows. 1076324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver """ 1077324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 1078324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver DEFAULT_PROGRAM_NAME = "default" 1079324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver MIN_TOKEN_INDEX = 0 1080324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 1081324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver def __init__(self, tokenSource=None, channel=DEFAULT_CHANNEL): 1082324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver CommonTokenStream.__init__(self, tokenSource, channel) 1083324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 1084324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver # You may have multiple, named streams of rewrite operations. 1085324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver # I'm calling these things "programs." 1086324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver # Maps String (name) -> rewrite (List) 1087324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver self.programs = {} 1088324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver self.programs[self.DEFAULT_PROGRAM_NAME] = [] 1089324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 1090324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver # Map String (program name) -> Integer index 1091324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver self.lastRewriteTokenIndexes = {} 1092324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 1093324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 1094324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver def rollback(self, *args): 1095324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver """ 1096324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver Rollback the instruction stream for a program so that 1097324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver the indicated instruction (via instructionIndex) is no 1098324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver longer in the stream. UNTESTED! 1099324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver """ 1100324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 1101324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if len(args) == 2: 1102324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver programName = args[0] 1103324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver instructionIndex = args[1] 1104324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver elif len(args) == 1: 1105324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver programName = self.DEFAULT_PROGRAM_NAME 1106324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver instructionIndex = args[0] 1107324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver else: 1108324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver raise TypeError("Invalid arguments") 1109324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 1110324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver p = self.programs.get(programName, None) 1111324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if p is not None: 1112324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver self.programs[programName] = ( 1113324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver p[self.MIN_TOKEN_INDEX:instructionIndex]) 1114324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 1115324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 1116324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver def deleteProgram(self, programName=DEFAULT_PROGRAM_NAME): 1117324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver """Reset the program so that no instructions exist""" 1118324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 1119324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver self.rollback(programName, self.MIN_TOKEN_INDEX) 1120324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 1121324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 1122324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver def insertAfter(self, *args): 1123324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if len(args) == 2: 1124324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver programName = self.DEFAULT_PROGRAM_NAME 1125324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver index = args[0] 1126324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver text = args[1] 1127324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 1128324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver elif len(args) == 3: 1129324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver programName = args[0] 1130324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver index = args[1] 1131324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver text = args[2] 1132324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 1133324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver else: 1134324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver raise TypeError("Invalid arguments") 1135324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 1136324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if isinstance(index, Token): 1137324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver # index is a Token, grap the stream index from it 1138324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver index = index.index 1139324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 1140324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver # to insert after, just insert before next index (even if past end) 1141324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver self.insertBefore(programName, index+1, text) 1142324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 1143324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 1144324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver def insertBefore(self, *args): 1145324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if len(args) == 2: 1146324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver programName = self.DEFAULT_PROGRAM_NAME 1147324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver index = args[0] 1148324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver text = args[1] 1149324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 1150324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver elif len(args) == 3: 1151324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver programName = args[0] 1152324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver index = args[1] 1153324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver text = args[2] 1154324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 1155324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver else: 1156324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver raise TypeError("Invalid arguments") 1157324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 1158324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if isinstance(index, Token): 1159324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver # index is a Token, grap the stream index from it 1160324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver index = index.index 1161324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 1162324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver op = InsertBeforeOp(self, index, text) 1163324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver rewrites = self.getProgram(programName) 1164324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver op.instructionIndex = len(rewrites) 1165324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver rewrites.append(op) 1166324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 1167324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 1168324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver def replace(self, *args): 1169324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if len(args) == 2: 1170324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver programName = self.DEFAULT_PROGRAM_NAME 1171324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver first = args[0] 1172324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver last = args[0] 1173324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver text = args[1] 1174324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 1175324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver elif len(args) == 3: 1176324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver programName = self.DEFAULT_PROGRAM_NAME 1177324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver first = args[0] 1178324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver last = args[1] 1179324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver text = args[2] 1180324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 1181324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver elif len(args) == 4: 1182324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver programName = args[0] 1183324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver first = args[1] 1184324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver last = args[2] 1185324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver text = args[3] 1186324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 1187324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver else: 1188324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver raise TypeError("Invalid arguments") 1189324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 1190324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if isinstance(first, Token): 1191324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver # first is a Token, grap the stream index from it 1192324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver first = first.index 1193324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 1194324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if isinstance(last, Token): 1195324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver # last is a Token, grap the stream index from it 1196324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver last = last.index 1197324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 1198324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if first > last or first < 0 or last < 0 or last >= len(self.tokens): 1199324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver raise ValueError( 1200324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver "replace: range invalid: %d..%d (size=%d)" 1201324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver % (first, last, len(self.tokens))) 1202324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 1203324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver op = ReplaceOp(self, first, last, text) 1204324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver rewrites = self.getProgram(programName) 1205324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver op.instructionIndex = len(rewrites) 1206324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver rewrites.append(op) 1207324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 1208324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 1209324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver def delete(self, *args): 1210324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver self.replace(*(list(args) + [None])) 1211324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 1212324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 1213324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver def getLastRewriteTokenIndex(self, programName=DEFAULT_PROGRAM_NAME): 1214324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return self.lastRewriteTokenIndexes.get(programName, -1) 1215324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 1216324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 1217324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver def setLastRewriteTokenIndex(self, programName, i): 1218324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver self.lastRewriteTokenIndexes[programName] = i 1219324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 1220324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 1221324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver def getProgram(self, name): 1222324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver p = self.programs.get(name, None) 1223324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if p is None: 1224324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver p = self.initializeProgram(name) 1225324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 1226324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return p 1227324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 1228324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 1229324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver def initializeProgram(self, name): 1230324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver p = [] 1231324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver self.programs[name] = p 1232324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return p 1233324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 1234324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 1235324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver def toOriginalString(self, start=None, end=None): 1236324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if self.p == -1: 1237324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver self.fillBuffer() 1238324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 1239324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if start is None: 1240324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver start = self.MIN_TOKEN_INDEX 1241324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if end is None: 1242324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver end = self.size() - 1 1243324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 1244324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver buf = StringIO() 1245324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver i = start 1246324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver while i >= self.MIN_TOKEN_INDEX and i <= end and i < len(self.tokens): 1247324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if self.get(i).type != EOF: 1248324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver buf.write(self.get(i).text) 1249324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver i += 1 1250324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 1251324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return buf.getvalue() 1252324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 1253324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 1254324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver def toString(self, *args): 1255324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if self.p == -1: 1256324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver self.fillBuffer() 1257324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 1258324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if len(args) == 0: 1259324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver programName = self.DEFAULT_PROGRAM_NAME 1260324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver start = self.MIN_TOKEN_INDEX 1261324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver end = self.size() - 1 1262324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 1263324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver elif len(args) == 1: 1264324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver programName = args[0] 1265324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver start = self.MIN_TOKEN_INDEX 1266324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver end = self.size() - 1 1267324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 1268324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver elif len(args) == 2: 1269324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver programName = self.DEFAULT_PROGRAM_NAME 1270324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver start = args[0] 1271324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver end = args[1] 1272324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 1273324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if start is None: 1274324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver start = self.MIN_TOKEN_INDEX 1275324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver elif not isinstance(start, int): 1276324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver start = start.index 1277324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 1278324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if end is None: 1279324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver end = len(self.tokens) - 1 1280324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver elif not isinstance(end, int): 1281324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver end = end.index 1282324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 1283324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver # ensure start/end are in range 1284324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if end >= len(self.tokens): 1285324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver end = len(self.tokens) - 1 1286324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 1287324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if start < 0: 1288324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver start = 0 1289324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 1290324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver rewrites = self.programs.get(programName) 1291324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if rewrites is None or len(rewrites) == 0: 1292324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver # no instructions to execute 1293324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return self.toOriginalString(start, end) 1294324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 1295324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver buf = StringIO() 1296324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 1297324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver # First, optimize instruction stream 1298324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver indexToOp = self.reduceToSingleOperationPerIndex(rewrites) 1299324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 1300324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver # Walk buffer, executing instructions and emitting tokens 1301324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver i = start 1302324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver while i <= end and i < len(self.tokens): 1303324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver op = indexToOp.get(i) 1304324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver # remove so any left have index size-1 1305324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver try: 1306324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver del indexToOp[i] 1307324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver except KeyError: 1308324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver pass 1309324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 1310324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver t = self.tokens[i] 1311324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if op is None: 1312324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver # no operation at that index, just dump token 1313324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if t.type != EOF: 1314324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver buf.write(t.text) 1315324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver i += 1 # move to next token 1316324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 1317324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver else: 1318324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver i = op.execute(buf) # execute operation and skip 1319324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 1320324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver # include stuff after end if it's last index in buffer 1321324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver # So, if they did an insertAfter(lastValidIndex, "foo"), include 1322324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver # foo if end==lastValidIndex. 1323324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if end == len(self.tokens) - 1: 1324324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver # Scan any remaining operations after last token 1325324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver # should be included (they will be inserts). 1326324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver for i in sorted(indexToOp.keys()): 1327324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver op = indexToOp[i] 1328324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if op.index >= len(self.tokens)-1: 1329324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver buf.write(op.text) 1330324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 1331324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return buf.getvalue() 1332324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 1333324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver __str__ = toString 1334324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 1335324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 1336324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver def reduceToSingleOperationPerIndex(self, rewrites): 1337324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver """ 1338324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver We need to combine operations and report invalid operations (like 1339324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver overlapping replaces that are not completed nested). Inserts to 1340324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver same index need to be combined etc... Here are the cases: 1341324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 1342324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver I.i.u I.j.v leave alone, nonoverlapping 1343324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver I.i.u I.i.v combine: Iivu 1344324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 1345324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver R.i-j.u R.x-y.v | i-j in x-y delete first R 1346324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver R.i-j.u R.i-j.v delete first R 1347324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver R.i-j.u R.x-y.v | x-y in i-j ERROR 1348324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver R.i-j.u R.x-y.v | boundaries overlap ERROR 1349324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 1350324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver Delete special case of replace (text==null): 1351324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver D.i-j.u D.x-y.v | boundaries overlapcombine to 1352324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver max(min)..max(right) 1353324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 1354324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver I.i.u R.x-y.v | i in (x+1)-ydelete I (since 1355324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver insert before we're not deleting 1356324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver i) 1357324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver I.i.u R.x-y.v | i not in (x+1)-yleave alone, 1358324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver nonoverlapping 1359324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 1360324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver R.x-y.v I.i.u | i in x-y ERROR 1361324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver R.x-y.v I.x.u R.x-y.uv (combine, delete I) 1362324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver R.x-y.v I.i.u | i not in x-y leave alone, nonoverlapping 1363324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 1364324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver I.i.u = insert u before op @ index i 1365324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver R.x-y.u = replace x-y indexed tokens with u 1366324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 1367324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver First we need to examine replaces. For any replace op: 1368324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 1369324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 1. wipe out any insertions before op within that range. 1370324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 2. Drop any replace op before that is contained completely within 1371324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver that range. 1372324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 3. Throw exception upon boundary overlap with any previous replace. 1373324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 1374324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver Then we can deal with inserts: 1375324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 1376324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 1. for any inserts to same index, combine even if not adjacent. 1377324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 2. for any prior replace with same left boundary, combine this 1378324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver insert with replace and delete this replace. 1379324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 3. throw exception if index in same range as previous replace 1380324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 1381324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver Don't actually delete; make op null in list. Easier to walk list. 1382324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver Later we can throw as we add to index -> op map. 1383324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 1384324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver Note that I.2 R.2-2 will wipe out I.2 even though, technically, the 1385324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver inserted stuff would be before the replace range. But, if you 1386324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver add tokens in front of a method body '{' and then delete the method 1387324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver body, I think the stuff before the '{' you added should disappear too. 1388324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 1389324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver Return a map from token index to operation. 1390324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver """ 1391324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 1392324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver # WALK REPLACES 1393324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver for i, rop in enumerate(rewrites): 1394324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if rop is None: 1395324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver continue 1396324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 1397324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if not isinstance(rop, ReplaceOp): 1398324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver continue 1399324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 1400324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver # Wipe prior inserts within range 1401324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver for j, iop in self.getKindOfOps(rewrites, InsertBeforeOp, i): 1402324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if iop.index == rop.index: 1403324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver # E.g., insert before 2, delete 2..2; update replace 1404324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver # text to include insert before, kill insert 1405324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver rewrites[iop.instructionIndex] = None 1406324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver rop.text = self.catOpText(iop.text, rop.text) 1407324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 1408324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver elif iop.index > rop.index and iop.index <= rop.lastIndex: 1409324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver # delete insert as it's a no-op. 1410324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver rewrites[j] = None 1411324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 1412324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver # Drop any prior replaces contained within 1413324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver for j, prevRop in self.getKindOfOps(rewrites, ReplaceOp, i): 1414324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if (prevRop.index >= rop.index 1415324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver and prevRop.lastIndex <= rop.lastIndex): 1416324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver # delete replace as it's a no-op. 1417324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver rewrites[j] = None 1418324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver continue 1419324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 1420324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver # throw exception unless disjoint or identical 1421324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver disjoint = (prevRop.lastIndex < rop.index 1422324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver or prevRop.index > rop.lastIndex) 1423324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver same = (prevRop.index == rop.index 1424324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver and prevRop.lastIndex == rop.lastIndex) 1425324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 1426324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver # Delete special case of replace (text==null): 1427324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver # D.i-j.u D.x-y.v| boundaries overlapcombine to 1428324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver # max(min)..max(right) 1429324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if prevRop.text is None and rop.text is None and not disjoint: 1430324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver # kill first delete 1431324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver rewrites[prevRop.instructionIndex] = None 1432324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 1433324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver rop.index = min(prevRop.index, rop.index) 1434324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver rop.lastIndex = max(prevRop.lastIndex, rop.lastIndex) 1435324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 1436324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver elif not disjoint and not same: 1437324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver raise ValueError( 1438324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver "replace op boundaries of %s overlap with previous %s" 1439324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver % (rop, prevRop)) 1440324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 1441324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver # WALK INSERTS 1442324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver for i, iop in enumerate(rewrites): 1443324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if iop is None: 1444324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver continue 1445324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 1446324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if not isinstance(iop, InsertBeforeOp): 1447324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver continue 1448324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 1449324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver # combine current insert with prior if any at same index 1450324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver for j, prevIop in self.getKindOfOps(rewrites, InsertBeforeOp, i): 1451324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if prevIop.index == iop.index: # combine objects 1452324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver # convert to strings...we're in process of toString'ing 1453324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver # whole token buffer so no lazy eval issue with any 1454324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver # templates 1455324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver iop.text = self.catOpText(iop.text, prevIop.text) 1456324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver # delete redundant prior insert 1457324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver rewrites[j] = None 1458324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 1459324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver # look for replaces where iop.index is in range; error 1460324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver for j, rop in self.getKindOfOps(rewrites, ReplaceOp, i): 1461324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if iop.index == rop.index: 1462324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver rop.text = self.catOpText(iop.text, rop.text) 1463324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver # delete current insert 1464324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver rewrites[i] = None 1465324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver continue 1466324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 1467324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if iop.index >= rop.index and iop.index <= rop.lastIndex: 1468324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver raise ValueError( 1469324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver "insert op %s within boundaries of previous %s" 1470324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver % (iop, rop)) 1471324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 1472324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver m = {} 1473324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver for i, op in enumerate(rewrites): 1474324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if op is None: 1475324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver # ignore deleted ops 1476324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver continue 1477324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 1478324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver assert op.index not in m, "should only be one op per index" 1479324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver m[op.index] = op 1480324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 1481324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return m 1482324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 1483324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 1484324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver def catOpText(self, a, b): 1485324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver x = "" 1486324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver y = "" 1487324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if a is not None: 1488324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver x = a 1489324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if b is not None: 1490324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver y = b 1491324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return x + y 1492324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 1493324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 1494324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver def getKindOfOps(self, rewrites, kind, before=None): 1495324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver """Get all operations before an index of a particular kind.""" 1496324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 1497324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if before is None: 1498324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver before = len(rewrites) 1499324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver elif before > len(rewrites): 1500324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver before = len(rewrites) 1501324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 1502324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver for i, op in enumerate(rewrites[:before]): 1503324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if op is None: 1504324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver # ignore deleted 1505324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver continue 1506324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if op.__class__ == kind: 1507324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver yield i, op 1508324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 1509324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 1510324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver def toDebugString(self, start=None, end=None): 1511324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if start is None: 1512324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver start = self.MIN_TOKEN_INDEX 1513324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if end is None: 1514324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver end = self.size() - 1 1515324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 1516324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver buf = StringIO() 1517324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver i = start 1518324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver while i >= self.MIN_TOKEN_INDEX and i <= end and i < len(self.tokens): 1519324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver buf.write(self.get(i)) 1520324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver i += 1 1521324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 1522324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return buf.getvalue() 1523