1324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver/* 2324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * [The "BSD licence"] 3324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * Copyright (c) 2005-2008 Terence Parr 4324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * All rights reserved. 5324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * 6324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * Conversion to C#: 7324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * Copyright (c) 2008-2009 Sam Harwell, Pixel Mine, Inc. 8324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * All rights reserved. 9324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * 10324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * Redistribution and use in source and binary forms, with or without 11324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * modification, are permitted provided that the following conditions 12324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * are met: 13324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * 1. Redistributions of source code must retain the above copyright 14324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * notice, this list of conditions and the following disclaimer. 15324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * 2. Redistributions in binary form must reproduce the above copyright 16324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * notice, this list of conditions and the following disclaimer in the 17324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * documentation and/or other materials provided with the distribution. 18324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * 3. The name of the author may not be used to endorse or promote products 19324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * derived from this software without specific prior written permission. 20324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * 21324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 22324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 23324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 24324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 25324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 26324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 27324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 28324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 29324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 30324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 31324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver */ 32324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 33324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruvernamespace Antlr.Runtime { 34324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver using System.Collections.Generic; 35324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver using CLSCompliant = System.CLSCompliantAttribute; 36324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver using IndexOutOfRangeException = System.IndexOutOfRangeException; 37324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver using StringBuilder = System.Text.StringBuilder; 38324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 39324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver /** Buffer all input tokens but do on-demand fetching of new tokens from 40324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * lexer. Useful when the parser or lexer has to set context/mode info before 41324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * proper lexing of future tokens. The ST template parser needs this, 42324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * for example, because it has to constantly flip back and forth between 43324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * inside/output templates. E.g., <names:{hi, <it>}> has to parse names 44324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * as part of an expression but "hi, <it>" as a nested template. 45324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * 46324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * You can't use this stream if you pass whitespace or other off-channel 47324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * tokens to the parser. The stream can't ignore off-channel tokens. 48324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * (UnbufferedTokenStream is the same way.) 49324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * 50324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * This is not a subclass of UnbufferedTokenStream because I don't want 51324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * to confuse small moving window of tokens it uses for the full buffer. 52324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver */ 53324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver [System.Serializable] 54324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver public class BufferedTokenStream : ITokenStream, ITokenStreamInformation { 55324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver private ITokenSource _tokenSource; 56324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 57324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver /** Record every single token pulled from the source so we can reproduce 58324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * chunks of it later. The buffer in LookaheadStream overlaps sometimes 59324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * as its moving window moves through the input. This list captures 60324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * everything so we can access complete input text. 61324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver */ 62324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver [CLSCompliant(false)] 63324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver protected List<IToken> _tokens = new List<IToken>(100); 64324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 65324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver /** Track the last mark() call result value for use in rewind(). */ 66324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver private int _lastMarker; 67324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 68324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver /** The index into the tokens list of the current token (next token 69324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * to consume). tokens[p] should be LT(1). p=-1 indicates need 70324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * to initialize with first token. The ctor doesn't get a token. 71324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * First call to LT(1) or whatever gets the first token and sets p=0; 72324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver */ 73324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver [CLSCompliant(false)] 74324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver protected int _p = -1; 75324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 76324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver public BufferedTokenStream() { 77324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 78324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 79324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver public BufferedTokenStream(ITokenSource tokenSource) { 80324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver this._tokenSource = tokenSource; 81324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 82324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 83324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver public virtual ITokenSource TokenSource { 84324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver get { 85324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return _tokenSource; 86324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 87324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver set { 88324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver this._tokenSource = value; 89324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver _tokens.Clear(); 90324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver _p = -1; 91324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 92324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 93324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 94324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver public virtual int Index { 95324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver get { 96324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return _p; 97324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 98324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 99324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 100324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver /// <summary> 101324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver /// How deep have we gone? 102324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver /// </summary> 103324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver public virtual int Range { 104324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver get; 105324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver protected set; 106324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 107324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 108324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver public virtual int Count { 109324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver get { 110324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return _tokens.Count; 111324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 112324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 113324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 114324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver public virtual string SourceName { 115324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver get { 116324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return _tokenSource.SourceName; 117324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 118324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 119324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 120324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver public virtual IToken LastToken { 121324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver get { 122324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return LB(1); 123324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 124324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 125324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 126324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver public virtual IToken LastRealToken { 127324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver get { 128324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver int i = 0; 129324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver IToken token; 130324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver do { 131324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver i++; 132324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token = LB(i); 133324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } while (token != null && token.Line <= 0); 134324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 135324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return token; 136324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 137324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 138324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 139324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver public virtual int MaxLookBehind { 140324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver get { 141324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return int.MaxValue; 142324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 143324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 144324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 145324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver public virtual int Mark() { 146324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if (_p == -1) 147324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver Setup(); 148324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver _lastMarker = Index; 149324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return _lastMarker; 150324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 151324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 152324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver public virtual void Release(int marker) { 153324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver // no resources to release 154324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 155324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 156324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver public virtual void Rewind(int marker) { 157324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver Seek(marker); 158324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 159324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 160324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver public virtual void Rewind() { 161324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver Seek(_lastMarker); 162324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 163324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 164324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver public virtual void Reset() { 165324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver _p = 0; 166324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver _lastMarker = 0; 167324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 168324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 169324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver public virtual void Seek(int index) { 170324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver _p = index; 171324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 172324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 173324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver /** Move the input pointer to the next incoming token. The stream 174324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * must become active with LT(1) available. consume() simply 175324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * moves the input pointer so that LT(1) points at the next 176324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * input symbol. Consume at least one token. 177324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * 178324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * Walk past any token not on the channel the parser is listening to. 179324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver */ 180324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver public virtual void Consume() { 181324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if (_p == -1) 182324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver Setup(); 183324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver _p++; 184324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver Sync(_p); 185324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 186324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 187324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver /** Make sure index i in tokens has a token. */ 188324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver protected virtual void Sync(int i) { 189324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver int n = i - _tokens.Count + 1; // how many more elements we need? 190324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if (n > 0) 191324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver Fetch(n); 192324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 193324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 194324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver /** add n elements to buffer */ 195324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver protected virtual void Fetch(int n) { 196324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver for (int i = 0; i < n; i++) { 197324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver IToken t = TokenSource.NextToken(); 198324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver t.TokenIndex = _tokens.Count; 199324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver _tokens.Add(t); 200324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if (t.Type == CharStreamConstants.EndOfFile) 201324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver break; 202324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 203324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 204324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 205324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver public virtual IToken Get(int i) { 206324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if (i < 0 || i >= _tokens.Count) { 207324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver throw new IndexOutOfRangeException("token index " + i + " out of range 0.." + (_tokens.Count - 1)); 208324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 209324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return _tokens[i]; 210324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 211324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 212324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver#if false // why is this different from GetTokens(start, count) ? 213324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver /// <summary> 214324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver /// Get all tokens from start..(start+count-1) inclusively 215324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver /// </summary> 216324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver public virtual List<IToken> Get(int start, int count) 217324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver { 218324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if (start < 0) 219324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver throw new ArgumentOutOfRangeException("start"); 220324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if (count < 0) 221324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver throw new ArgumentOutOfRangeException("count"); 222324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if (start + count >= _tokens.Count) 223324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver throw new ArgumentException(); 224324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 225324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if (_p == -1) 226324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver Setup(); 227324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 228324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver List<IToken> subset = new List<IToken>(count); 229324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver for (int i = 0; i < count; i++) 230324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver { 231324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver IToken token = _tokens[i]; 232324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if (token.Type == TokenTypes.EndOfFile) 233324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver break; 234324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 235324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver subset.Add(token); 236324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 237324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 238324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return subset; 239324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 240324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver#endif 241324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 242324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver public virtual int LA(int i) { 243324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return LT(i).Type; 244324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 245324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 246324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver protected virtual IToken LB(int k) { 247324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if ((_p - k) < 0) 248324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return null; 249324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 250324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return _tokens[_p - k]; 251324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 252324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 253324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver public virtual IToken LT(int k) { 254324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if (_p == -1) 255324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver Setup(); 256324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if (k == 0) 257324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return null; 258324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if (k < 0) 259324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return LB(-k); 260324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 261324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver int i = _p + k - 1; 262324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver Sync(i); 263324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if (i >= _tokens.Count) { 264324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver // EOF must be last token 265324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return _tokens[_tokens.Count - 1]; 266324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 267324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 268324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if (i > Range) 269324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver Range = i; 270324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 271324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return _tokens[_p + k - 1]; 272324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 273324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 274324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver protected virtual void Setup() { 275324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver Sync(0); 276324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver _p = 0; 277324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 278324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 279324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver public virtual List<IToken> GetTokens() { 280324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return _tokens; 281324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 282324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 283324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver public virtual List<IToken> GetTokens(int start, int stop) { 284324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return GetTokens(start, stop, default(BitSet)); 285324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 286324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 287324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver /** Given a start and stop index, return a List of all tokens in 288324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * the token type BitSet. Return null if no tokens were found. This 289324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * method looks at both on and off channel tokens. 290324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver */ 291324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver public virtual List<IToken> GetTokens(int start, int stop, BitSet types) { 292324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if (_p == -1) 293324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver Setup(); 294324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if (stop >= _tokens.Count) 295324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver stop = _tokens.Count - 1; 296324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if (start < 0) 297324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver start = 0; 298324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if (start > stop) 299324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return null; 300324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 301324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver // list = tokens[start:stop]:{Token t, t.getType() in types} 302324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver List<IToken> filteredTokens = new List<IToken>(); 303324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver for (int i = start; i <= stop; i++) { 304324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver IToken t = _tokens[i]; 305324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if (types == null || types.Member(t.Type)) { 306324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver filteredTokens.Add(t); 307324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 308324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 309324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if (filteredTokens.Count == 0) { 310324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver filteredTokens = null; 311324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 312324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return filteredTokens; 313324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 314324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 315324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver public virtual List<IToken> GetTokens(int start, int stop, IEnumerable<int> types) { 316324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return GetTokens(start, stop, new BitSet(types)); 317324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 318324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 319324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver public virtual List<IToken> GetTokens(int start, int stop, int ttype) { 320324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return GetTokens(start, stop, BitSet.Of(ttype)); 321324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 322324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 323324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver public override string ToString() { 324324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if (_p == -1) 325324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver Setup(); 326324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 327324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver Fill(); 328324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return ToString(0, _tokens.Count - 1); 329324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 330324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 331324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver public virtual string ToString(int start, int stop) { 332324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if (start < 0 || stop < 0) 333324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return null; 334324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if (_p == -1) 335324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver Setup(); 336324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if (stop >= _tokens.Count) 337324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver stop = _tokens.Count - 1; 338324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 339324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver StringBuilder buf = new StringBuilder(); 340324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver for (int i = start; i <= stop; i++) { 341324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver IToken t = _tokens[i]; 342324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if (t.Type == CharStreamConstants.EndOfFile) 343324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver break; 344324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver buf.Append(t.Text); 345324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 346324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 347324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return buf.ToString(); 348324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 349324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 350324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver public virtual string ToString(IToken start, IToken stop) { 351324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if (start != null && stop != null) { 352324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return ToString(start.TokenIndex, stop.TokenIndex); 353324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 354324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return null; 355324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 356324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 357324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver public virtual void Fill() { 358324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if (_p == -1) 359324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver Setup(); 360324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 361324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if (_tokens[_p].Type == CharStreamConstants.EndOfFile) 362324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return; 363324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 364324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver int i = _p + 1; 365324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver Sync(i); 366324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver while (_tokens[i].Type != CharStreamConstants.EndOfFile) { 367324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver i++; 368324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver Sync(i); 369324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 370324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 371324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 372324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver} 373