1324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver/* 2324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * [The "BSD licence"] 3324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * Copyright (c) 2005-2008 Terence Parr 4324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * All rights reserved. 5324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * 6324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * Conversion to C#: 7324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * Copyright (c) 2008-2009 Sam Harwell, Pixel Mine, Inc. 8324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * All rights reserved. 9324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * 10324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * Redistribution and use in source and binary forms, with or without 11324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * modification, are permitted provided that the following conditions 12324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * are met: 13324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * 1. Redistributions of source code must retain the above copyright 14324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * notice, this list of conditions and the following disclaimer. 15324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * 2. Redistributions in binary form must reproduce the above copyright 16324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * notice, this list of conditions and the following disclaimer in the 17324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * documentation and/or other materials provided with the distribution. 18324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * 3. The name of the author may not be used to endorse or promote products 19324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * derived from this software without specific prior written permission. 20324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * 21324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 22324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 23324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 24324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 25324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 26324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 27324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 28324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 29324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 30324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 31324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver */ 32324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 33324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruvernamespace Antlr.Runtime 34324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver{ 35324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver using System.Collections.Generic; 36324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver using CLSCompliant = System.CLSCompliantAttribute; 37324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver using IndexOutOfRangeException = System.IndexOutOfRangeException; 38324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver using StringBuilder = System.Text.StringBuilder; 39324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 40324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver /** Buffer all input tokens but do on-demand fetching of new tokens from 41324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * lexer. Useful when the parser or lexer has to set context/mode info before 42324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * proper lexing of future tokens. The ST template parser needs this, 43324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * for example, because it has to constantly flip back and forth between 44324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * inside/output templates. E.g., <names:{hi, <it>}> has to parse names 45324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * as part of an expression but "hi, <it>" as a nested template. 46324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * 47324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * You can't use this stream if you pass whitespace or other off-channel 48324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * tokens to the parser. The stream can't ignore off-channel tokens. 49324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * (UnbufferedTokenStream is the same way.) 50324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * 51324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * This is not a subclass of UnbufferedTokenStream because I don't want 52324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * to confuse small moving window of tokens it uses for the full buffer. 53324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver */ 54324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver [System.Serializable] 55324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver public class BufferedTokenStream : ITokenStream, ITokenStreamInformation 56324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver { 57324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver private ITokenSource _tokenSource; 58324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 59324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver /** Record every single token pulled from the source so we can reproduce 60324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * chunks of it later. The buffer in LookaheadStream overlaps sometimes 61324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * as its moving window moves through the input. This list captures 62324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * everything so we can access complete input text. 63324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver */ 64324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver [CLSCompliant(false)] 65324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver protected List<IToken> _tokens = new List<IToken>(100); 66324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 67324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver /** Track the last mark() call result value for use in rewind(). */ 68324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver private int _lastMarker; 69324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 70324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver /** The index into the tokens list of the current token (next token 71324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * to consume). tokens[p] should be LT(1). p=-1 indicates need 72324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * to initialize with first token. The ctor doesn't get a token. 73324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * First call to LT(1) or whatever gets the first token and sets p=0; 74324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver */ 75324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver [CLSCompliant(false)] 76324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver protected int _p = -1; 77324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 78324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver public BufferedTokenStream() 79324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver { 80324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 81324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 82324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver public BufferedTokenStream(ITokenSource tokenSource) 83324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver { 84324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver this._tokenSource = tokenSource; 85324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 86324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 87324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver public virtual ITokenSource TokenSource 88324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver { 89324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver get 90324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver { 91324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return _tokenSource; 92324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 93324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver set 94324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver { 95324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver this._tokenSource = value; 96324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver _tokens.Clear(); 97324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver _p = -1; 98324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 99324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 100324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 101324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver public virtual int Index 102324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver { 103324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver get 104324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver { 105324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return _p; 106324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 107324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 108324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 109324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver /// <summary> 110324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver /// How deep have we gone? 111324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver /// </summary> 112324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver public virtual int Range 113324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver { 114324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver get; 115324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver protected set; 116324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 117324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 118324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver public virtual int Count 119324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver { 120324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver get 121324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver { 122324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return _tokens.Count; 123324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 124324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 125324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 126324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver public virtual string SourceName 127324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver { 128324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver get 129324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver { 130324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return _tokenSource.SourceName; 131324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 132324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 133324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 134324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver public virtual IToken LastToken 135324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver { 136324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver get 137324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver { 138324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return LB(1); 139324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 140324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 141324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 142324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver public virtual IToken LastRealToken 143324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver { 144324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver get 145324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver { 146324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver int i = 0; 147324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver IToken token; 148324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver do 149324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver { 150324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver i++; 151324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver token = LB(i); 152324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } while (token != null && token.Line <= 0); 153324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 154324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return token; 155324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 156324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 157324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 158324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver public virtual int MaxLookBehind 159324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver { 160324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver get 161324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver { 162324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return int.MaxValue; 163324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 164324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 165324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 166324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver public virtual int Mark() 167324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver { 168324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if (_p == -1) 169324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver Setup(); 170324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver _lastMarker = Index; 171324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return _lastMarker; 172324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 173324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 174324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver public virtual void Release(int marker) 175324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver { 176324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver // no resources to release 177324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 178324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 179324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver public virtual void Rewind(int marker) 180324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver { 181324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver Seek(marker); 182324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 183324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 184324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver public virtual void Rewind() 185324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver { 186324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver Seek(_lastMarker); 187324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 188324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 189324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver public virtual void Reset() 190324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver { 191324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver _p = 0; 192324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver _lastMarker = 0; 193324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 194324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 195324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver public virtual void Seek(int index) 196324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver { 197324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver _p = index; 198324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 199324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 200324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver /** Move the input pointer to the next incoming token. The stream 201324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * must become active with LT(1) available. consume() simply 202324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * moves the input pointer so that LT(1) points at the next 203324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * input symbol. Consume at least one token. 204324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * 205324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * Walk past any token not on the channel the parser is listening to. 206324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver */ 207324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver public virtual void Consume() 208324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver { 209324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if (_p == -1) 210324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver Setup(); 211324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver _p++; 212324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver Sync(_p); 213324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 214324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 215324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver /** Make sure index i in tokens has a token. */ 216324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver protected virtual void Sync(int i) 217324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver { 218324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver int n = i - _tokens.Count + 1; // how many more elements we need? 219324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if (n > 0) 220324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver Fetch(n); 221324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 222324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 223324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver /** add n elements to buffer */ 224324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver protected virtual void Fetch(int n) 225324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver { 226324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver for (int i = 0; i < n; i++) 227324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver { 228324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver IToken t = TokenSource.NextToken(); 229324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver t.TokenIndex = _tokens.Count; 230324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver _tokens.Add(t); 231324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if (t.Type == CharStreamConstants.EndOfFile) 232324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver break; 233324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 234324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 235324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 236324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver public virtual IToken Get(int i) 237324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver { 238324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if (i < 0 || i >= _tokens.Count) 239324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver { 240324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver throw new IndexOutOfRangeException("token index " + i + " out of range 0.." + (_tokens.Count - 1)); 241324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 242324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return _tokens[i]; 243324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 244324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 245324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver#if false // why is this different from GetTokens(start, count) ? 246324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver /// <summary> 247324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver /// Get all tokens from start..(start+count-1) inclusively 248324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver /// </summary> 249324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver public virtual List<IToken> Get(int start, int count) 250324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver { 251324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if (start < 0) 252324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver throw new ArgumentOutOfRangeException("start"); 253324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if (count < 0) 254324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver throw new ArgumentOutOfRangeException("count"); 255324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if (start + count >= _tokens.Count) 256324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver throw new ArgumentException(); 257324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 258324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if (_p == -1) 259324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver Setup(); 260324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 261324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver List<IToken> subset = new List<IToken>(count); 262324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver for (int i = 0; i < count; i++) 263324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver { 264324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver IToken token = _tokens[i]; 265324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if (token.Type == TokenTypes.EndOfFile) 266324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver break; 267324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 268324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver subset.Add(token); 269324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 270324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 271324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return subset; 272324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 273324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver#endif 274324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 275324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver public virtual int LA(int i) 276324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver { 277324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return LT(i).Type; 278324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 279324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 280324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver protected virtual IToken LB(int k) 281324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver { 282324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if ((_p - k) < 0) 283324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return null; 284324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 285324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return _tokens[_p - k]; 286324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 287324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 288324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver public virtual IToken LT(int k) 289324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver { 290324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if (_p == -1) 291324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver Setup(); 292324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if (k == 0) 293324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return null; 294324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if (k < 0) 295324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return LB(-k); 296324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 297324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver int i = _p + k - 1; 298324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver Sync(i); 299324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if (i >= _tokens.Count) 300324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver { 301324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver // EOF must be last token 302324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return _tokens[_tokens.Count - 1]; 303324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 304324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 305324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if (i > Range) 306324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver Range = i; 307324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 308324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return _tokens[_p + k - 1]; 309324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 310324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 311324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver protected virtual void Setup() 312324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver { 313324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver Sync(0); 314324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver _p = 0; 315324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 316324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 317324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver public virtual List<IToken> GetTokens() 318324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver { 319324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return _tokens; 320324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 321324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 322324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver public virtual List<IToken> GetTokens(int start, int stop) 323324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver { 324324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return GetTokens(start, stop, default(BitSet)); 325324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 326324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 327324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver /** Given a start and stop index, return a List of all tokens in 328324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * the token type BitSet. Return null if no tokens were found. This 329324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * method looks at both on and off channel tokens. 330324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver */ 331324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver public virtual List<IToken> GetTokens(int start, int stop, BitSet types) 332324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver { 333324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if (_p == -1) 334324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver Setup(); 335324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if (stop >= _tokens.Count) 336324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver stop = _tokens.Count - 1; 337324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if (start < 0) 338324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver start = 0; 339324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if (start > stop) 340324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return null; 341324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 342324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver // list = tokens[start:stop]:{Token t, t.getType() in types} 343324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver List<IToken> filteredTokens = new List<IToken>(); 344324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver for (int i = start; i <= stop; i++) 345324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver { 346324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver IToken t = _tokens[i]; 347324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if (types == null || types.Member(t.Type)) 348324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver { 349324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver filteredTokens.Add(t); 350324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 351324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 352324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if (filteredTokens.Count == 0) 353324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver { 354324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver filteredTokens = null; 355324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 356324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return filteredTokens; 357324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 358324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 359324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver public virtual List<IToken> GetTokens(int start, int stop, IEnumerable<int> types) 360324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver { 361324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return GetTokens(start, stop, new BitSet(types)); 362324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 363324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 364324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver public virtual List<IToken> GetTokens(int start, int stop, int ttype) 365324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver { 366324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return GetTokens(start, stop, BitSet.Of(ttype)); 367324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 368324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 369324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver public override string ToString() 370324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver { 371324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if (_p == -1) 372324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver Setup(); 373324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 374324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver Fill(); 375324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return ToString(0, _tokens.Count - 1); 376324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 377324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 378324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver public virtual string ToString(int start, int stop) 379324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver { 380324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if (start < 0 || stop < 0) 381324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return null; 382324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if (_p == -1) 383324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver Setup(); 384324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if (stop >= _tokens.Count) 385324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver stop = _tokens.Count - 1; 386324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 387324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver StringBuilder buf = new StringBuilder(); 388324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver for (int i = start; i <= stop; i++) 389324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver { 390324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver IToken t = _tokens[i]; 391324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if (t.Type == CharStreamConstants.EndOfFile) 392324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver break; 393324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver buf.Append(t.Text); 394324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 395324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 396324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return buf.ToString(); 397324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 398324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 399324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver public virtual string ToString(IToken start, IToken stop) 400324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver { 401324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if (start != null && stop != null) 402324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver { 403324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return ToString(start.TokenIndex, stop.TokenIndex); 404324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 405324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return null; 406324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 407324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 408324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver public virtual void Fill() 409324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver { 410324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if (_p == -1) 411324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver Setup(); 412324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 413324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if (_tokens[_p].Type == CharStreamConstants.EndOfFile) 414324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return; 415324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 416324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver int i = _p + 1; 417324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver Sync(i); 418324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver while (_tokens[i].Type != CharStreamConstants.EndOfFile) 419324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver { 420324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver i++; 421324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver Sync(i); 422324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 423324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 424324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 425324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver} 426