1324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver/* 2324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * NOTE TO JL: The original contains references to 3.5-only stuff! 3324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * 4324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * [The "BSD licence"] 5324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * Copyright (c) 2005-2008 Terence Parr 6324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * All rights reserved. 7324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * 8324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * Conversion to C#: 9324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * Copyright (c) 2008-2009 Sam Harwell, Pixel Mine, Inc. 10324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * All rights reserved. 11324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * 12324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * Redistribution and use in source and binary forms, with or without 13324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * modification, are permitted provided that the following conditions 14324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * are met: 15324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * 1. Redistributions of source code must retain the above copyright 16324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * notice, this list of conditions and the following disclaimer. 17324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * 2. Redistributions in binary form must reproduce the above copyright 18324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * notice, this list of conditions and the following disclaimer in the 19324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * documentation and/or other materials provided with the distribution. 20324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * 3. The name of the author may not be used to endorse or promote products 21324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * derived from this software without specific prior written permission. 22324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * 23324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 24324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 25324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 26324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 27324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 28324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 29324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 30324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 31324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 32324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 33324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver */ 34324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 35324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruvernamespace Antlr.Runtime { 36324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver using System.Collections.Generic; 37324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 38324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver using ArgumentOutOfRangeException = System.ArgumentOutOfRangeException; 39324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver using InvalidOperationException = System.InvalidOperationException; 40324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver using StringBuilder = System.Text.StringBuilder; 41324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 42324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver /** <summary> 43324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * The most common stream of tokens is one where every token is buffered up 44324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * and tokens are prefiltered for a certain channel (the parser will only 45324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * see these tokens and cannot change the filter channel number during the 46324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * parse). 47324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * </summary> 48324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * 49324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * <remarks>TODO: how to access the full token stream? How to track all tokens matched per rule?</remarks> 50324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver */ 51324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver [System.Serializable] 52324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver public class LegacyCommonTokenStream : ITokenStream { 53324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver [System.NonSerialized] 54324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver ITokenSource _tokenSource; 55324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 56324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver /** <summary> 57324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * Record every single token pulled from the source so we can reproduce 58324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * chunks of it later. 59324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * </summary> 60324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver */ 61324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver protected List<IToken> tokens; 62324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 63324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver /** <summary>Map<tokentype, channel> to override some Tokens' channel numbers</summary> */ 64324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver protected IDictionary<int, int> channelOverrideMap; 65324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 66324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver /** <summary>Set<tokentype>; discard any tokens with this type</summary> */ 67324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver protected Dictionary<int, int> discardSet; 68324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 69324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver /** <summary>Skip tokens on any channel but this one; this is how we skip whitespace...</summary> */ 70324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver protected int channel = TokenChannels.Default; 71324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 72324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver /** <summary>By default, track all incoming tokens</summary> */ 73324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver protected bool discardOffChannelTokens = false; 74324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 75324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver /** <summary>Track the last mark() call result value for use in rewind().</summary> */ 76324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver protected int lastMarker; 77324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 78324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver /** <summary> 79324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * The index into the tokens list of the current token (next token 80324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * to consume). p==-1 indicates that the tokens list is empty 81324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * </summary> 82324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver */ 83324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver protected int p = -1; 84324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 85324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver public LegacyCommonTokenStream() { 86324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver tokens = new List<IToken>(500); 87324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 88324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 89324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver public LegacyCommonTokenStream(ITokenSource tokenSource) 90324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver : this() { 91324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver this._tokenSource = tokenSource; 92324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 93324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 94324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver public LegacyCommonTokenStream(ITokenSource tokenSource, int channel) 95324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver : this(tokenSource) { 96324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver this.channel = channel; 97324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 98324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 99324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver public virtual int Index { 100324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver get { 101324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return p; 102324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 103324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 104324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 105324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver /// <summary> 106324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver /// How deep have we gone? 107324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver /// </summary> 108324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver public virtual int Range { 109324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver get; 110324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver protected set; 111324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 112324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 113324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver /** <summary>Reset this token stream by setting its token source.</summary> */ 114324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver public virtual void SetTokenSource(ITokenSource tokenSource) { 115324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver this._tokenSource = tokenSource; 116324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver tokens.Clear(); 117324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver p = -1; 118324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver channel = TokenChannels.Default; 119324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 120324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 121324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver /** <summary> 122324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * Load all tokens from the token source and put in tokens. 123324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * This is done upon first LT request because you might want to 124324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * set some token type / channel overrides before filling buffer. 125324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * </summary> 126324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver */ 127324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver public virtual void FillBuffer() { 128324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver // fast return if the buffer is already full 129324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if (p != -1) 130324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return; 131324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 132324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver int index = 0; 133324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver IToken t = _tokenSource.NextToken(); 134324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver while (t != null && t.Type != CharStreamConstants.EndOfFile) { 135324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver bool discard = false; 136324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver // is there a channel override for token type? 137324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver int channelI; 138324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if (channelOverrideMap != null && channelOverrideMap.TryGetValue(t.Type, out channelI)) 139324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver t.Channel = channelI; 140324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 141324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver //if ( channelOverrideMap != null && channelOverrideMap.ContainsKey( t.getType() ) ) 142324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver //{ 143324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver // object channelI = channelOverrideMap.get( t.getType() ); 144324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver // if ( channelI != null ) 145324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver // { 146324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver // t.setChannel( (int)channelI ); 147324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver // } 148324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver //} 149324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if (discardSet != null && 150324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver discardSet.ContainsKey(t.Type)) { 151324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver discard = true; 152324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } else if (discardOffChannelTokens && t.Channel != this.channel) { 153324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver discard = true; 154324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 155324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if (!discard) { 156324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver t.TokenIndex = index; 157324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver tokens.Add(t); 158324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver index++; 159324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 160324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver t = _tokenSource.NextToken(); 161324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 162324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver // leave p pointing at first token on channel 163324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver p = 0; 164324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver p = SkipOffTokenChannels(p); 165324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 166324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 167324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver /** <summary> 168324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * Move the input pointer to the next incoming token. The stream 169324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * must become active with LT(1) available. consume() simply 170324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * moves the input pointer so that LT(1) points at the next 171324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * input symbol. Consume at least one token. 172324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * </summary> 173324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * 174324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * <remarks> 175324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * Walk past any token not on the channel the parser is listening to. 176324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * </remarks> 177324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver */ 178324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver public virtual void Consume() { 179324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if (p < tokens.Count) { 180324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver p++; 181324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver p = SkipOffTokenChannels(p); // leave p on valid token 182324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 183324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 184324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 185324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver /** <summary>Given a starting index, return the index of the first on-channel token.</summary> */ 186324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver protected virtual int SkipOffTokenChannels(int i) { 187324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver int n = tokens.Count; 188324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver while (i < n && ((IToken)tokens[i]).Channel != channel) { 189324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver i++; 190324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 191324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return i; 192324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 193324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 194324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver protected virtual int SkipOffTokenChannelsReverse(int i) { 195324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver while (i >= 0 && ((IToken)tokens[i]).Channel != channel) { 196324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver i--; 197324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 198324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return i; 199324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 200324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 201324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver /** <summary> 202324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * A simple filter mechanism whereby you can tell this token stream 203324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * to force all tokens of type ttype to be on channel. For example, 204324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * when interpreting, we cannot exec actions so we need to tell 205324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * the stream to force all WS and NEWLINE to be a different, ignored 206324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * channel. 207324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * </summary> 208324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver */ 209324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver public virtual void SetTokenTypeChannel(int ttype, int channel) { 210324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if (channelOverrideMap == null) { 211324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver channelOverrideMap = new Dictionary<int, int>(); 212324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 213324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver channelOverrideMap[ttype] = channel; 214324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 215324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 216324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver public virtual void DiscardTokenType(int ttype) { 217324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if (discardSet == null) { 218324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver discardSet = new Dictionary<int, int>(); 219324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 220324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver discardSet.Add(ttype, ttype); 221324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 222324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 223324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver public virtual void SetDiscardOffChannelTokens(bool discardOffChannelTokens) { 224324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver this.discardOffChannelTokens = discardOffChannelTokens; 225324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 226324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 227324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver public virtual IList<IToken> GetTokens() { 228324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if (p == -1) { 229324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver FillBuffer(); 230324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 231324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return tokens; 232324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 233324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 234324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver public virtual IList<IToken> GetTokens(int start, int stop) { 235324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return GetTokens(start, stop, (BitSet)null); 236324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 237324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 238324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver /** <summary> 239324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * Given a start and stop index, return a List of all tokens in 240324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * the token type BitSet. Return null if no tokens were found. This 241324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * method looks at both on and off channel tokens. 242324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * </summary> 243324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver */ 244324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver public virtual IList<IToken> GetTokens(int start, int stop, BitSet types) { 245324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if (p == -1) { 246324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver FillBuffer(); 247324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 248324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if (stop >= tokens.Count) { 249324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver stop = tokens.Count - 1; 250324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 251324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if (start < 0) { 252324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver start = 0; 253324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 254324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if (start > stop) { 255324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return null; 256324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 257324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 258324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver // list = tokens[start:stop]:{Token t, t.getType() in types} 259324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver IList<IToken> filteredTokens = new List<IToken>(); 260324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver for (int i = start; i <= stop; i++) { 261324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver IToken t = tokens[i]; 262324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if (types == null || types.Member(t.Type)) { 263324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver filteredTokens.Add(t); 264324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 265324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 266324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if (filteredTokens.Count == 0) { 267324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver filteredTokens = null; 268324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 269324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return filteredTokens; 270324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 271324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 272324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver public virtual IList<IToken> GetTokens(int start, int stop, IList<int> types) { 273324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return GetTokens(start, stop, new BitSet(types)); 274324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 275324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 276324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver public virtual IList<IToken> GetTokens(int start, int stop, int ttype) { 277324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return GetTokens(start, stop, BitSet.Of(ttype)); 278324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 279324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 280324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver /** <summary> 281324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * Get the ith token from the current position 1..n where k=1 is the 282324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * first symbol of lookahead. 283324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * </summary> 284324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver */ 285324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver public virtual IToken LT(int k) { 286324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if (p == -1) { 287324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver FillBuffer(); 288324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 289324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if (k == 0) { 290324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return null; 291324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 292324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if (k < 0) { 293324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return LB(-k); 294324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 295324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver //System.out.print("LT(p="+p+","+k+")="); 296324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if ((p + k - 1) >= tokens.Count) { 297324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return tokens[tokens.Count - 1]; 298324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 299324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver //System.out.println(tokens.get(p+k-1)); 300324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver int i = p; 301324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver int n = 1; 302324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver // find k good tokens 303324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver while (n < k) { 304324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver // skip off-channel tokens 305324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver i = SkipOffTokenChannels(i + 1); // leave p on valid token 306324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver n++; 307324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 308324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if (i >= tokens.Count) { 309324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return tokens[tokens.Count - 1]; 310324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 311324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 312324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if (i > Range) 313324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver Range = i; 314324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 315324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return (IToken)tokens[i]; 316324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 317324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 318324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver /** <summary>Look backwards k tokens on-channel tokens</summary> */ 319324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver protected virtual IToken LB(int k) { 320324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver //System.out.print("LB(p="+p+","+k+") "); 321324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if (p == -1) { 322324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver FillBuffer(); 323324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 324324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if (k == 0) { 325324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return null; 326324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 327324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if ((p - k) < 0) { 328324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return null; 329324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 330324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 331324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver int i = p; 332324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver int n = 1; 333324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver // find k good tokens looking backwards 334324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver while (n <= k) { 335324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver // skip off-channel tokens 336324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver i = SkipOffTokenChannelsReverse(i - 1); // leave p on valid token 337324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver n++; 338324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 339324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if (i < 0) { 340324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return null; 341324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 342324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return (IToken)tokens[i]; 343324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 344324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 345324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver /** <summary> 346324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * Return absolute token i; ignore which channel the tokens are on; 347324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * that is, count all tokens not just on-channel tokens. 348324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * </summary> 349324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver */ 350324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver public virtual IToken Get(int i) { 351324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return (IToken)tokens[i]; 352324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 353324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 354324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver#if false 355324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver /** Get all tokens from start..stop inclusively */ 356324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver public virtual List<IToken> Get(int start, int count) 357324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver { 358324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if (start < 0) 359324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver throw new ArgumentOutOfRangeException("start"); 360324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if (count < 0) 361324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver throw new ArgumentOutOfRangeException("count"); 362324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 363324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if (p == -1) 364324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver FillBuffer(); 365324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 366324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return new List<IToken>(tokens.Skip(start).Take(count)); 367324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 368324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver#endif 369324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 370324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver public virtual int LA(int i) { 371324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return LT(i).Type; 372324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 373324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 374324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver public virtual int Mark() { 375324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if (p == -1) { 376324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver FillBuffer(); 377324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 378324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver lastMarker = Index; 379324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return lastMarker; 380324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 381324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 382324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver public virtual void Release(int marker) { 383324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver // no resources to release 384324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 385324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 386324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver public virtual int Count { 387324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver get { 388324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return tokens.Count; 389324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 390324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 391324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 392324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver public virtual void Rewind(int marker) { 393324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver Seek(marker); 394324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 395324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 396324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver public virtual void Rewind() { 397324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver Seek(lastMarker); 398324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 399324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 400324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver public virtual void Reset() { 401324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver p = 0; 402324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver lastMarker = 0; 403324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 404324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 405324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver public virtual void Seek(int index) { 406324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver p = index; 407324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 408324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 409324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver public virtual ITokenSource TokenSource { 410324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver get { 411324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return _tokenSource; 412324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 413324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 414324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 415324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver public virtual string SourceName { 416324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver get { 417324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return TokenSource.SourceName; 418324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 419324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 420324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 421324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver public override string ToString() { 422324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if (p == -1) { 423324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver throw new InvalidOperationException("Buffer is not yet filled."); 424324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 425324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return ToString(0, tokens.Count - 1); 426324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 427324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 428324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver public virtual string ToString(int start, int stop) { 429324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if (start < 0 || stop < 0) { 430324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return null; 431324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 432324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if (p == -1) { 433324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver throw new InvalidOperationException("Buffer is not yet filled."); 434324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 435324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if (stop >= tokens.Count) { 436324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver stop = tokens.Count - 1; 437324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 438324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver StringBuilder buf = new StringBuilder(); 439324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver for (int i = start; i <= stop; i++) { 440324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver IToken t = tokens[i]; 441324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver buf.Append(t.Text); 442324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 443324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return buf.ToString(); 444324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 445324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 446324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver public virtual string ToString(IToken start, IToken stop) { 447324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if (start != null && stop != null) { 448324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return ToString(start.TokenIndex, stop.TokenIndex); 449324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 450324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return null; 451324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 452324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 453324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver} 454