1324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver/* 2324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * [The "BSD licence"] 3324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * Copyright (c) 2005-2008 Terence Parr 4324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * All rights reserved. 5324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * 6324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * Conversion to C#: 7324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * Copyright (c) 2008-2009 Sam Harwell, Pixel Mine, Inc. 8324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * All rights reserved. 9324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * 10324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * Redistribution and use in source and binary forms, with or without 11324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * modification, are permitted provided that the following conditions 12324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * are met: 13324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * 1. Redistributions of source code must retain the above copyright 14324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * notice, this list of conditions and the following disclaimer. 15324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * 2. Redistributions in binary form must reproduce the above copyright 16324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * notice, this list of conditions and the following disclaimer in the 17324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * documentation and/or other materials provided with the distribution. 18324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * 3. The name of the author may not be used to endorse or promote products 19324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * derived from this software without specific prior written permission. 20324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * 21324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 22324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 23324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 24324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 25324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 26324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 27324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 28324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 29324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 30324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 31324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver */ 32324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 33324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruvernamespace Antlr.Runtime { 34324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver using System.Collections.Generic; 35324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 36324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver using InvalidOperationException = System.InvalidOperationException; 37324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver using StringBuilder = System.Text.StringBuilder; 38324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 39324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver /** <summary> 40324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * The most common stream of tokens is one where every token is buffered up 41324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * and tokens are prefiltered for a certain channel (the parser will only 42324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * see these tokens and cannot change the filter channel number during the 43324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * parse). 44324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * </summary> 45324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * 46324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * <remarks>TODO: how to access the full token stream? How to track all tokens matched per rule?</remarks> 47324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver */ 48324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver [System.Serializable] 49324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver public class CommonTokenStream : BufferedTokenStream { 50324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver /** Skip tokens on any channel but this one; this is how we skip whitespace... */ 51324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver private int _channel; 52324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 53324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver public CommonTokenStream() { 54324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 55324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 56324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver public CommonTokenStream(ITokenSource tokenSource) 57324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver : this(tokenSource, TokenChannels.Default) { 58324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 59324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 60324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver public CommonTokenStream(ITokenSource tokenSource, int channel) 61324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver : base(tokenSource) { 62324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver this._channel = channel; 63324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 64324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 65324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver public int Channel { 66324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver get { 67324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return _channel; 68324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 69324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 70324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 71324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver /** Reset this token stream by setting its token source. */ 72324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver public override ITokenSource TokenSource { 73324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver get { 74324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return base.TokenSource; 75324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 76324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver set { 77324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver base.TokenSource = value; 78324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver _channel = TokenChannels.Default; 79324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 80324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 81324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 82324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver /** Always leave p on an on-channel token. */ 83324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver public override void Consume() { 84324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if (_p == -1) 85324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver Setup(); 86324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver _p++; 87324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver Sync(_p); 88324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver while (_tokens[_p].Channel != _channel) { 89324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver _p++; 90324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver Sync(_p); 91324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 92324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 93324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 94324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver protected override IToken LB(int k) { 95324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if (k == 0 || (_p - k) < 0) 96324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return null; 97324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 98324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver int i = _p; 99324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver int n = 1; 100324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver // find k good tokens looking backwards 101324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver while (n <= k) { 102324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver // skip off-channel tokens 103324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver i = SkipOffTokenChannelsReverse(i - 1); 104324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver n++; 105324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 106324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if (i < 0) 107324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return null; 108324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return _tokens[i]; 109324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 110324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 111324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver public override IToken LT(int k) { 112324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if (_p == -1) 113324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver Setup(); 114324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if (k == 0) 115324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return null; 116324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if (k < 0) 117324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return LB(-k); 118324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver int i = _p; 119324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver int n = 1; // we know tokens[p] is a good one 120324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver // find k good tokens 121324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver while (n < k) { 122324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver // skip off-channel tokens 123324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver i = SkipOffTokenChannels(i + 1); 124324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver n++; 125324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 126324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 127324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if (i > Range) 128324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver Range = i; 129324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 130324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return _tokens[i]; 131324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 132324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 133324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver /** Given a starting index, return the index of the first on-channel 134324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * token. 135324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver */ 136324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver protected virtual int SkipOffTokenChannels(int i) { 137324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver Sync(i); 138324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver while (_tokens[i].Channel != _channel) { 139324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver // also stops at EOF (it's on channel) 140324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver i++; 141324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver Sync(i); 142324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 143324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return i; 144324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 145324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 146324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver protected virtual int SkipOffTokenChannelsReverse(int i) { 147324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver while (i >= 0 && ((IToken)_tokens[i]).Channel != _channel) { 148324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver i--; 149324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 150324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 151324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return i; 152324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 153324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 154324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver protected override void Setup() { 155324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver _p = 0; 156324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver Sync(0); 157324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver int i = 0; 158324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver while (_tokens[i].Channel != _channel) { 159324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver i++; 160324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver Sync(i); 161324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 162324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver _p = i; 163324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 164324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 165324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver} 166