1324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver/* 2324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * [The "BSD licence"] 3324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * Copyright (c) 2005-2008 Terence Parr 4324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * All rights reserved. 5324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * 6324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * Conversion to C#: 7324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * Copyright (c) 2008-2009 Sam Harwell, Pixel Mine, Inc. 8324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * All rights reserved. 9324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * 10324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * Redistribution and use in source and binary forms, with or without 11324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * modification, are permitted provided that the following conditions 12324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * are met: 13324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * 1. Redistributions of source code must retain the above copyright 14324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * notice, this list of conditions and the following disclaimer. 15324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * 2. Redistributions in binary form must reproduce the above copyright 16324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * notice, this list of conditions and the following disclaimer in the 17324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * documentation and/or other materials provided with the distribution. 18324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * 3. The name of the author may not be used to endorse or promote products 19324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * derived from this software without specific prior written permission. 20324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * 21324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 22324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 23324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 24324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 25324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 26324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 27324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 28324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 29324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 30324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 31324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver */ 32324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 33324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruvernamespace Antlr.Runtime 34324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver{ 35324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver using System.Collections.Generic; 36324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 37324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver using InvalidOperationException = System.InvalidOperationException; 38324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver using StringBuilder = System.Text.StringBuilder; 39324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 40324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver /** <summary> 41324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * The most common stream of tokens is one where every token is buffered up 42324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * and tokens are prefiltered for a certain channel (the parser will only 43324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * see these tokens and cannot change the filter channel number during the 44324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * parse). 45324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * </summary> 46324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * 47324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * <remarks>TODO: how to access the full token stream? How to track all tokens matched per rule?</remarks> 48324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver */ 49324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver [System.Serializable] 50324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver public class CommonTokenStream : BufferedTokenStream 51324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver { 52324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver /** Skip tokens on any channel but this one; this is how we skip whitespace... */ 53324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver private int _channel; 54324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 55324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver public CommonTokenStream() 56324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver { 57324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 58324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 59324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver public CommonTokenStream(ITokenSource tokenSource) 60324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver : this(tokenSource, TokenChannels.Default) 61324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver { 62324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 63324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 64324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver public CommonTokenStream(ITokenSource tokenSource, int channel) 65324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver : base(tokenSource) 66324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver { 67324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver this._channel = channel; 68324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 69324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 70324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver public int Channel 71324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver { 72324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver get 73324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver { 74324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return _channel; 75324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 76324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 77324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 78324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver /** Reset this token stream by setting its token source. */ 79324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver public override ITokenSource TokenSource 80324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver { 81324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver get 82324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver { 83324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return base.TokenSource; 84324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 85324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver set 86324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver { 87324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver base.TokenSource = value; 88324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver _channel = TokenChannels.Default; 89324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 90324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 91324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 92324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver /** Always leave p on an on-channel token. */ 93324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver public override void Consume() 94324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver { 95324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if (_p == -1) 96324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver Setup(); 97324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver _p++; 98324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver _p = SkipOffTokenChannels(_p); 99324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 100324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 101324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver protected override IToken LB(int k) 102324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver { 103324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if (k == 0 || (_p - k) < 0) 104324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return null; 105324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 106324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver int i = _p; 107324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver int n = 1; 108324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver // find k good tokens looking backwards 109324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver while (n <= k) 110324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver { 111324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver // skip off-channel tokens 112324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver i = SkipOffTokenChannelsReverse(i - 1); 113324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver n++; 114324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 115324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if (i < 0) 116324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return null; 117324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return _tokens[i]; 118324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 119324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 120324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver public override IToken LT(int k) 121324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver { 122324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if (_p == -1) 123324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver Setup(); 124324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if (k == 0) 125324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return null; 126324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if (k < 0) 127324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return LB(-k); 128324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver int i = _p; 129324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver int n = 1; // we know tokens[p] is a good one 130324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver // find k good tokens 131324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver while (n < k) 132324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver { 133324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver // skip off-channel tokens 134324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver i = SkipOffTokenChannels(i + 1); 135324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver n++; 136324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 137324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 138324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if (i > Range) 139324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver Range = i; 140324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 141324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return _tokens[i]; 142324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 143324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 144324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver /** Given a starting index, return the index of the first on-channel 145324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver * token. 146324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver */ 147324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver protected virtual int SkipOffTokenChannels(int i) 148324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver { 149324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver Sync(i); 150324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver while (_tokens[i].Channel != _channel) 151324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver { 152324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver // also stops at EOF (it's on channel) 153324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver i++; 154324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver Sync(i); 155324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 156324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return i; 157324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 158324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 159324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver protected virtual int SkipOffTokenChannelsReverse(int i) 160324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver { 161324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver while (i >= 0 && ((IToken)_tokens[i]).Channel != _channel) 162324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver { 163324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver i--; 164324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 165324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 166324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver return i; 167324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 168324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 169324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver protected override void Setup() 170324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver { 171324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver _p = 0; 172324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver _p = SkipOffTokenChannels(_p); 173324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 174324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 175324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver} 176