13447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein/* 23447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein [The "BSD license"] 33447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein Copyright (c) 2005-2009 Terence Parr 43447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein All rights reserved. 53447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein 63447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein Redistribution and use in source and binary forms, with or without 73447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein modification, are permitted provided that the following conditions 83447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein are met: 93447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein 1. Redistributions of source code must retain the above copyright 103447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein notice, this list of conditions and the following disclaimer. 113447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein 2. Redistributions in binary form must reproduce the above copyright 123447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein notice, this list of conditions and the following disclaimer in the 133447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein documentation and/or other materials provided with the distribution. 143447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein 3. The name of the author may not be used to endorse or promote products 153447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein derived from this software without specific prior written permission. 163447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein 173447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 183447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 193447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 203447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 213447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 223447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 233447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 243447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 253447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 263447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 273447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein */ 283447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein 293447a5916aa62f44de24cc441fc9987116ddff52Andrew Sappersteinpackage org.antlr.runtime; 303447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein 313447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein/** The most common stream of tokens where every token is buffered up 323447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein * and tokens are filtered for a certain channel (the parser will only 333447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein * see these tokens). 343447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein * 353447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein * Even though it buffers all of the tokens, this token stream pulls tokens 363447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein * from the tokens source on demand. In other words, until you ask for a 373447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein * token using consume(), LT(), etc. the stream does not pull from the lexer. 383447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein * 393447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein * The only difference between this stream and BufferedTokenStream superclass 403447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein * is that this stream knows how to ignore off channel tokens. There may be 413447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein * a performance advantage to using the superclass if you don't pass 423447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein * whitespace and comments etc. to the parser on a hidden channel (i.e., 433447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein * you set $channel instead of calling skip() in lexer rules.) 443447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein * 453447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein * @see org.antlr.runtime.UnbufferedTokenStream 463447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein * @see org.antlr.runtime.BufferedTokenStream 473447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein */ 483447a5916aa62f44de24cc441fc9987116ddff52Andrew Sappersteinpublic class CommonTokenStream extends BufferedTokenStream { 493447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein /** Skip tokens on any channel but this one; this is how we skip whitespace... */ 503447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein protected int channel = Token.DEFAULT_CHANNEL; 513447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein 523447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein public CommonTokenStream() { ; } 533447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein 543447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein public CommonTokenStream(TokenSource tokenSource) { 553447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein super(tokenSource); 563447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 573447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein 583447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein public CommonTokenStream(TokenSource tokenSource, int channel) { 593447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein this(tokenSource); 603447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein this.channel = channel; 613447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 623447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein 633447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein /** Always leave p on an on-channel token. */ 643447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein public void consume() { 653447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein if ( p == -1 ) setup(); 663447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein p++; 673447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein sync(p); 683447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein while ( tokens.get(p).getChannel()!=channel ) { 693447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein p++; 703447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein sync(p); 713447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 723447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 733447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein 743447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein protected Token LB(int k) { 753447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein if ( k==0 || (p-k)<0 ) return null; 763447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein 773447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein int i = p; 783447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein int n = 1; 793447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein // find k good tokens looking backwards 803447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein while ( n<=k ) { 813447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein // skip off-channel tokens 823447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein i = skipOffTokenChannelsReverse(i-1); 833447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein n++; 843447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 853447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein if ( i<0 ) return null; 863447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein return tokens.get(i); 873447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 883447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein 893447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein public Token LT(int k) { 903447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein //System.out.println("enter LT("+k+")"); 913447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein if ( p == -1 ) setup(); 923447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein if ( k == 0 ) return null; 933447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein if ( k < 0 ) return LB(-k); 943447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein int i = p; 953447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein int n = 1; // we know tokens[p] is a good one 963447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein // find k good tokens 973447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein while ( n<k ) { 983447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein // skip off-channel tokens 993447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein i = skipOffTokenChannels(i+1); 1003447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein n++; 1013447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 102324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver if ( i>range ) range = i; 1033447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein return tokens.get(i); 1043447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 1053447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein 1063447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein /** Given a starting index, return the index of the first on-channel 1073447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein * token. 1083447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein */ 1093447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein protected int skipOffTokenChannels(int i) { 1103447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein sync(i); 1113447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein while ( tokens.get(i).getChannel()!=channel ) { // also stops at EOF (it's onchannel) 1123447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein i++; 1133447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein sync(i); 1143447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 1153447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein return i; 1163447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 1173447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein 1183447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein protected int skipOffTokenChannelsReverse(int i) { 1193447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein while ( i>=0 && ((Token)tokens.get(i)).getChannel()!=channel ) { 1203447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein i--; 1213447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 1223447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein return i; 1233447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 1243447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein 125324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver public void reset() { 126324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver super.reset(); 127324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver p = skipOffTokenChannels(0); 128324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver } 129324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver 130324c4644fee44b9898524c09511bd33c3f12e2dfBen Gruver protected void setup() { 1313447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein p = 0; 1323447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein sync(0); 1333447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein int i = 0; 1343447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein while ( tokens.get(i).getChannel()!=channel ) { 1353447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein i++; 1363447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein sync(i); 1373447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 1383447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein p = i; 1393447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 1403447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein 1413447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein /** Count EOF just once. */ 1423447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein public int getNumberOfOnChannelTokens() { 1433447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein int n = 0; 1443447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein fill(); 1453447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein for (int i = 0; i < tokens.size(); i++) { 1463447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein Token t = tokens.get(i); 1473447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein if ( t.getChannel()==channel ) n++; 1483447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein if ( t.getType()==Token.EOF ) break; 1493447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 1503447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein return n; 1513447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 1523447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein 1533447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein /** Reset this token stream by setting its token source. */ 1543447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein public void setTokenSource(TokenSource tokenSource) { 1553447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein super.setTokenSource(tokenSource); 1563447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein channel = Token.DEFAULT_CHANNEL; 1573447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 1583447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein} 159