13447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein/* 23447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein [The "BSD license"] 33447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein Copyright (c) 2005-2009 Terence Parr 43447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein All rights reserved. 53447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein 63447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein Redistribution and use in source and binary forms, with or without 73447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein modification, are permitted provided that the following conditions 83447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein are met: 93447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein 1. Redistributions of source code must retain the above copyright 103447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein notice, this list of conditions and the following disclaimer. 113447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein 2. Redistributions in binary form must reproduce the above copyright 123447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein notice, this list of conditions and the following disclaimer in the 133447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein documentation and/or other materials provided with the distribution. 143447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein 3. The name of the author may not be used to endorse or promote products 153447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein derived from this software without specific prior written permission. 163447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein 173447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 183447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 193447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 203447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 213447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 223447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 233447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 243447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 253447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 263447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 273447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein */ 283447a5916aa62f44de24cc441fc9987116ddff52Andrew Sappersteinpackage org.antlr.runtime; 293447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein 303447a5916aa62f44de24cc441fc9987116ddff52Andrew Sappersteinimport java.util.*; 313447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein 323447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein/** The most common stream of tokens is one where every token is buffered up 333447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein * and tokens are prefiltered for a certain channel (the parser will only 343447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein * see these tokens and cannot change the filter channel number during the 353447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein * parse). 363447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein * 373447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein * TODO: how to access the full token stream? How to track all tokens matched per rule? 383447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein */ 393447a5916aa62f44de24cc441fc9987116ddff52Andrew Sappersteinpublic class LegacyCommonTokenStream implements TokenStream { 403447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein protected TokenSource tokenSource; 413447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein 423447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein /** Record every single token pulled from the source so we can reproduce 433447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein * chunks of it later. 443447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein */ 453447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein protected List tokens; 463447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein 473447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein /** Map<tokentype, channel> to override some Tokens' channel numbers */ 483447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein protected Map channelOverrideMap; 493447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein 503447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein /** Set<tokentype>; discard any tokens with this type */ 513447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein protected Set discardSet; 523447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein 533447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein /** Skip tokens on any channel but this one; this is how we skip whitespace... */ 543447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein protected int channel = Token.DEFAULT_CHANNEL; 553447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein 563447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein /** By default, track all incoming tokens */ 573447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein protected boolean discardOffChannelTokens = false; 583447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein 593447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein /** Track the last mark() call result value for use in rewind(). */ 603447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein protected int lastMarker; 613447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein 623447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein protected int range = -1; // how deep have we gone? 633447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein 643447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein /** The index into the tokens list of the current token (next token 653447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein * to consume). p==-1 indicates that the tokens list is empty 663447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein */ 673447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein protected int p = -1; 683447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein 693447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein public LegacyCommonTokenStream() { 703447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein tokens = new ArrayList(500); 713447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 723447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein 733447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein public LegacyCommonTokenStream(TokenSource tokenSource) { 743447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein this(); 753447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein this.tokenSource = tokenSource; 763447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 773447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein 783447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein public LegacyCommonTokenStream(TokenSource tokenSource, int channel) { 793447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein this(tokenSource); 803447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein this.channel = channel; 813447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 823447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein 833447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein /** Reset this token stream by setting its token source. */ 843447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein public void setTokenSource(TokenSource tokenSource) { 853447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein this.tokenSource = tokenSource; 863447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein tokens.clear(); 873447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein p = -1; 883447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein channel = Token.DEFAULT_CHANNEL; 893447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 903447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein 913447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein /** Load all tokens from the token source and put in tokens. 923447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein * This is done upon first LT request because you might want to 933447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein * set some token type / channel overrides before filling buffer. 943447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein */ 953447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein protected void fillBuffer() { 963447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein int index = 0; 973447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein Token t = tokenSource.nextToken(); 983447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein while ( t!=null && t.getType()!=CharStream.EOF ) { 993447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein boolean discard = false; 1003447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein // is there a channel override for token type? 1013447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein if ( channelOverrideMap!=null ) { 1023447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein Integer channelI = (Integer) 1033447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein channelOverrideMap.get(new Integer(t.getType())); 1043447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein if ( channelI!=null ) { 1053447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein t.setChannel(channelI.intValue()); 1063447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 1073447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 1083447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein if ( discardSet!=null && 1093447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein discardSet.contains(new Integer(t.getType())) ) 1103447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein { 1113447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein discard = true; 1123447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 1133447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein else if ( discardOffChannelTokens && t.getChannel()!=this.channel ) { 1143447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein discard = true; 1153447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 1163447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein if ( !discard ) { 1173447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein t.setTokenIndex(index); 1183447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein tokens.add(t); 1193447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein index++; 1203447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 1213447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein t = tokenSource.nextToken(); 1223447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 1233447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein // leave p pointing at first token on channel 1243447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein p = 0; 1253447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein p = skipOffTokenChannels(p); 1263447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 1273447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein 1283447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein /** Move the input pointer to the next incoming token. The stream 1293447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein * must become active with LT(1) available. consume() simply 1303447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein * moves the input pointer so that LT(1) points at the next 1313447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein * input symbol. Consume at least one token. 1323447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein * 1333447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein * Walk past any token not on the channel the parser is listening to. 1343447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein */ 1353447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein public void consume() { 1363447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein if ( p<tokens.size() ) { 1373447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein p++; 1383447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein p = skipOffTokenChannels(p); // leave p on valid token 1393447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 1403447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 1413447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein 1423447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein /** Given a starting index, return the index of the first on-channel 1433447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein * token. 1443447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein */ 1453447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein protected int skipOffTokenChannels(int i) { 1463447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein int n = tokens.size(); 1473447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein while ( i<n && ((Token)tokens.get(i)).getChannel()!=channel ) { 1483447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein i++; 1493447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 1503447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein return i; 1513447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 1523447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein 1533447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein protected int skipOffTokenChannelsReverse(int i) { 1543447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein while ( i>=0 && ((Token)tokens.get(i)).getChannel()!=channel ) { 1553447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein i--; 1563447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 1573447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein return i; 1583447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 1593447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein 1603447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein /** A simple filter mechanism whereby you can tell this token stream 1613447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein * to force all tokens of type ttype to be on channel. For example, 1623447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein * when interpreting, we cannot exec actions so we need to tell 1633447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein * the stream to force all WS and NEWLINE to be a different, ignored 1643447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein * channel. 1653447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein */ 1663447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein public void setTokenTypeChannel(int ttype, int channel) { 1673447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein if ( channelOverrideMap==null ) { 1683447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein channelOverrideMap = new HashMap(); 1693447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 1703447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein channelOverrideMap.put(new Integer(ttype), new Integer(channel)); 1713447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 1723447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein 1733447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein public void discardTokenType(int ttype) { 1743447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein if ( discardSet==null ) { 1753447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein discardSet = new HashSet(); 1763447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 1773447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein discardSet.add(new Integer(ttype)); 1783447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 1793447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein 1803447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein public void discardOffChannelTokens(boolean discardOffChannelTokens) { 1813447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein this.discardOffChannelTokens = discardOffChannelTokens; 1823447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 1833447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein 1843447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein public List getTokens() { 1853447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein if ( p == -1 ) { 1863447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein fillBuffer(); 1873447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 1883447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein return tokens; 1893447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 1903447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein 1913447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein public List getTokens(int start, int stop) { 1923447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein return getTokens(start, stop, (BitSet)null); 1933447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 1943447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein 1953447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein /** Given a start and stop index, return a List of all tokens in 1963447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein * the token type BitSet. Return null if no tokens were found. This 1973447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein * method looks at both on and off channel tokens. 1983447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein */ 1993447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein public List getTokens(int start, int stop, BitSet types) { 2003447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein if ( p == -1 ) { 2013447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein fillBuffer(); 2023447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 2033447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein if ( stop>=tokens.size() ) { 2043447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein stop=tokens.size()-1; 2053447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 2063447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein if ( start<0 ) { 2073447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein start=0; 2083447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 2093447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein if ( start>stop ) { 2103447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein return null; 2113447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 2123447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein 2133447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein // list = tokens[start:stop]:{Token t, t.getType() in types} 2143447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein List filteredTokens = new ArrayList(); 2153447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein for (int i=start; i<=stop; i++) { 2163447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein Token t = (Token)tokens.get(i); 2173447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein if ( types==null || types.member(t.getType()) ) { 2183447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein filteredTokens.add(t); 2193447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 2203447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 2213447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein if ( filteredTokens.size()==0 ) { 2223447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein filteredTokens = null; 2233447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 2243447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein return filteredTokens; 2253447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 2263447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein 2273447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein public List getTokens(int start, int stop, List types) { 2283447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein return getTokens(start,stop,new BitSet(types)); 2293447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 2303447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein 2313447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein public List getTokens(int start, int stop, int ttype) { 2323447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein return getTokens(start,stop,BitSet.of(ttype)); 2333447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 2343447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein 2353447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein /** Get the ith token from the current position 1..n where k=1 is the 2363447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein * first symbol of lookahead. 2373447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein */ 2383447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein public Token LT(int k) { 2393447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein if ( p == -1 ) { 2403447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein fillBuffer(); 2413447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 2423447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein if ( k==0 ) { 2433447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein return null; 2443447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 2453447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein if ( k<0 ) { 2463447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein return LB(-k); 2473447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 2483447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein //System.out.print("LT(p="+p+","+k+")="); 2493447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein if ( (p+k-1) >= tokens.size() ) { 2503447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein return (Token)tokens.get(tokens.size()-1); 2513447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 2523447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein //System.out.println(tokens.get(p+k-1)); 2533447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein int i = p; 2543447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein int n = 1; 2553447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein // find k good tokens 2563447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein while ( n<k ) { 2573447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein // skip off-channel tokens 2583447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein i = skipOffTokenChannels(i+1); // leave p on valid token 2593447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein n++; 2603447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 2613447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein if ( i>=tokens.size() ) { 2623447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein return (Token)tokens.get(tokens.size()-1); // must be EOF 2633447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 2643447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein 2653447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein if ( i>range ) range = i; 2663447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein return (Token)tokens.get(i); 2673447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 2683447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein 2693447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein /** Look backwards k tokens on-channel tokens */ 2703447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein protected Token LB(int k) { 2713447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein //System.out.print("LB(p="+p+","+k+") "); 2723447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein if ( p == -1 ) { 2733447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein fillBuffer(); 2743447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 2753447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein if ( k==0 ) { 2763447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein return null; 2773447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 2783447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein if ( (p-k)<0 ) { 2793447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein return null; 2803447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 2813447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein 2823447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein int i = p; 2833447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein int n = 1; 2843447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein // find k good tokens looking backwards 2853447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein while ( n<=k ) { 2863447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein // skip off-channel tokens 2873447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein i = skipOffTokenChannelsReverse(i-1); // leave p on valid token 2883447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein n++; 2893447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 2903447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein if ( i<0 ) { 2913447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein return null; 2923447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 2933447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein return (Token)tokens.get(i); 2943447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 2953447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein 2963447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein /** Return absolute token i; ignore which channel the tokens are on; 2973447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein * that is, count all tokens not just on-channel tokens. 2983447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein */ 2993447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein public Token get(int i) { 3003447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein return (Token)tokens.get(i); 3013447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 3023447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein 3033447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein /** Get all tokens from start..stop inclusively */ 3043447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein public List get(int start, int stop) { 3053447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein if ( p == -1 ) fillBuffer(); 3063447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein if ( start<0 || stop<0 ) return null; 3073447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein return tokens.subList(start, stop); 3083447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 3093447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein 3103447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein public int LA(int i) { 3113447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein return LT(i).getType(); 3123447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 3133447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein 3143447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein public int mark() { 3153447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein if ( p == -1 ) { 3163447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein fillBuffer(); 3173447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 3183447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein lastMarker = index(); 3193447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein return lastMarker; 3203447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 3213447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein 3223447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein public void release(int marker) { 3233447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein // no resources to release 3243447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 3253447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein 3263447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein public int size() { 3273447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein return tokens.size(); 3283447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 3293447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein 3303447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein public int index() { 3313447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein return p; 3323447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 3333447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein 3343447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein public int range() { 3353447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein return range; 3363447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 3373447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein 3383447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein public void rewind(int marker) { 3393447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein seek(marker); 3403447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 3413447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein 3423447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein public void rewind() { 3433447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein seek(lastMarker); 3443447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 3453447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein 3463447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein public void reset() { 3473447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein p = 0; 3483447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein lastMarker = 0; 3493447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 3503447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein 3513447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein public void seek(int index) { 3523447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein p = index; 3533447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 3543447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein 3553447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein public TokenSource getTokenSource() { 3563447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein return tokenSource; 3573447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 3583447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein 3593447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein public String getSourceName() { 3603447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein return getTokenSource().getSourceName(); 3613447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 3623447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein 3633447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein public String toString() { 3643447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein if ( p == -1 ) { 3653447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein fillBuffer(); 3663447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 3673447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein return toString(0, tokens.size()-1); 3683447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 3693447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein 3703447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein public String toString(int start, int stop) { 3713447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein if ( start<0 || stop<0 ) { 3723447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein return null; 3733447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 3743447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein if ( p == -1 ) { 3753447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein fillBuffer(); 3763447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 3773447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein if ( stop>=tokens.size() ) { 3783447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein stop = tokens.size()-1; 3793447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 3803447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein StringBuffer buf = new StringBuffer(); 3813447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein for (int i = start; i <= stop; i++) { 3823447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein Token t = (Token)tokens.get(i); 3833447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein buf.append(t.getText()); 3843447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 3853447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein return buf.toString(); 3863447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 3873447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein 3883447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein public String toString(Token start, Token stop) { 3893447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein if ( start!=null && stop!=null ) { 3903447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein return toString(start.getTokenIndex(), stop.getTokenIndex()); 3913447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 3923447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein return null; 3933447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 3943447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein} 395