13447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein/* 23447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein [The "BSD license"] 33447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein Copyright (c) 2005-2009 Terence Parr 43447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein All rights reserved. 53447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein 63447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein Redistribution and use in source and binary forms, with or without 73447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein modification, are permitted provided that the following conditions 83447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein are met: 93447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein 1. Redistributions of source code must retain the above copyright 103447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein notice, this list of conditions and the following disclaimer. 113447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein 2. Redistributions in binary form must reproduce the above copyright 123447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein notice, this list of conditions and the following disclaimer in the 133447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein documentation and/or other materials provided with the distribution. 143447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein 3. The name of the author may not be used to endorse or promote products 153447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein derived from this software without specific prior written permission. 163447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein 173447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 183447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 193447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 203447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 213447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 223447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 233447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 243447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 253447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 263447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 273447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein */ 283447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein 293447a5916aa62f44de24cc441fc9987116ddff52Andrew Sappersteinpackage org.antlr.runtime; 303447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein 313447a5916aa62f44de24cc441fc9987116ddff52Andrew Sappersteinimport java.util.List; 323447a5916aa62f44de24cc441fc9987116ddff52Andrew Sappersteinimport java.util.ArrayList; 333447a5916aa62f44de24cc441fc9987116ddff52Andrew Sappersteinimport java.util.NoSuchElementException; 343447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein 353447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein/** Buffer all input tokens but do on-demand fetching of new tokens from 363447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein * lexer. Useful when the parser or lexer has to set context/mode info before 373447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein * proper lexing of future tokens. The ST template parser needs this, 383447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein * for example, because it has to constantly flip back and forth between 393447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein * inside/output templates. E.g., <names:{hi, <it>}> has to parse names 403447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein * as part of an expression but "hi, <it>" as a nested template. 413447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein * 423447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein * You can't use this stream if you pass whitespace or other off-channel 433447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein * tokens to the parser. The stream can't ignore off-channel tokens. 443447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein * (UnbufferedTokenStream is the same way.) 453447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein * 463447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein * This is not a subclass of UnbufferedTokenStream because I don't want 473447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein * to confuse small moving window of tokens it uses for the full buffer. 483447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein */ 493447a5916aa62f44de24cc441fc9987116ddff52Andrew Sappersteinpublic class BufferedTokenStream implements TokenStream { 503447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein protected TokenSource tokenSource; 513447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein 523447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein /** Record every single token pulled from the source so we can reproduce 533447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein * chunks of it later. The buffer in LookaheadStream overlaps sometimes 543447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein * as its moving window moves through the input. This list captures 553447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein * everything so we can access complete input text. 563447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein */ 573447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein protected List<Token> tokens = new ArrayList<Token>(100); 583447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein 593447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein /** Track the last mark() call result value for use in rewind(). */ 603447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein protected int lastMarker; 613447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein 623447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein /** The index into the tokens list of the current token (next token 633447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein * to consume). tokens[p] should be LT(1). p=-1 indicates need 643447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein * to initialize with first token. The ctor doesn't get a token. 653447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein * First call to LT(1) or whatever gets the first token and sets p=0; 663447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein */ 673447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein protected int p = -1; 683447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein 693447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein protected int range = -1; // how deep have we gone? 703447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein 713447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein public BufferedTokenStream() {;} 723447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein 733447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein public BufferedTokenStream(TokenSource tokenSource) { 743447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein this.tokenSource = tokenSource; 753447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 763447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein 773447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein public TokenSource getTokenSource() { return tokenSource; } 783447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein 793447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein public int index() { return p; } 803447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein 813447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein public int range() { return range; } 823447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein 833447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein public int mark() { 843447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein if ( p == -1 ) setup(); 853447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein lastMarker = index(); 863447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein return lastMarker; 873447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 883447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein 893447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein public void release(int marker) { 903447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein // no resources to release 913447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 923447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein 933447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein public void rewind(int marker) { 943447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein seek(marker); 953447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 963447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein 973447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein public void rewind() { 983447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein seek(lastMarker); 993447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 1003447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein 1013447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein public void reset() { 1023447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein p = 0; 1033447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein lastMarker = 0; 1043447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 1053447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein 1063447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein public void seek(int index) { p = index; } 1073447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein 1083447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein public int size() { return tokens.size(); } 1093447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein 1103447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein /** Move the input pointer to the next incoming token. The stream 1113447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein * must become active with LT(1) available. consume() simply 1123447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein * moves the input pointer so that LT(1) points at the next 1133447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein * input symbol. Consume at least one token. 1143447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein * 1153447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein * Walk past any token not on the channel the parser is listening to. 1163447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein */ 1173447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein public void consume() { 1183447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein if ( p == -1 ) setup(); 1193447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein p++; 1203447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein sync(p); 1213447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 1223447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein 1233447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein /** Make sure index i in tokens has a token. */ 1243447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein protected void sync(int i) { 1253447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein int n = i - tokens.size() + 1; // how many more elements we need? 1263447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein //System.out.println("sync("+i+") needs "+n); 1273447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein if ( n > 0 ) fetch(n); 1283447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 1293447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein 1303447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein /** add n elements to buffer */ 1313447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein protected void fetch(int n) { 1323447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein for (int i=1; i<=n; i++) { 1333447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein Token t = tokenSource.nextToken(); 1343447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein t.setTokenIndex(tokens.size()); 1353447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein //System.out.println("adding "+t+" at index "+tokens.size()); 1363447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein tokens.add(t); 1373447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein if ( t.getType()==Token.EOF ) break; 1383447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 1393447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 1403447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein 1413447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein public Token get(int i) { 1423447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein if ( i < 0 || i >= tokens.size() ) { 1433447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein throw new NoSuchElementException("token index "+i+" out of range 0.."+(tokens.size()-1)); 1443447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 1453447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein return tokens.get(i); 1463447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 1473447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein 1483447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein /** Get all tokens from start..stop inclusively */ 1493447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein public List get(int start, int stop) { 1503447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein if ( start<0 || stop<0 ) return null; 1513447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein if ( p == -1 ) setup(); 1523447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein List subset = new ArrayList(); 1533447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein if ( stop>=tokens.size() ) stop = tokens.size()-1; 1543447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein for (int i = start; i <= stop; i++) { 1553447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein Token t = tokens.get(i); 1563447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein if ( t.getType()==Token.EOF ) break; 1573447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein subset.add(t); 1583447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 1593447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein return subset; 1603447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 1613447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein 1623447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein public int LA(int i) { return LT(i).getType(); } 1633447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein 1643447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein protected Token LB(int k) { 1653447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein if ( (p-k)<0 ) return null; 1663447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein return tokens.get(p-k); 1673447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 1683447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein 1693447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein public Token LT(int k) { 1703447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein if ( p == -1 ) setup(); 1713447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein if ( k==0 ) return null; 1723447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein if ( k < 0 ) return LB(-k); 1733447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein 1743447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein int i = p + k - 1; 1753447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein sync(i); 1763447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein if ( i >= tokens.size() ) { // return EOF token 1773447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein // EOF must be last token 1783447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein return tokens.get(tokens.size()-1); 1793447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 1803447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein if ( i>range ) range = i; 1813447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein return tokens.get(i); 1823447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 1833447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein 1843447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein protected void setup() { sync(0); p = 0; } 1853447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein 1863447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein /** Reset this token stream by setting its token source. */ 1873447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein public void setTokenSource(TokenSource tokenSource) { 1883447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein this.tokenSource = tokenSource; 1893447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein tokens.clear(); 1903447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein p = -1; 1913447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 1923447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein 1933447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein public List getTokens() { return tokens; } 1943447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein 1953447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein public List getTokens(int start, int stop) { 1963447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein return getTokens(start, stop, (BitSet)null); 1973447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 1983447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein 1993447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein /** Given a start and stop index, return a List of all tokens in 2003447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein * the token type BitSet. Return null if no tokens were found. This 2013447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein * method looks at both on and off channel tokens. 2023447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein */ 2033447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein public List getTokens(int start, int stop, BitSet types) { 2043447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein if ( p == -1 ) setup(); 2053447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein if ( stop>=tokens.size() ) stop=tokens.size()-1; 2063447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein if ( start<0 ) start=0; 2073447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein if ( start>stop ) return null; 2083447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein 2093447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein // list = tokens[start:stop]:{Token t, t.getType() in types} 2103447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein List<Token> filteredTokens = new ArrayList<Token>(); 2113447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein for (int i=start; i<=stop; i++) { 2123447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein Token t = tokens.get(i); 2133447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein if ( types==null || types.member(t.getType()) ) { 2143447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein filteredTokens.add(t); 2153447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 2163447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 2173447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein if ( filteredTokens.size()==0 ) { 2183447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein filteredTokens = null; 2193447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 2203447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein return filteredTokens; 2213447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 2223447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein 2233447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein public List getTokens(int start, int stop, List types) { 2243447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein return getTokens(start,stop,new BitSet(types)); 2253447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 2263447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein 2273447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein public List getTokens(int start, int stop, int ttype) { 2283447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein return getTokens(start,stop,BitSet.of(ttype)); 2293447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 2303447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein 2313447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein public String getSourceName() { return tokenSource.getSourceName(); } 2323447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein 2333447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein /** Grab *all* tokens from stream and return string */ 2343447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein public String toString() { 2353447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein if ( p == -1 ) setup(); 2363447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein fill(); 2373447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein return toString(0, tokens.size()-1); 2383447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 2393447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein 2403447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein public String toString(int start, int stop) { 2413447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein if ( start<0 || stop<0 ) return null; 2423447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein if ( p == -1 ) setup(); 2433447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein if ( stop>=tokens.size() ) stop = tokens.size()-1; 2443447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein StringBuffer buf = new StringBuffer(); 2453447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein for (int i = start; i <= stop; i++) { 2463447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein Token t = tokens.get(i); 2473447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein if ( t.getType()==Token.EOF ) break; 2483447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein buf.append(t.getText()); 2493447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 2503447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein return buf.toString(); 2513447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 2523447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein 2533447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein public String toString(Token start, Token stop) { 2543447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein if ( start!=null && stop!=null ) { 2553447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein return toString(start.getTokenIndex(), stop.getTokenIndex()); 2563447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 2573447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein return null; 2583447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 2593447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein 2603447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein /** Get all tokens from lexer until EOF */ 2613447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein public void fill() { 2623447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein if ( p == -1 ) setup(); 2633447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein if ( tokens.get(p).getType()==Token.EOF ) return; 2643447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein 2653447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein int i = p+1; 2663447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein sync(i); 2673447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein while ( tokens.get(i).getType()!=Token.EOF ) { 2683447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein i++; 2693447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein sync(i); 2703447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 2713447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein } 2723447a5916aa62f44de24cc441fc9987116ddff52Andrew Sapperstein} 273