1/* 2 [The "BSD license"] 3 Copyright (c) 2005-2009 Terence Parr 4 All rights reserved. 5 6 Redistribution and use in source and binary forms, with or without 7 modification, are permitted provided that the following conditions 8 are met: 9 1. Redistributions of source code must retain the above copyright 10 notice, this list of conditions and the following disclaimer. 11 2. Redistributions in binary form must reproduce the above copyright 12 notice, this list of conditions and the following disclaimer in the 13 documentation and/or other materials provided with the distribution. 14 3. The name of the author may not be used to endorse or promote products 15 derived from this software without specific prior written permission. 16 17 THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 18 IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 19 OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 20 IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 21 INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 22 NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 23 DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 24 THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 25 (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 26 THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 */ 28 29package org.antlr.runtime; 30 31import java.util.List; 32import java.util.ArrayList; 33import java.util.NoSuchElementException; 34 35/** Buffer all input tokens but do on-demand fetching of new tokens from 36 * lexer. Useful when the parser or lexer has to set context/mode info before 37 * proper lexing of future tokens. The ST template parser needs this, 38 * for example, because it has to constantly flip back and forth between 39 * inside/output templates. E.g., <names:{hi, <it>}> has to parse names 40 * as part of an expression but "hi, <it>" as a nested template. 41 * 42 * You can't use this stream if you pass whitespace or other off-channel 43 * tokens to the parser. The stream can't ignore off-channel tokens. 44 * (UnbufferedTokenStream is the same way.) 45 * 46 * This is not a subclass of UnbufferedTokenStream because I don't want 47 * to confuse small moving window of tokens it uses for the full buffer. 48 */ 49public class BufferedTokenStream implements TokenStream { 50 protected TokenSource tokenSource; 51 52 /** Record every single token pulled from the source so we can reproduce 53 * chunks of it later. The buffer in LookaheadStream overlaps sometimes 54 * as its moving window moves through the input. This list captures 55 * everything so we can access complete input text. 56 */ 57 protected List<Token> tokens = new ArrayList<Token>(100); 58 59 /** Track the last mark() call result value for use in rewind(). */ 60 protected int lastMarker; 61 62 /** The index into the tokens list of the current token (next token 63 * to consume). tokens[p] should be LT(1). p=-1 indicates need 64 * to initialize with first token. The ctor doesn't get a token. 65 * First call to LT(1) or whatever gets the first token and sets p=0; 66 */ 67 protected int p = -1; 68 69 protected int range = -1; // how deep have we gone? 70 71 public BufferedTokenStream() {;} 72 73 public BufferedTokenStream(TokenSource tokenSource) { 74 this.tokenSource = tokenSource; 75 } 76 77 public TokenSource getTokenSource() { return tokenSource; } 78 79 public int index() { return p; } 80 81 public int range() { return range; } 82 83 public int mark() { 84 if ( p == -1 ) setup(); 85 lastMarker = index(); 86 return lastMarker; 87 } 88 89 public void release(int marker) { 90 // no resources to release 91 } 92 93 public void rewind(int marker) { 94 seek(marker); 95 } 96 97 public void rewind() { 98 seek(lastMarker); 99 } 100 101 public void reset() { 102 p = 0; 103 lastMarker = 0; 104 } 105 106 public void seek(int index) { p = index; } 107 108 public int size() { return tokens.size(); } 109 110 /** Move the input pointer to the next incoming token. The stream 111 * must become active with LT(1) available. consume() simply 112 * moves the input pointer so that LT(1) points at the next 113 * input symbol. Consume at least one token. 114 * 115 * Walk past any token not on the channel the parser is listening to. 116 */ 117 public void consume() { 118 if ( p == -1 ) setup(); 119 p++; 120 sync(p); 121 } 122 123 /** Make sure index i in tokens has a token. */ 124 protected void sync(int i) { 125 int n = i - tokens.size() + 1; // how many more elements we need? 126 //System.out.println("sync("+i+") needs "+n); 127 if ( n > 0 ) fetch(n); 128 } 129 130 /** add n elements to buffer */ 131 protected void fetch(int n) { 132 for (int i=1; i<=n; i++) { 133 Token t = tokenSource.nextToken(); 134 t.setTokenIndex(tokens.size()); 135 //System.out.println("adding "+t+" at index "+tokens.size()); 136 tokens.add(t); 137 if ( t.getType()==Token.EOF ) break; 138 } 139 } 140 141 public Token get(int i) { 142 if ( i < 0 || i >= tokens.size() ) { 143 throw new NoSuchElementException("token index "+i+" out of range 0.."+(tokens.size()-1)); 144 } 145 return tokens.get(i); 146 } 147 148 /** Get all tokens from start..stop inclusively */ 149 public List get(int start, int stop) { 150 if ( start<0 || stop<0 ) return null; 151 if ( p == -1 ) setup(); 152 List subset = new ArrayList(); 153 if ( stop>=tokens.size() ) stop = tokens.size()-1; 154 for (int i = start; i <= stop; i++) { 155 Token t = tokens.get(i); 156 if ( t.getType()==Token.EOF ) break; 157 subset.add(t); 158 } 159 return subset; 160 } 161 162 public int LA(int i) { return LT(i).getType(); } 163 164 protected Token LB(int k) { 165 if ( (p-k)<0 ) return null; 166 return tokens.get(p-k); 167 } 168 169 public Token LT(int k) { 170 if ( p == -1 ) setup(); 171 if ( k==0 ) return null; 172 if ( k < 0 ) return LB(-k); 173 174 int i = p + k - 1; 175 sync(i); 176 if ( i >= tokens.size() ) { // return EOF token 177 // EOF must be last token 178 return tokens.get(tokens.size()-1); 179 } 180 if ( i>range ) range = i; 181 return tokens.get(i); 182 } 183 184 protected void setup() { sync(0); p = 0; } 185 186 /** Reset this token stream by setting its token source. */ 187 public void setTokenSource(TokenSource tokenSource) { 188 this.tokenSource = tokenSource; 189 tokens.clear(); 190 p = -1; 191 } 192 193 public List getTokens() { return tokens; } 194 195 public List getTokens(int start, int stop) { 196 return getTokens(start, stop, (BitSet)null); 197 } 198 199 /** Given a start and stop index, return a List of all tokens in 200 * the token type BitSet. Return null if no tokens were found. This 201 * method looks at both on and off channel tokens. 202 */ 203 public List getTokens(int start, int stop, BitSet types) { 204 if ( p == -1 ) setup(); 205 if ( stop>=tokens.size() ) stop=tokens.size()-1; 206 if ( start<0 ) start=0; 207 if ( start>stop ) return null; 208 209 // list = tokens[start:stop]:{Token t, t.getType() in types} 210 List<Token> filteredTokens = new ArrayList<Token>(); 211 for (int i=start; i<=stop; i++) { 212 Token t = tokens.get(i); 213 if ( types==null || types.member(t.getType()) ) { 214 filteredTokens.add(t); 215 } 216 } 217 if ( filteredTokens.size()==0 ) { 218 filteredTokens = null; 219 } 220 return filteredTokens; 221 } 222 223 public List getTokens(int start, int stop, List types) { 224 return getTokens(start,stop,new BitSet(types)); 225 } 226 227 public List getTokens(int start, int stop, int ttype) { 228 return getTokens(start,stop,BitSet.of(ttype)); 229 } 230 231 public String getSourceName() { return tokenSource.getSourceName(); } 232 233 /** Grab *all* tokens from stream and return string */ 234 public String toString() { 235 if ( p == -1 ) setup(); 236 fill(); 237 return toString(0, tokens.size()-1); 238 } 239 240 public String toString(int start, int stop) { 241 if ( start<0 || stop<0 ) return null; 242 if ( p == -1 ) setup(); 243 if ( stop>=tokens.size() ) stop = tokens.size()-1; 244 StringBuffer buf = new StringBuffer(); 245 for (int i = start; i <= stop; i++) { 246 Token t = tokens.get(i); 247 if ( t.getType()==Token.EOF ) break; 248 buf.append(t.getText()); 249 } 250 return buf.toString(); 251 } 252 253 public String toString(Token start, Token stop) { 254 if ( start!=null && stop!=null ) { 255 return toString(start.getTokenIndex(), stop.getTokenIndex()); 256 } 257 return null; 258 } 259 260 /** Get all tokens from lexer until EOF */ 261 public void fill() { 262 if ( p == -1 ) setup(); 263 if ( tokens.get(p).getType()==Token.EOF ) return; 264 265 int i = p+1; 266 sync(i); 267 while ( tokens.get(i).getType()!=Token.EOF ) { 268 i++; 269 sync(i); 270 } 271 } 272} 273