1/* 2 [The "BSD licence"] 3 Copyright (c) 2005-2006 Terence Parr 4 All rights reserved. 5 6 Redistribution and use in source and binary forms, with or without 7 modification, are permitted provided that the following conditions 8 are met: 9 1. Redistributions of source code must retain the above copyright 10 notice, this list of conditions and the following disclaimer. 11 2. Redistributions in binary form must reproduce the above copyright 12 notice, this list of conditions and the following disclaimer in the 13 documentation and/or other materials provided with the distribution. 14 3. The name of the author may not be used to endorse or promote products 15 derived from this software without specific prior written permission. 16 17 THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 18 IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 19 OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 20 IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 21 INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 22 NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 23 DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 24 THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 25 (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 26 THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27*/ 28package org.antlr.runtime { 29 30 /** The most common stream of tokens is one where every token is buffered up 31 * and tokens are prefiltered for a certain channel (the parser will only 32 * see these tokens and cannot change the filter channel number during the 33 * parse). 34 * 35 * TODO: how to access the full token stream? How to track all tokens matched per rule? 36 */ 37 public class CommonTokenStream implements TokenStream { 38 protected var _tokenSource:TokenSource; 39 40 /** Record every single token pulled from the source so we can reproduce 41 * chunks of it later. 42 */ 43 protected var tokens:Array = new Array(); 44 45 /** Map<tokentype, channel> to override some Tokens' channel numbers */ 46 protected var channelOverrideMap:Array; 47 48 /** Set<tokentype>; discard any tokens with this type */ 49 protected var discardSet:Array; 50 51 /** Skip tokens on any channel but this one; this is how we skip whitespace... */ 52 protected var channel:int = TokenConstants.DEFAULT_CHANNEL; 53 54 /** By default, track all incoming tokens */ 55 protected var _discardOffChannelTokens:Boolean = false; 56 57 /** Track the last mark() call result value for use in rewind(). */ 58 protected var lastMarker:int; 59 60 /** The index into the tokens list of the current token (next token 61 * to consume). p==-1 indicates that the tokens list is empty 62 */ 63 protected var p:int = -1; 64 65 public function CommonTokenStream(tokenSource:TokenSource = null, channel:int = TokenConstants.DEFAULT_CHANNEL) { 66 _tokenSource = tokenSource; 67 this.channel = channel; 68 } 69 70 /** Reset this token stream by setting its token source. */ 71 public function set tokenSource(tokenSource:TokenSource):void { 72 _tokenSource = tokenSource; 73 tokens.clear(); 74 p = -1; 75 channel = TokenConstants.DEFAULT_CHANNEL; 76 } 77 78 /** Load all tokens from the token source and put in tokens. 79 * This is done upon first LT request because you might want to 80 * set some token type / channel overrides before filling buffer. 81 */ 82 protected function fillBuffer():void { 83 var index:int = 0; 84 var t:Token = tokenSource.nextToken(); 85 while ( t!=null && t.type != CharStreamConstants.EOF ) { 86 var discard:Boolean = false; 87 // is there a channel override for token type? 88 if ( channelOverrideMap != null ) { 89 if (channelOverrideMap[t.type] != undefined) { 90 t.channel = channelOverrideMap[t.type]; 91 } 92 } 93 if ( discardSet !=null && 94 discardSet[t.type] == true ) 95 { 96 discard = true; 97 } 98 else if ( _discardOffChannelTokens && t.channel != this.channel ) { 99 discard = true; 100 } 101 if ( !discard ) { 102 t.tokenIndex = index; 103 tokens.push(t); 104 index++; 105 } 106 t = tokenSource.nextToken(); 107 } 108 // leave p pointing at first token on channel 109 p = 0; 110 p = skipOffTokenChannels(p); 111 } 112 113 /** Move the input pointer to the next incoming token. The stream 114 * must become active with LT(1) available. consume() simply 115 * moves the input pointer so that LT(1) points at the next 116 * input symbol. Consume at least one token. 117 * 118 * Walk past any token not on the channel the parser is listening to. 119 */ 120 public function consume():void { 121 if ( p<tokens.length ) { 122 p++; 123 p = skipOffTokenChannels(p); // leave p on valid token 124 } 125 } 126 127 /** Given a starting index, return the index of the first on-channel 128 * token. 129 */ 130 protected function skipOffTokenChannels(i:int):int { 131 var n:int = tokens.length; 132 while ( i<n && (Token(tokens[i])).channel != channel) { 133 i++; 134 } 135 return i; 136 } 137 138 protected function skipOffTokenChannelsReverse(i:int):int { 139 while ( i>= 0 && (Token(tokens[i])).channel != channel) { 140 i--; 141 } 142 return i; 143 } 144 145 /** A simple filter mechanism whereby you can tell this token stream 146 * to force all tokens of type ttype to be on channel. For example, 147 * when interpreting, we cannot exec actions so we need to tell 148 * the stream to force all WS and NEWLINE to be a different, ignored 149 * channel. 150 */ 151 public function setTokenTypeChannel(ttype:int, channel:int):void { 152 if ( channelOverrideMap==null ) { 153 channelOverrideMap = new Array(); 154 } 155 channelOverrideMap[ttype] = channel; 156 } 157 158 public function discardTokenType(ttype:int):void { 159 if ( discardSet==null ) { 160 discardSet = new Array(); 161 } 162 discardSet[ttype] = true; 163 } 164 165 public function discardOffChannelTokens(discardOffChannelTokens:Boolean):void { 166 _discardOffChannelTokens = discardOffChannelTokens; 167 } 168 169 public function getTokens():Array { 170 if ( p == -1 ) { 171 fillBuffer(); 172 } 173 return tokens; 174 } 175 176 public function getTokensRange(start:int, stop:int):Array { 177 return getTokensBitSet(start, stop, null); 178 } 179 180 /** Given a start and stop index, return a List of all tokens in 181 * the token type BitSet. Return null if no tokens were found. This 182 * method looks at both on and off channel tokens. 183 * 184 * Renamed from getTokens 185 */ 186 public function getTokensBitSet(start:int, stop:int, types:BitSet):Array { 187 if ( p == -1 ) { 188 fillBuffer(); 189 } 190 if ( stop>=tokens.length ) { 191 stop=tokens.length-1; 192 } 193 if ( start<0 ) { 194 start=0; 195 } 196 if ( start>stop ) { 197 return null; 198 } 199 200 // list = tokens[start:stop]:{Token t, t.getType() in types} 201 var filteredTokens:Array = new Array(); 202 for (var i:int=start; i<=stop; i++) { 203 var t:Token = Token(tokens[i]); 204 if ( types==null || types.member(t.type) ) { 205 filteredTokens.push(t); 206 } 207 } 208 if ( filteredTokens.length==0 ) { 209 filteredTokens = null; 210 } 211 return filteredTokens; 212 } 213 214 public function getTokensArray(start:int, stop:int, types:Array):Array { 215 return getTokensBitSet(start,stop,new BitSet(types)); 216 } 217 218 public function getTokensInt(start:int, stop:int, ttype:int):Array { 219 return getTokensBitSet(start,stop,BitSet.of(ttype)); 220 } 221 222 /** Get the ith token from the current position 1..n where k=1 is the 223 * first symbol of lookahead. 224 */ 225 public function LT(k:int):Token { 226 if ( p == -1 ) { 227 fillBuffer(); 228 } 229 if ( k==0 ) { 230 return null; 231 } 232 if ( k<0 ) { 233 return LB(-k); 234 } 235 //System.out.print("LT(p="+p+","+k+")="); 236 if ( (p+k-1) >= tokens.length ) { 237 return TokenConstants.EOF_TOKEN; 238 } 239 //System.out.println(tokens.get(p+k-1)); 240 var i:int = p; 241 var n:int = 1; 242 // find k good tokens 243 while ( n<k ) { 244 // skip off-channel tokens 245 i = skipOffTokenChannels(i+1); // leave p on valid token 246 n++; 247 } 248 if ( i>=tokens.length ) { 249 return TokenConstants.EOF_TOKEN; 250 } 251 return Token(tokens[i]); 252 } 253 254 /** Look backwards k tokens on-channel tokens */ 255 protected function LB(k:int):Token { 256 //System.out.print("LB(p="+p+","+k+") "); 257 if ( p == -1 ) { 258 fillBuffer(); 259 } 260 if ( k==0 ) { 261 return null; 262 } 263 if ( (p-k)<0 ) { 264 return null; 265 } 266 267 var i:int = p; 268 var n:int = 1; 269 // find k good tokens looking backwards 270 while ( n<=k ) { 271 // skip off-channel tokens 272 i = skipOffTokenChannelsReverse(i-1); // leave p on valid token 273 n++; 274 } 275 if ( i<0 ) { 276 return null; 277 } 278 return Token(tokens[i]); 279 } 280 281 /** Return absolute token i; ignore which channel the tokens are on; 282 * that is, count all tokens not just on-channel tokens. 283 */ 284 public function getToken(i:int):Token { 285 return Token(tokens[i]); 286 } 287 288 public function LA(i:int):int { 289 return LT(i).type; 290 } 291 292 public function mark():int { 293 if ( p == -1 ) { 294 fillBuffer(); 295 } 296 lastMarker = index; 297 return lastMarker; 298 } 299 300 public function release(marker:int):void { 301 // no resources to release 302 } 303 304 public function get size():int { 305 return tokens.length; 306 } 307 308 public function get index():int { 309 return p; 310 } 311 312 public function reset():void { 313 p = 0; 314 lastMarker = 0; 315 } 316 317 public function rewindTo(marker:int):void { 318 seek(marker); 319 } 320 321 public function rewind():void { 322 seek(lastMarker); 323 } 324 325 public function seek(index:int):void { 326 p = index; 327 } 328 329 public function get tokenSource():TokenSource { 330 return _tokenSource; 331 } 332 333 public function get sourceName():String { 334 return tokenSource.sourceName; 335 } 336 337 public function toString():String { 338 if ( p == -1 ) { 339 fillBuffer(); 340 } 341 return toStringWithRange(0, tokens.length-1); 342 } 343 344 public function toStringWithRange(start:int, stop:int):String { 345 if ( start<0 || stop<0 ) { 346 return null; 347 } 348 if ( p == -1 ) { 349 fillBuffer(); 350 } 351 if ( stop>=tokens.length ) { 352 stop = tokens.length-1; 353 } 354 var buf:String = ""; 355 for (var i:int = start; i <= stop; i++) { 356 var t:Token = Token(tokens[i]); 357 buf += t.text; 358 } 359 return buf.toString(); 360 } 361 362 public function toStringWithTokenRange(start:Token, stop:Token):String { 363 if ( start!=null && stop!=null ) { 364 return toStringWithRange(start.tokenIndex, stop.tokenIndex); 365 } 366 return null; 367 } 368 } 369 370 371}