1/*
2 [The "BSD licence"]
3 Copyright (c) 2005-2006 Terence Parr
4 All rights reserved.
5
6 Redistribution and use in source and binary forms, with or without
7 modification, are permitted provided that the following conditions
8 are met:
9 1. Redistributions of source code must retain the above copyright
10    notice, this list of conditions and the following disclaimer.
11 2. Redistributions in binary form must reproduce the above copyright
12    notice, this list of conditions and the following disclaimer in the
13    documentation and/or other materials provided with the distribution.
14 3. The name of the author may not be used to endorse or promote products
15    derived from this software without specific prior written permission.
16
17 THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
18 IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
19 OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
20 IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
21 INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
22 NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23 DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24 THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25 (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
26 THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27*/
28package org.antlr.runtime {
29
30	/** The most common stream of tokens is one where every token is buffered up
31	 *  and tokens are prefiltered for a certain channel (the parser will only
32	 *  see these tokens and cannot change the filter channel number during the
33	 *  parse).
34	 *
35	 *  TODO: how to access the full token stream?  How to track all tokens matched per rule?
36	 */
37	public class CommonTokenStream implements TokenStream {
38	    protected var _tokenSource:TokenSource;
39
40		/** Record every single token pulled from the source so we can reproduce
41		 *  chunks of it later.
42		 */
43		protected var tokens:Array = new Array();
44
45		/** Map<tokentype, channel> to override some Tokens' channel numbers */
46		protected var channelOverrideMap:Array;
47
48		/** Set<tokentype>; discard any tokens with this type */
49		protected var discardSet:Array;
50
51		/** Skip tokens on any channel but this one; this is how we skip whitespace... */
52		protected var channel:int = TokenConstants.DEFAULT_CHANNEL;
53
54		/** By default, track all incoming tokens */
55		protected var _discardOffChannelTokens:Boolean = false;
56
57		/** Track the last mark() call result value for use in rewind(). */
58		protected var lastMarker:int;
59
60		/** The index into the tokens list of the current token (next token
61	     *  to consume).  p==-1 indicates that the tokens list is empty
62	     */
63	    protected var p:int = -1;
64
65		public function CommonTokenStream(tokenSource:TokenSource = null, channel:int = TokenConstants.DEFAULT_CHANNEL) {
66			_tokenSource = tokenSource;
67			this.channel = channel;
68		}
69
70		/** Reset this token stream by setting its token source. */
71		public function set tokenSource(tokenSource:TokenSource):void {
72			_tokenSource = tokenSource;
73			tokens.clear();
74			p = -1;
75			channel = TokenConstants.DEFAULT_CHANNEL;
76		}
77
78		/** Load all tokens from the token source and put in tokens.
79		 *  This is done upon first LT request because you might want to
80		 *  set some token type / channel overrides before filling buffer.
81		 */
82		protected function fillBuffer():void {
83			var index:int = 0;
84			var t:Token = tokenSource.nextToken();
85			while ( t!=null && t.type != CharStreamConstants.EOF ) {
86				var discard:Boolean = false;
87				// is there a channel override for token type?
88				if ( channelOverrideMap != null ) {
89					if (channelOverrideMap[t.type] != undefined) {
90						t.channel = channelOverrideMap[t.type];
91					}
92				}
93				if ( discardSet !=null &&
94					 discardSet[t.type] == true )
95				{
96					discard = true;
97				}
98				else if ( _discardOffChannelTokens && t.channel != this.channel ) {
99					discard = true;
100				}
101				if ( !discard )	{
102					t.tokenIndex = index;
103					tokens.push(t);
104					index++;
105				}
106				t = tokenSource.nextToken();
107			}
108			// leave p pointing at first token on channel
109			p = 0;
110			p = skipOffTokenChannels(p);
111	    }
112
113		/** Move the input pointer to the next incoming token.  The stream
114		 *  must become active with LT(1) available.  consume() simply
115		 *  moves the input pointer so that LT(1) points at the next
116		 *  input symbol. Consume at least one token.
117		 *
118		 *  Walk past any token not on the channel the parser is listening to.
119		 */
120		public function consume():void {
121			if ( p<tokens.length ) {
122	            p++;
123				p = skipOffTokenChannels(p); // leave p on valid token
124	        }
125	    }
126
127		/** Given a starting index, return the index of the first on-channel
128		 *  token.
129		 */
130		protected function skipOffTokenChannels(i:int):int {
131			var n:int = tokens.length;
132			while ( i<n && (Token(tokens[i])).channel != channel) {
133				i++;
134			}
135			return i;
136		}
137
138		protected function skipOffTokenChannelsReverse(i:int):int {
139			while ( i>= 0 && (Token(tokens[i])).channel != channel) {
140				i--;
141			}
142			return i;
143		}
144
145		/** A simple filter mechanism whereby you can tell this token stream
146		 *  to force all tokens of type ttype to be on channel.  For example,
147		 *  when interpreting, we cannot exec actions so we need to tell
148		 *  the stream to force all WS and NEWLINE to be a different, ignored
149		 *  channel.
150		 */
151		public function setTokenTypeChannel(ttype:int, channel:int):void {
152			if ( channelOverrideMap==null ) {
153				channelOverrideMap = new Array();
154			}
155	        channelOverrideMap[ttype] = channel;
156		}
157
158		public function discardTokenType(ttype:int):void {
159			if ( discardSet==null ) {
160				discardSet = new Array();
161			}
162	        discardSet[ttype] = true;
163		}
164
165		public function discardOffChannelTokens(discardOffChannelTokens:Boolean):void {
166			_discardOffChannelTokens = discardOffChannelTokens;
167		}
168
169		public function getTokens():Array {
170			if ( p == -1 ) {
171				fillBuffer();
172			}
173			return tokens;
174		}
175
176		public function getTokensRange(start:int, stop:int):Array {
177			return getTokensBitSet(start, stop, null);
178		}
179
180		/** Given a start and stop index, return a List of all tokens in
181		 *  the token type BitSet.  Return null if no tokens were found.  This
182		 *  method looks at both on and off channel tokens.
183		 *
184		 * Renamed from getTokens
185		 */
186		public function getTokensBitSet(start:int, stop:int, types:BitSet):Array {
187			if ( p == -1 ) {
188				fillBuffer();
189			}
190			if ( stop>=tokens.length ) {
191				stop=tokens.length-1;
192			}
193			if ( start<0 ) {
194				start=0;
195			}
196			if ( start>stop ) {
197				return null;
198			}
199
200			// list = tokens[start:stop]:{Token t, t.getType() in types}
201			var filteredTokens:Array = new Array();
202			for (var i:int=start; i<=stop; i++) {
203				var t:Token = Token(tokens[i]);
204				if ( types==null || types.member(t.type) ) {
205					filteredTokens.push(t);
206				}
207			}
208			if ( filteredTokens.length==0 ) {
209				filteredTokens = null;
210			}
211			return filteredTokens;
212		}
213
214		public function getTokensArray(start:int, stop:int, types:Array):Array {
215			return getTokensBitSet(start,stop,new BitSet(types));
216		}
217
218		public function getTokensInt(start:int, stop:int, ttype:int):Array {
219			return getTokensBitSet(start,stop,BitSet.of(ttype));
220		}
221
222		/** Get the ith token from the current position 1..n where k=1 is the
223		 *  first symbol of lookahead.
224		 */
225		public function LT(k:int):Token {
226			if ( p == -1 ) {
227				fillBuffer();
228			}
229			if ( k==0 ) {
230				return null;
231			}
232			if ( k<0 ) {
233				return LB(-k);
234			}
235			//System.out.print("LT(p="+p+","+k+")=");
236			if ( (p+k-1) >= tokens.length ) {
237				return TokenConstants.EOF_TOKEN;
238			}
239			//System.out.println(tokens.get(p+k-1));
240			var i:int = p;
241			var n:int = 1;
242			// find k good tokens
243			while ( n<k ) {
244				// skip off-channel tokens
245				i = skipOffTokenChannels(i+1); // leave p on valid token
246				n++;
247			}
248			if ( i>=tokens.length ) {
249				return TokenConstants.EOF_TOKEN;
250			}
251	        return Token(tokens[i]);
252	    }
253
254		/** Look backwards k tokens on-channel tokens */
255		protected function LB(k:int):Token {
256			//System.out.print("LB(p="+p+","+k+") ");
257			if ( p == -1 ) {
258				fillBuffer();
259			}
260			if ( k==0 ) {
261				return null;
262			}
263			if ( (p-k)<0 ) {
264				return null;
265			}
266
267			var i:int = p;
268			var n:int = 1;
269			// find k good tokens looking backwards
270			while ( n<=k ) {
271				// skip off-channel tokens
272				i = skipOffTokenChannelsReverse(i-1); // leave p on valid token
273				n++;
274			}
275			if ( i<0 ) {
276				return null;
277			}
278			return Token(tokens[i]);
279		}
280
281		/** Return absolute token i; ignore which channel the tokens are on;
282		 *  that is, count all tokens not just on-channel tokens.
283		 */
284		public function getToken(i:int):Token {
285			return Token(tokens[i]);
286		}
287
288	    public function LA(i:int):int {
289	        return LT(i).type;
290	    }
291
292	    public function mark():int {
293			if ( p == -1 ) {
294				fillBuffer();
295			}
296			lastMarker = index;
297			return lastMarker;
298		}
299
300		public function release(marker:int):void {
301			// no resources to release
302		}
303
304		public function get size():int {
305			return tokens.length;
306		}
307
308	    public function get index():int {
309	        return p;
310	    }
311
312		public function reset():void {
313			p = 0;
314			lastMarker = 0;
315		}
316
317		public function rewindTo(marker:int):void {
318			seek(marker);
319		}
320
321		public function rewind():void {
322			seek(lastMarker);
323		}
324
325		public function seek(index:int):void {
326			p = index;
327		}
328
329		public function get tokenSource():TokenSource {
330			return _tokenSource;
331		}
332
333		public function get sourceName():String {
334			return tokenSource.sourceName;
335		}
336
337		public function toString():String {
338			if ( p == -1 ) {
339				fillBuffer();
340			}
341			return toStringWithRange(0, tokens.length-1);
342		}
343
344		public function toStringWithRange(start:int, stop:int):String {
345			if ( start<0 || stop<0 ) {
346				return null;
347			}
348			if ( p == -1 ) {
349				fillBuffer();
350			}
351			if ( stop>=tokens.length ) {
352				stop = tokens.length-1;
353			}
354	 		var buf:String = "";
355			for (var i:int = start; i <= stop; i++) {
356				var t:Token = Token(tokens[i]);
357				buf += t.text;
358			}
359			return buf.toString();
360		}
361
362		public function toStringWithTokenRange(start:Token, stop:Token):String {
363			if ( start!=null && stop!=null ) {
364				return toStringWithRange(start.tokenIndex, stop.tokenIndex);
365			}
366			return null;
367		}
368	}
369
370
371}