1/*
2 [The "BSD license"]
3 Copyright (c) 2005-2009 Terence Parr
4 All rights reserved.
5
6 Redistribution and use in source and binary forms, with or without
7 modification, are permitted provided that the following conditions
8 are met:
9 1. Redistributions of source code must retain the above copyright
10     notice, this list of conditions and the following disclaimer.
11 2. Redistributions in binary form must reproduce the above copyright
12     notice, this list of conditions and the following disclaimer in the
13     documentation and/or other materials provided with the distribution.
14 3. The name of the author may not be used to endorse or promote products
15     derived from this software without specific prior written permission.
16
17 THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
18 IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
19 OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
20 IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
21 INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
22 NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23 DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24 THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25 (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
26 THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27 */
28
29package org.antlr.runtime;
30
31import java.util.List;
32import java.util.ArrayList;
33import java.util.NoSuchElementException;
34
35/** Buffer all input tokens but do on-demand fetching of new tokens from
36 *  lexer. Useful when the parser or lexer has to set context/mode info before
37 *  proper lexing of future tokens. The ST template parser needs this,
38 *  for example, because it has to constantly flip back and forth between
39 *  inside/output templates. E.g., <names:{hi, <it>}> has to parse names
40 *  as part of an expression but "hi, <it>" as a nested template.
41 *
42 *  You can't use this stream if you pass whitespace or other off-channel
43 *  tokens to the parser. The stream can't ignore off-channel tokens.
44 *  (UnbufferedTokenStream is the same way.)
45 *
46 *  This is not a subclass of UnbufferedTokenStream because I don't want
47 *  to confuse small moving window of tokens it uses for the full buffer.
48 */
49public class BufferedTokenStream implements TokenStream {
50    protected TokenSource tokenSource;
51
52    /** Record every single token pulled from the source so we can reproduce
53     *  chunks of it later.  The buffer in LookaheadStream overlaps sometimes
54     *  as its moving window moves through the input.  This list captures
55     *  everything so we can access complete input text.
56     */
57    protected List<Token> tokens = new ArrayList<Token>(100);
58
59    /** Track the last mark() call result value for use in rewind(). */
60    protected int lastMarker;
61
62    /** The index into the tokens list of the current token (next token
63     *  to consume).  tokens[p] should be LT(1).  p=-1 indicates need
64     *  to initialize with first token.  The ctor doesn't get a token.
65     *  First call to LT(1) or whatever gets the first token and sets p=0;
66     */
67    protected int p = -1;
68
69	protected int range = -1; // how deep have we gone?
70
71    public BufferedTokenStream() {;}
72
73    public BufferedTokenStream(TokenSource tokenSource) {
74        this.tokenSource = tokenSource;
75    }
76
77    public TokenSource getTokenSource() { return tokenSource; }
78
79	public int index() { return p; }
80
81	public int range() { return range; }
82
83    public int mark() {
84        if ( p == -1 ) setup();
85		lastMarker = index();
86		return lastMarker;
87	}
88
89	public void release(int marker) {
90		// no resources to release
91	}
92
93    public void rewind(int marker) {
94        seek(marker);
95    }
96
97    public void rewind() {
98        seek(lastMarker);
99    }
100
101    public void reset() {
102        p = 0;
103        lastMarker = 0;
104    }
105
106    public void seek(int index) { p = index; }
107
108    public int size() { return tokens.size(); }
109
110    /** Move the input pointer to the next incoming token.  The stream
111     *  must become active with LT(1) available.  consume() simply
112     *  moves the input pointer so that LT(1) points at the next
113     *  input symbol. Consume at least one token.
114     *
115     *  Walk past any token not on the channel the parser is listening to.
116     */
117    public void consume() {
118        if ( p == -1 ) setup();
119        p++;
120        sync(p);
121    }
122
123    /** Make sure index i in tokens has a token. */
124    protected void sync(int i) {
125        int n = i - tokens.size() + 1; // how many more elements we need?
126        //System.out.println("sync("+i+") needs "+n);
127        if ( n > 0 ) fetch(n);
128    }
129
130    /** add n elements to buffer */
131    protected void fetch(int n) {
132        for (int i=1; i<=n; i++) {
133            Token t = tokenSource.nextToken();
134            t.setTokenIndex(tokens.size());
135            //System.out.println("adding "+t+" at index "+tokens.size());
136            tokens.add(t);
137            if ( t.getType()==Token.EOF ) break;
138        }
139    }
140
141    public Token get(int i) {
142        if ( i < 0 || i >= tokens.size() ) {
143            throw new NoSuchElementException("token index "+i+" out of range 0.."+(tokens.size()-1));
144        }
145        return tokens.get(i);
146    }
147
148	/** Get all tokens from start..stop inclusively */
149	public List get(int start, int stop) {
150		if ( start<0 || stop<0 ) return null;
151		if ( p == -1 ) setup();
152		List subset = new ArrayList();
153		if ( stop>=tokens.size() ) stop = tokens.size()-1;
154		for (int i = start; i <= stop; i++) {
155			Token t = tokens.get(i);
156			if ( t.getType()==Token.EOF ) break;
157			subset.add(t);
158		}
159		return subset;
160	}
161
162	public int LA(int i) { return LT(i).getType(); }
163
164    protected Token LB(int k) {
165        if ( (p-k)<0 ) return null;
166        return tokens.get(p-k);
167    }
168
169    public Token LT(int k) {
170        if ( p == -1 ) setup();
171        if ( k==0 ) return null;
172        if ( k < 0 ) return LB(-k);
173
174		int i = p + k - 1;
175		sync(i);
176        if ( i >= tokens.size() ) { // return EOF token
177            // EOF must be last token
178            return tokens.get(tokens.size()-1);
179        }
180		if ( i>range ) range = i;
181        return tokens.get(i);
182    }
183
184    protected void setup() { sync(0); p = 0; }
185
186    /** Reset this token stream by setting its token source. */
187    public void setTokenSource(TokenSource tokenSource) {
188        this.tokenSource = tokenSource;
189        tokens.clear();
190        p = -1;
191    }
192
193    public List getTokens() { return tokens; }
194
195    public List getTokens(int start, int stop) {
196        return getTokens(start, stop, (BitSet)null);
197    }
198
199    /** Given a start and stop index, return a List of all tokens in
200     *  the token type BitSet.  Return null if no tokens were found.  This
201     *  method looks at both on and off channel tokens.
202     */
203    public List getTokens(int start, int stop, BitSet types) {
204        if ( p == -1 ) setup();
205        if ( stop>=tokens.size() ) stop=tokens.size()-1;
206        if ( start<0 ) start=0;
207        if ( start>stop ) return null;
208
209        // list = tokens[start:stop]:{Token t, t.getType() in types}
210        List<Token> filteredTokens = new ArrayList<Token>();
211        for (int i=start; i<=stop; i++) {
212            Token t = tokens.get(i);
213            if ( types==null || types.member(t.getType()) ) {
214                filteredTokens.add(t);
215            }
216        }
217        if ( filteredTokens.size()==0 ) {
218            filteredTokens = null;
219        }
220        return filteredTokens;
221    }
222
223    public List getTokens(int start, int stop, List types) {
224        return getTokens(start,stop,new BitSet(types));
225    }
226
227    public List getTokens(int start, int stop, int ttype) {
228        return getTokens(start,stop,BitSet.of(ttype));
229    }
230
231    public String getSourceName() {	return tokenSource.getSourceName();	}
232
233    /** Grab *all* tokens from stream and return string */
234    public String toString() {
235        if ( p == -1 ) setup();
236        fill();
237        return toString(0, tokens.size()-1);
238    }
239
240    public String toString(int start, int stop) {
241        if ( start<0 || stop<0 ) return null;
242        if ( p == -1 ) setup();
243        if ( stop>=tokens.size() ) stop = tokens.size()-1;
244        StringBuffer buf = new StringBuffer();
245        for (int i = start; i <= stop; i++) {
246            Token t = tokens.get(i);
247            if ( t.getType()==Token.EOF ) break;
248            buf.append(t.getText());
249        }
250        return buf.toString();
251    }
252
253    public String toString(Token start, Token stop) {
254        if ( start!=null && stop!=null ) {
255            return toString(start.getTokenIndex(), stop.getTokenIndex());
256        }
257        return null;
258    }
259
260    /** Get all tokens from lexer until EOF */
261    public void fill() {
262        if ( p == -1 ) setup();
263        if ( tokens.get(p).getType()==Token.EOF ) return;
264
265        int i = p+1;
266        sync(i);
267        while ( tokens.get(i).getType()!=Token.EOF ) {
268            i++;
269            sync(i);
270        }
271    }
272}
273