1/*
2 * [The "BSD licence"]
3 * Copyright (c) 2005-2008 Terence Parr
4 * All rights reserved.
5 *
6 * Conversion to C#:
7 * Copyright (c) 2008-2009 Sam Harwell, Pixel Mine, Inc.
8 * All rights reserved.
9 *
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions
12 * are met:
13 * 1. Redistributions of source code must retain the above copyright
14 *    notice, this list of conditions and the following disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright
16 *    notice, this list of conditions and the following disclaimer in the
17 *    documentation and/or other materials provided with the distribution.
18 * 3. The name of the author may not be used to endorse or promote products
19 *    derived from this software without specific prior written permission.
20 *
21 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
22 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
23 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
24 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
25 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
26 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
27 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
28 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
29 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
30 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
31 */
32
33namespace Antlr.Runtime {
34    using System.Collections.Generic;
35    using CLSCompliant = System.CLSCompliantAttribute;
36    using IndexOutOfRangeException = System.IndexOutOfRangeException;
37    using StringBuilder = System.Text.StringBuilder;
38
39    /** Buffer all input tokens but do on-demand fetching of new tokens from
40     *  lexer. Useful when the parser or lexer has to set context/mode info before
41     *  proper lexing of future tokens. The ST template parser needs this,
42     *  for example, because it has to constantly flip back and forth between
43     *  inside/output templates. E.g., <names:{hi, <it>}> has to parse names
44     *  as part of an expression but "hi, <it>" as a nested template.
45     *
46     *  You can't use this stream if you pass whitespace or other off-channel
47     *  tokens to the parser. The stream can't ignore off-channel tokens.
48     *  (UnbufferedTokenStream is the same way.)
49     *
50     *  This is not a subclass of UnbufferedTokenStream because I don't want
51     *  to confuse small moving window of tokens it uses for the full buffer.
52     */
53    [System.Serializable]
54    public class BufferedTokenStream : ITokenStream, ITokenStreamInformation {
55        private ITokenSource _tokenSource;
56
57        /** Record every single token pulled from the source so we can reproduce
58         *  chunks of it later.  The buffer in LookaheadStream overlaps sometimes
59         *  as its moving window moves through the input.  This list captures
60         *  everything so we can access complete input text.
61         */
62        [CLSCompliant(false)]
63        protected List<IToken> _tokens = new List<IToken>(100);
64
65        /** Track the last mark() call result value for use in rewind(). */
66        private int _lastMarker;
67
68        /** The index into the tokens list of the current token (next token
69         *  to consume).  tokens[p] should be LT(1).  p=-1 indicates need
70         *  to initialize with first token.  The ctor doesn't get a token.
71         *  First call to LT(1) or whatever gets the first token and sets p=0;
72         */
73        [CLSCompliant(false)]
74        protected int _p = -1;
75
76        public BufferedTokenStream() {
77        }
78
79        public BufferedTokenStream(ITokenSource tokenSource) {
80            this._tokenSource = tokenSource;
81        }
82
83        public virtual ITokenSource TokenSource {
84            get {
85                return _tokenSource;
86            }
87            set {
88                this._tokenSource = value;
89                _tokens.Clear();
90                _p = -1;
91            }
92        }
93
94        public virtual int Index {
95            get {
96                return _p;
97            }
98        }
99
100        /// <summary>
101        /// How deep have we gone?
102        /// </summary>
103        public virtual int Range {
104            get;
105            protected set;
106        }
107
108        public virtual int Count {
109            get {
110                return _tokens.Count;
111            }
112        }
113
114        public virtual string SourceName {
115            get {
116                return _tokenSource.SourceName;
117            }
118        }
119
120        public virtual IToken LastToken {
121            get {
122                return LB(1);
123            }
124        }
125
126        public virtual IToken LastRealToken {
127            get {
128                int i = 0;
129                IToken token;
130                do {
131                    i++;
132                    token = LB(i);
133                } while (token != null && token.Line <= 0);
134
135                return token;
136            }
137        }
138
139        public virtual int MaxLookBehind {
140            get {
141                return int.MaxValue;
142            }
143        }
144
145        public virtual int Mark() {
146            if (_p == -1)
147                Setup();
148            _lastMarker = Index;
149            return _lastMarker;
150        }
151
152        public virtual void Release(int marker) {
153            // no resources to release
154        }
155
156        public virtual void Rewind(int marker) {
157            Seek(marker);
158        }
159
160        public virtual void Rewind() {
161            Seek(_lastMarker);
162        }
163
164        public virtual void Reset() {
165            _p = 0;
166            _lastMarker = 0;
167        }
168
169        public virtual void Seek(int index) {
170            _p = index;
171        }
172
173        /** Move the input pointer to the next incoming token.  The stream
174         *  must become active with LT(1) available.  consume() simply
175         *  moves the input pointer so that LT(1) points at the next
176         *  input symbol. Consume at least one token.
177         *
178         *  Walk past any token not on the channel the parser is listening to.
179         */
180        public virtual void Consume() {
181            if (_p == -1)
182                Setup();
183            _p++;
184            Sync(_p);
185        }
186
187        /** Make sure index i in tokens has a token. */
188        protected virtual void Sync(int i) {
189            int n = i - _tokens.Count + 1; // how many more elements we need?
190            if (n > 0)
191                Fetch(n);
192        }
193
194        /** add n elements to buffer */
195        protected virtual void Fetch(int n) {
196            for (int i = 0; i < n; i++) {
197                IToken t = TokenSource.NextToken();
198                t.TokenIndex = _tokens.Count;
199                _tokens.Add(t);
200                if (t.Type == CharStreamConstants.EndOfFile)
201                    break;
202            }
203        }
204
205        public virtual IToken Get(int i) {
206            if (i < 0 || i >= _tokens.Count) {
207                throw new IndexOutOfRangeException("token index " + i + " out of range 0.." + (_tokens.Count - 1));
208            }
209            return _tokens[i];
210        }
211
212#if false // why is this different from GetTokens(start, count) ?
213        /// <summary>
214        /// Get all tokens from start..(start+count-1) inclusively
215        /// </summary>
216        public virtual List<IToken> Get(int start, int count)
217        {
218            if (start < 0)
219                throw new ArgumentOutOfRangeException("start");
220            if (count < 0)
221                throw new ArgumentOutOfRangeException("count");
222            if (start + count >= _tokens.Count)
223                throw new ArgumentException();
224
225            if (_p == -1)
226                Setup();
227
228            List<IToken> subset = new List<IToken>(count);
229            for (int i = 0; i < count; i++)
230            {
231                IToken token = _tokens[i];
232                if (token.Type == TokenTypes.EndOfFile)
233                    break;
234
235                subset.Add(token);
236            }
237
238            return subset;
239        }
240#endif
241
242        public virtual int LA(int i) {
243            return LT(i).Type;
244        }
245
246        protected virtual IToken LB(int k) {
247            if ((_p - k) < 0)
248                return null;
249
250            return _tokens[_p - k];
251        }
252
253        public virtual IToken LT(int k) {
254            if (_p == -1)
255                Setup();
256            if (k == 0)
257                return null;
258            if (k < 0)
259                return LB(-k);
260
261            int i = _p + k - 1;
262            Sync(i);
263            if (i >= _tokens.Count) {
264                // EOF must be last token
265                return _tokens[_tokens.Count - 1];
266            }
267
268            if (i > Range)
269                Range = i;
270
271            return _tokens[_p + k - 1];
272        }
273
274        protected virtual void Setup() {
275            Sync(0);
276            _p = 0;
277        }
278
279        public virtual List<IToken> GetTokens() {
280            return _tokens;
281        }
282
283        public virtual List<IToken> GetTokens(int start, int stop) {
284            return GetTokens(start, stop, default(BitSet));
285        }
286
287        /** Given a start and stop index, return a List of all tokens in
288         *  the token type BitSet.  Return null if no tokens were found.  This
289         *  method looks at both on and off channel tokens.
290         */
291        public virtual List<IToken> GetTokens(int start, int stop, BitSet types) {
292            if (_p == -1)
293                Setup();
294            if (stop >= _tokens.Count)
295                stop = _tokens.Count - 1;
296            if (start < 0)
297                start = 0;
298            if (start > stop)
299                return null;
300
301            // list = tokens[start:stop]:{Token t, t.getType() in types}
302            List<IToken> filteredTokens = new List<IToken>();
303            for (int i = start; i <= stop; i++) {
304                IToken t = _tokens[i];
305                if (types == null || types.Member(t.Type)) {
306                    filteredTokens.Add(t);
307                }
308            }
309            if (filteredTokens.Count == 0) {
310                filteredTokens = null;
311            }
312            return filteredTokens;
313        }
314
315        public virtual List<IToken> GetTokens(int start, int stop, IEnumerable<int> types) {
316            return GetTokens(start, stop, new BitSet(types));
317        }
318
319        public virtual List<IToken> GetTokens(int start, int stop, int ttype) {
320            return GetTokens(start, stop, BitSet.Of(ttype));
321        }
322
323        public override string ToString() {
324            if (_p == -1)
325                Setup();
326
327            Fill();
328            return ToString(0, _tokens.Count - 1);
329        }
330
331        public virtual string ToString(int start, int stop) {
332            if (start < 0 || stop < 0)
333                return null;
334            if (_p == -1)
335                Setup();
336            if (stop >= _tokens.Count)
337                stop = _tokens.Count - 1;
338
339            StringBuilder buf = new StringBuilder();
340            for (int i = start; i <= stop; i++) {
341                IToken t = _tokens[i];
342                if (t.Type == CharStreamConstants.EndOfFile)
343                    break;
344                buf.Append(t.Text);
345            }
346
347            return buf.ToString();
348        }
349
350        public virtual string ToString(IToken start, IToken stop) {
351            if (start != null && stop != null) {
352                return ToString(start.TokenIndex, stop.TokenIndex);
353            }
354            return null;
355        }
356
357        public virtual void Fill() {
358            if (_p == -1)
359                Setup();
360
361            if (_tokens[_p].Type == CharStreamConstants.EndOfFile)
362                return;
363
364            int i = _p + 1;
365            Sync(i);
366            while (_tokens[i].Type != CharStreamConstants.EndOfFile) {
367                i++;
368                Sync(i);
369            }
370        }
371    }
372}
373