UnbufferedTokenStream.cs revision 324c4644fee44b9898524c09511bd33c3f12e2df 
1/*
2 * [The "BSD licence"]
3 * Copyright (c) 2005-2008 Terence Parr
4 * All rights reserved.
5 *
6 * Conversion to C#:
7 * Copyright (c) 2009 Sam Harwell
8 * All rights reserved.
9 *
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions
12 * are met:
13 * 1. Redistributions of source code must retain the above copyright
14 *    notice, this list of conditions and the following disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright
16 *    notice, this list of conditions and the following disclaimer in the
17 *    documentation and/or other materials provided with the distribution.
18 * 3. The name of the author may not be used to endorse or promote products
19 *    derived from this software without specific prior written permission.
20 *
21 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
22 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
23 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
24 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
25 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
26 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
27 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
28 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
29 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
30 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
31 */
32
33namespace Antlr.Runtime
34{
35    using Antlr.Runtime.Misc;
36    using CLSCompliant = System.CLSCompliantAttribute;
37    using NotSupportedException = System.NotSupportedException;
38    using IndexOutOfRangeException = System.IndexOutOfRangeException;
39
40    /** A token stream that pulls tokens from the code source on-demand and
41     *  without tracking a complete buffer of the tokens. This stream buffers
42     *  the minimum number of tokens possible.  It's the same as
43     *  OnDemandTokenStream except that OnDemandTokenStream buffers all tokens.
44     *
45     *  You can't use this stream if you pass whitespace or other off-channel
46     *  tokens to the parser. The stream can't ignore off-channel tokens.
47     *
48     *  You can only look backwards 1 token: LT(-1).
49     *
50     *  Use this when you need to read from a socket or other infinite stream.
51     *
52     *  @see BufferedTokenStream
53     *  @see CommonTokenStream
54     */
55    public class UnbufferedTokenStream : LookaheadStream<IToken>, ITokenStream, ITokenStreamInformation
56    {
57        [CLSCompliant(false)]
58        protected ITokenSource tokenSource;
59        protected int tokenIndex; // simple counter to set token index in tokens
60
61        /** Skip tokens on any channel but this one; this is how we skip whitespace... */
62        protected int channel = TokenChannels.Default;
63
64        private readonly ListStack<IToken> _realTokens = new ListStack<IToken>() { null };
65
66        public UnbufferedTokenStream(ITokenSource tokenSource)
67        {
68            this.tokenSource = tokenSource;
69        }
70
71        public ITokenSource TokenSource
72        {
73            get
74            {
75                return this.tokenSource;
76            }
77        }
78
79        public string SourceName
80        {
81            get
82            {
83                return TokenSource.SourceName;
84            }
85        }
86
87        #region ITokenStreamInformation Members
88
89        public IToken LastToken
90        {
91            get
92            {
93                return LB(1);
94            }
95        }
96
97        public IToken LastRealToken
98        {
99            get
100            {
101                return _realTokens.Peek();
102            }
103        }
104
105        public int MaxLookBehind
106        {
107            get
108            {
109                return 1;
110            }
111        }
112
113        public override int Mark()
114        {
115            _realTokens.Push(_realTokens.Peek());
116            return base.Mark();
117        }
118
119        public override void Release(int marker)
120        {
121            base.Release(marker);
122            _realTokens.Pop();
123        }
124
125        public override void Clear()
126        {
127            _realTokens.Clear();
128            _realTokens.Push(null);
129        }
130
131        public override void Consume()
132        {
133            base.Consume();
134            if (PreviousElement != null && PreviousElement.Line > 0)
135                _realTokens[_realTokens.Count - 1] = PreviousElement;
136        }
137
138        #endregion
139
140        public override IToken NextElement()
141        {
142            IToken t = this.tokenSource.NextToken();
143            t.TokenIndex = this.tokenIndex++;
144            return t;
145        }
146
147        public override bool IsEndOfFile(IToken o)
148        {
149            return o.Type == CharStreamConstants.EndOfFile;
150        }
151
152        public IToken Get(int i)
153        {
154            throw new NotSupportedException("Absolute token indexes are meaningless in an unbuffered stream");
155        }
156
157        public int LA(int i)
158        {
159            return LT(i).Type;
160        }
161
162        public string ToString(int start, int stop)
163        {
164            return "n/a";
165        }
166
167        public string ToString(IToken start, IToken stop)
168        {
169            return "n/a";
170        }
171    }
172}
173