1/*
2 * [The "BSD license"]
3 *  Copyright (c) 2010 Terence Parr
4 *  All rights reserved.
5 *
6 *  Redistribution and use in source and binary forms, with or without
7 *  modification, are permitted provided that the following conditions
8 *  are met:
9 *  1. Redistributions of source code must retain the above copyright
10 *      notice, this list of conditions and the following disclaimer.
11 *  2. Redistributions in binary form must reproduce the above copyright
12 *      notice, this list of conditions and the following disclaimer in the
13 *      documentation and/or other materials provided with the distribution.
14 *  3. The name of the author may not be used to endorse or promote products
15 *      derived from this software without specific prior written permission.
16 *
17 *  THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
18 *  IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
19 *  OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
20 *  IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
21 *  INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
22 *  NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23 *  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24 *  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25 *  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
26 *  THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27 */
28
29package org.antlr.test;
30
31import org.antlr.runtime.*;
32import org.antlr.tool.Grammar;
33import org.antlr.tool.Interpreter;
34import org.junit.Test;
35
36/** This actually tests new (12/4/09) buffered but on-demand fetching stream */
37public class TestCommonTokenStream extends BaseTest {
38    @Test public void testFirstToken() throws Exception {
39        Grammar g = new Grammar(
40            "lexer grammar t;\n"+
41            "ID : 'a'..'z'+;\n" +
42            "INT : '0'..'9'+;\n" +
43            "SEMI : ';';\n" +
44            "ASSIGN : '=';\n" +
45            "PLUS : '+';\n" +
46            "MULT : '*';\n" +
47            "WS : ' '+;\n");
48        // Tokens: 012345678901234567
49        // Input:  x = 3 * 0 + 2 * 0;
50        CharStream input = new ANTLRStringStream("x = 3 * 0 + 2 * 0;");
51        Interpreter lexEngine = new Interpreter(g, input);
52        BufferedTokenStream tokens = new BufferedTokenStream(lexEngine);
53
54        String result = tokens.LT(1).getText();
55        String expecting = "x";
56        assertEquals(expecting, result);
57    }
58
59    @Test public void test2ndToken() throws Exception {
60        Grammar g = new Grammar(
61            "lexer grammar t;\n"+
62            "ID : 'a'..'z'+;\n" +
63            "INT : '0'..'9'+;\n" +
64            "SEMI : ';';\n" +
65            "ASSIGN : '=';\n" +
66            "PLUS : '+';\n" +
67            "MULT : '*';\n" +
68            "WS : ' '+;\n");
69        // Tokens: 012345678901234567
70        // Input:  x = 3 * 0 + 2 * 0;
71        CharStream input = new ANTLRStringStream("x = 3 * 0 + 2 * 0;");
72        Interpreter lexEngine = new Interpreter(g, input);
73        BufferedTokenStream tokens = new BufferedTokenStream(lexEngine);
74
75        String result = tokens.LT(2).getText();
76        String expecting = " ";
77        assertEquals(expecting, result);
78    }
79
80    @Test public void testCompleteBuffer() throws Exception {
81        Grammar g = new Grammar(
82            "lexer grammar t;\n"+
83            "ID : 'a'..'z'+;\n" +
84            "INT : '0'..'9'+;\n" +
85            "SEMI : ';';\n" +
86            "ASSIGN : '=';\n" +
87            "PLUS : '+';\n" +
88            "MULT : '*';\n" +
89            "WS : ' '+;\n");
90        // Tokens: 012345678901234567
91        // Input:  x = 3 * 0 + 2 * 0;
92        CharStream input = new ANTLRStringStream("x = 3 * 0 + 2 * 0;");
93        Interpreter lexEngine = new Interpreter(g, input);
94        BufferedTokenStream tokens = new BufferedTokenStream(lexEngine);
95
96        int i = 1;
97        Token t = tokens.LT(i);
98        while ( t.getType()!=Token.EOF ) {
99            i++;
100            t = tokens.LT(i);
101        }
102        tokens.LT(i++); // push it past end
103        tokens.LT(i++);
104
105        String result = tokens.toString();
106        String expecting = "x = 3 * 0 + 2 * 0;";
107        assertEquals(expecting, result);
108    }
109
110    @Test public void testCompleteBufferAfterConsuming() throws Exception {
111        Grammar g = new Grammar(
112            "lexer grammar t;\n"+
113            "ID : 'a'..'z'+;\n" +
114            "INT : '0'..'9'+;\n" +
115            "SEMI : ';';\n" +
116            "ASSIGN : '=';\n" +
117            "PLUS : '+';\n" +
118            "MULT : '*';\n" +
119            "WS : ' '+;\n");
120        // Tokens: 012345678901234567
121        // Input:  x = 3 * 0 + 2 * 0;
122        CharStream input = new ANTLRStringStream("x = 3 * 0 + 2 * 0;");
123        Interpreter lexEngine = new Interpreter(g, input);
124        BufferedTokenStream tokens = new BufferedTokenStream(lexEngine);
125
126        Token t = tokens.LT(1);
127        while ( t.getType()!=Token.EOF ) {
128            tokens.consume();
129            t = tokens.LT(1);
130        }
131        tokens.consume();
132        tokens.LT(1); // push it past end
133        tokens.consume();
134        tokens.LT(1);
135
136        String result = tokens.toString();
137        String expecting = "x = 3 * 0 + 2 * 0;";
138        assertEquals(expecting, result);
139    }
140
141    @Test public void testLookback() throws Exception {
142        Grammar g = new Grammar(
143            "lexer grammar t;\n"+
144            "ID : 'a'..'z'+;\n" +
145            "INT : '0'..'9'+;\n" +
146            "SEMI : ';';\n" +
147            "ASSIGN : '=';\n" +
148            "PLUS : '+';\n" +
149            "MULT : '*';\n" +
150            "WS : ' '+;\n");
151        // Tokens: 012345678901234567
152        // Input:  x = 3 * 0 + 2 * 0;
153        CharStream input = new ANTLRStringStream("x = 3 * 0 + 2 * 0;");
154        Interpreter lexEngine = new Interpreter(g, input);
155        BufferedTokenStream tokens = new BufferedTokenStream(lexEngine);
156
157        tokens.consume(); // get x into buffer
158        Token t = tokens.LT(-1);
159        assertEquals("x", t.getText());
160
161        tokens.consume();
162        tokens.consume(); // consume '='
163        t = tokens.LT(-3);
164        assertEquals("x", t.getText());
165        t = tokens.LT(-2);
166        assertEquals(" ", t.getText());
167        t = tokens.LT(-1);
168        assertEquals("=", t.getText());
169    }
170
171    @Test public void testOffChannel() throws Exception {
172        TokenSource lexer = // simulate input " x =34  ;\n"
173            new TokenSource() {
174                int i = 0;
175                Token[] tokens = {
176                    new CommonToken(1," "),
177                    new CommonToken(1,"x"),
178                    new CommonToken(1," "),
179                    new CommonToken(1,"="),
180                    new CommonToken(1,"34"),
181                    new CommonToken(1," "),
182                    new CommonToken(1," "),
183                    new CommonToken(1,";"),
184                    new CommonToken(1,"\n"),
185                    new CommonToken(Token.EOF,"")
186                };
187                {
188                    tokens[0].setChannel(Lexer.HIDDEN);
189                    tokens[2].setChannel(Lexer.HIDDEN);
190                    tokens[5].setChannel(Lexer.HIDDEN);
191                    tokens[6].setChannel(Lexer.HIDDEN);
192                    tokens[8].setChannel(Lexer.HIDDEN);
193                }
194                public Token nextToken() {
195                    return tokens[i++];
196                }
197                public String getSourceName() { return "test"; }
198            };
199
200        CommonTokenStream tokens = new CommonTokenStream(lexer);
201
202        assertEquals("x", tokens.LT(1).getText()); // must skip first off channel token
203        tokens.consume();
204        assertEquals("=", tokens.LT(1).getText());
205        assertEquals("x", tokens.LT(-1).getText());
206
207        tokens.consume();
208        assertEquals("34", tokens.LT(1).getText());
209        assertEquals("=", tokens.LT(-1).getText());
210
211        tokens.consume();
212        assertEquals(";", tokens.LT(1).getText());
213        assertEquals("34", tokens.LT(-1).getText());
214
215        tokens.consume();
216        assertEquals(Token.EOF, tokens.LA(1));
217        assertEquals(";", tokens.LT(-1).getText());
218
219        assertEquals("34", tokens.LT(-2).getText());
220        assertEquals("=", tokens.LT(-3).getText());
221        assertEquals("x", tokens.LT(-4).getText());
222    }
223}