1/*
2 * [The "BSD license"]
3 *  Copyright (c) 2010 Terence Parr
4 *  All rights reserved.
5 *
6 *  Redistribution and use in source and binary forms, with or without
7 *  modification, are permitted provided that the following conditions
8 *  are met:
9 *  1. Redistributions of source code must retain the above copyright
10 *      notice, this list of conditions and the following disclaimer.
11 *  2. Redistributions in binary form must reproduce the above copyright
12 *      notice, this list of conditions and the following disclaimer in the
13 *      documentation and/or other materials provided with the distribution.
14 *  3. The name of the author may not be used to endorse or promote products
15 *      derived from this software without specific prior written permission.
16 *
17 *  THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
18 *  IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
19 *  OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
20 *  IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
21 *  INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
22 *  NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23 *  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24 *  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25 *  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
26 *  THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27 */
28package org.antlr.test;
29
30import org.antlr.Tool;
31import org.antlr.codegen.CodeGenerator;
32import org.stringtemplate.v4.ST;
33import org.antlr.tool.Grammar;
34import org.junit.Test;
35
36public class TestLexer extends BaseTest {
37	protected boolean debug = false;
38
39	/** Public default constructor used by TestRig */
40	public TestLexer() {
41	}
42
43	@Test public void testSetText() throws Exception {
44		// this must return A not I to the parser; calling a nonfragment rule
45		// from a nonfragment rule does not set the overall token.
46		String grammar =
47			"grammar P;\n"+
48			"a : A {System.out.println(input);} ;\n"+
49			"A : '\\\\' 't' {setText(\"\t\");} ;\n" +
50			"WS : (' '|'\\n') {$channel=HIDDEN;} ;";
51		String found = execParser("P.g", grammar, "PParser", "PLexer",
52				    "a", "\\t", debug);
53		assertEquals("\t\n", found);
54	}
55
56	@Test public void testRefToRuleDoesNotSetTokenNorEmitAnother() throws Exception {
57		// this must return A not I to the parser; calling a nonfragment rule
58		// from a nonfragment rule does not set the overall token.
59		String grammar =
60			"grammar P;\n"+
61			"a : A EOF {System.out.println(input);} ;\n"+
62			"A : '-' I ;\n" +
63			"I : '0'..'9'+ ;\n"+
64			"WS : (' '|'\\n') {$channel=HIDDEN;} ;";
65		String found = execParser("P.g", grammar, "PParser", "PLexer",
66				    "a", "-34", debug);
67		assertEquals("-34\n", found);
68	}
69
70	@Test public void testRefToRuleDoesNotSetChannel() throws Exception {
71		// this must set channel of A to HIDDEN.  $channel is local to rule
72		// like $type.
73		String grammar =
74			"grammar P;\n"+
75			"a : A EOF {System.out.println($A.text+\", channel=\"+$A.channel);} ;\n"+
76			"A : '-' WS I ;\n" +
77			"I : '0'..'9'+ ;\n"+
78			"WS : (' '|'\\n') {$channel=HIDDEN;} ;";
79		String found = execParser("P.g", grammar, "PParser", "PLexer",
80				    "a", "- 34", debug);
81		assertEquals("- 34, channel=0\n", found);
82	}
83
84	@Test public void testWeCanSetType() throws Exception {
85		String grammar =
86			"grammar P;\n"+
87			"tokens {X;}\n" +
88			"a : X EOF {System.out.println(input);} ;\n"+
89			"A : '-' I {$type = X;} ;\n" +
90			"I : '0'..'9'+ ;\n"+
91			"WS : (' '|'\\n') {$channel=HIDDEN;} ;";
92		String found = execParser("P.g", grammar, "PParser", "PLexer",
93				    "a", "-34", debug);
94		assertEquals("-34\n", found);
95	}
96
97	@Test public void testRefToFragment() throws Exception {
98		// this must return A not I to the parser; calling a nonfragment rule
99		// from a nonfragment rule does not set the overall token.
100		String grammar =
101			"grammar P;\n"+
102			"a : A {System.out.println(input);} ;\n"+
103			"A : '-' I ;\n" +
104			"fragment I : '0'..'9'+ ;\n"+
105			"WS : (' '|'\\n') {$channel=HIDDEN;} ;";
106		String found = execParser("P.g", grammar, "PParser", "PLexer",
107				    "a", "-34", debug);
108		assertEquals("-34\n", found);
109	}
110
111	@Test public void testMultipleRefToFragment() throws Exception {
112		// this must return A not I to the parser; calling a nonfragment rule
113		// from a nonfragment rule does not set the overall token.
114		String grammar =
115			"grammar P;\n"+
116			"a : A EOF {System.out.println(input);} ;\n"+
117			"A : I '.' I ;\n" +
118			"fragment I : '0'..'9'+ ;\n"+
119			"WS : (' '|'\\n') {$channel=HIDDEN;} ;";
120		String found = execParser("P.g", grammar, "PParser", "PLexer",
121				    "a", "3.14159", debug);
122		assertEquals("3.14159\n", found);
123	}
124
125	@Test public void testLabelInSubrule() throws Exception {
126		// can we see v outside?
127		String grammar =
128			"grammar P;\n"+
129			"a : A EOF ;\n"+
130			"A : 'hi' WS (v=I)? {$channel=0; System.out.println($v.text);} ;\n" +
131			"fragment I : '0'..'9'+ ;\n"+
132			"WS : (' '|'\\n') {$channel=HIDDEN;} ;";
133		String found = execParser("P.g", grammar, "PParser", "PLexer",
134				    "a", "hi 342", debug);
135		assertEquals("342\n", found);
136	}
137
138	@Test public void testRefToTokenInLexer() throws Exception {
139		String grammar =
140			"grammar P;\n"+
141			"a : A EOF ;\n"+
142			"A : I {System.out.println($I.text);} ;\n" +
143			"fragment I : '0'..'9'+ ;\n"+
144			"WS : (' '|'\\n') {$channel=HIDDEN;} ;";
145		String found = execParser("P.g", grammar, "PParser", "PLexer",
146				    "a", "342", debug);
147		assertEquals("342\n", found);
148	}
149
150	@Test public void testListLabelInLexer() throws Exception {
151		String grammar =
152			"grammar P;\n"+
153			"a : A ;\n"+
154			"A : i+=I+ {for (Object t : $i) System.out.print(\" \"+((Token)t).getText());} ;\n" +
155			"fragment I : '0'..'9'+ ;\n"+
156			"WS : (' '|'\\n') {$channel=HIDDEN;} ;";
157		String found = execParser("P.g", grammar, "PParser", "PLexer",
158				    "a", "33 297", debug);
159		assertEquals(" 33 297\n", found);
160	}
161
162	@Test public void testDupListRefInLexer() throws Exception {
163		String grammar =
164			"grammar P;\n"+
165			"a : A ;\n"+
166			"A : i+=I WS i+=I {$channel=0; for (Object t : $i) System.out.print(\" \"+((Token)t).getText());} ;\n" +
167			"fragment I : '0'..'9'+ ;\n"+
168			"WS : (' '|'\\n') {$channel=HIDDEN;} ;";
169		String found = execParser("P.g", grammar, "PParser", "PLexer",
170				    "a", "33 297", debug);
171		assertEquals(" 33 297\n", found);
172	}
173
174	@Test public void testCharLabelInLexer() {
175		String grammar =
176			"grammar T;\n" +
177			"a : B ;\n" +
178			"B : x='a' {System.out.println((char)$x);} ;\n" ;
179		String found = execParser("T.g", grammar, "TParser", "TLexer",
180								  "a", "a", debug);
181		assertEquals("a\n", found);
182	}
183
184	@Test public void testRepeatedLabelInLexer() {
185		String grammar =
186			"lexer grammar T;\n" +
187			"B : x='a' x='b' ;\n" ;
188		boolean found =
189			rawGenerateAndBuildRecognizer(
190				"T.g", grammar, null, "T", false);
191		boolean expecting = true; // should be ok
192		assertEquals(expecting, found);
193	}
194
195	@Test public void testRepeatedRuleLabelInLexer() {
196		String grammar =
197			"lexer grammar T;\n" +
198			"B : x=A x=A ;\n" +
199			"fragment A : 'a' ;\n" ;
200		boolean found =
201			rawGenerateAndBuildRecognizer(
202				"T.g", grammar, null, "T", false);
203		boolean expecting = true; // should be ok
204		assertEquals(expecting, found);
205	}
206
207	@Test public void testIsolatedEOTEdge() {
208		String grammar =
209			"lexer grammar T;\n" +
210			"QUOTED_CONTENT \n" +
211			"        : 'q' (~'q')* (('x' 'q') )* 'q' ; \n";
212		boolean found =
213			rawGenerateAndBuildRecognizer(
214				"T.g", grammar, null, "T", false);
215		boolean expecting = true; // should be ok
216		assertEquals(expecting, found);
217	}
218
219	@Test public void testEscapedLiterals() {
220		/* Grammar:
221			A : '\"' ;  should match a single double-quote: "
222			B : '\\\"' ; should match input \"
223		*/
224		String grammar =
225			"lexer grammar T;\n" +
226			"A : '\\\"' ;\n" +
227			"B : '\\\\\\\"' ;\n" ; // '\\\"'
228		boolean found =
229			rawGenerateAndBuildRecognizer(
230				"T.g", grammar, null, "T", false);
231		boolean expecting = true; // should be ok
232		assertEquals(expecting, found);
233	}
234
235    @Test public void testNewlineLiterals() throws Exception {
236        Grammar g = new Grammar(
237            "lexer grammar T;\n" +
238            "A : '\\n\\n' ;\n"  // ANTLR sees '\n\n'
239        );
240        String expecting = "match(\"\\n\\n\")";
241
242        Tool antlr = newTool();
243        antlr.setOutputDirectory(null); // write to /dev/null
244        CodeGenerator generator = new CodeGenerator(antlr, g, "Java");
245        g.setCodeGenerator(generator);
246        generator.genRecognizer(); // codegen phase sets some vars we need
247        ST codeST = generator.getRecognizerST();
248        String code = codeST.render();
249        int m = code.indexOf("match(\"");
250        String found = code.substring(m,m+expecting.length());
251
252        assertEquals(expecting, found);
253    }
254}
255