1/*
2 * [The "BSD license"]
3 *  Copyright (c) 2010 Terence Parr
4 *  All rights reserved.
5 *
6 *  Redistribution and use in source and binary forms, with or without
7 *  modification, are permitted provided that the following conditions
8 *  are met:
9 *  1. Redistributions of source code must retain the above copyright
10 *      notice, this list of conditions and the following disclaimer.
11 *  2. Redistributions in binary form must reproduce the above copyright
12 *      notice, this list of conditions and the following disclaimer in the
13 *      documentation and/or other materials provided with the distribution.
14 *  3. The name of the author may not be used to endorse or promote products
15 *      derived from this software without specific prior written permission.
16 *
17 *  THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
18 *  IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
19 *  OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
20 *  IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
21 *  INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
22 *  NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23 *  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24 *  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25 *  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
26 *  THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27 */
28package org.antlr.test;
29
30import org.junit.Test;
31
32/** Test hetero trees in parsers and tree parsers */
33public class TestHeteroAST extends BaseTest {
34	protected boolean debug = false;
35
36	// PARSERS -- AUTO AST
37
38    @Test public void testToken() throws Exception {
39        String grammar =
40            "grammar T;\n" +
41            "options {output=AST;}\n" +
42            "@members {static class V extends CommonTree {\n" +
43            "  public V(Token t) { token=t;}\n" +
44            "  public String toString() { return token.getText()+\"<V>\";}\n" +
45            "}\n" +
46            "}\n"+
47            "a : ID<V> ;\n"+
48            "ID : 'a'..'z'+ ;\n" +
49            "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n";
50        String found = execParser("T.g", grammar, "TParser", "TLexer",
51                    "a", "a", debug);
52        assertEquals("a<V>\n", found);
53    }
54
55	@Test public void testTokenCommonTree() throws Exception {
56		String grammar =
57			"grammar T;\n" +
58			"options {output=AST;}\n" +
59			"a : ID<CommonTree> ;\n"+
60			"ID : 'a'..'z'+ ;\n" +
61			"WS : (' '|'\\n') {$channel=HIDDEN;} ;\n";
62		String found = execParser("T.g", grammar, "TParser", "TLexer",
63					"a", "a", debug);
64		assertEquals("a\n", found);
65	}
66
67    @Test public void testTokenWithQualifiedType() throws Exception {
68        String grammar =
69            "grammar T;\n" +
70            "options {output=AST;}\n" +
71            "@members {static class V extends CommonTree {\n" +
72            "  public V(Token t) { token=t;}\n" +
73            "  public String toString() { return token.getText()+\"<V>\";}\n" +
74            "}\n" +
75            "}\n"+
76            "a : ID<TParser.V> ;\n"+ // TParser.V is qualified name
77            "ID : 'a'..'z'+ ;\n" +
78            "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n";
79        String found = execParser("T.g", grammar, "TParser", "TLexer",
80                    "a", "a", debug);
81        assertEquals("a<V>\n", found);
82    }
83
84	@Test public void testNamedType() throws Exception {
85		String grammar =
86			"grammar T;\n" +
87			"options {output=AST;}\n" +
88			"@members {static class V extends CommonTree {\n" +
89			"  public V(Token t) { token=t;}\n" +
90			"  public String toString() { return token.getText()+\"<V>\";}\n" +
91			"}\n" +
92			"}\n"+
93			"a : ID<node=V> ;\n"+
94			"ID : 'a'..'z'+ ;\n" +
95			"WS : (' '|'\\n') {$channel=HIDDEN;} ;\n";
96		String found = execParser("T.g", grammar, "TParser", "TLexer",
97					"a", "a", debug);
98		assertEquals("a<V>\n", found);
99	}
100
101
102	@Test public void testTokenWithLabel() throws Exception {
103		String grammar =
104			"grammar T;\n" +
105			"options {output=AST;}\n" +
106			"@members {static class V extends CommonTree {\n" +
107			"  public V(Token t) { token=t;}\n" +
108			"  public String toString() { return token.getText()+\"<V>\";}\n" +
109			"}\n" +
110			"}\n"+
111			"a : x=ID<V> ;\n"+
112			"ID : 'a'..'z'+ ;\n" +
113			"WS : (' '|'\\n') {$channel=HIDDEN;} ;\n";
114		String found = execParser("T.g", grammar, "TParser", "TLexer",
115				    "a", "a", debug);
116		assertEquals("a<V>\n", found);
117	}
118
119	@Test public void testTokenWithListLabel() throws Exception {
120		String grammar =
121			"grammar T;\n" +
122			"options {output=AST;}\n" +
123			"@members {static class V extends CommonTree {\n" +
124			"  public V(Token t) { token=t;}\n" +
125			"  public String toString() { return token.getText()+\"<V>\";}\n" +
126			"}\n" +
127			"}\n"+
128			"a : x+=ID<V> ;\n"+
129			"ID : 'a'..'z'+ ;\n" +
130			"WS : (' '|'\\n') {$channel=HIDDEN;} ;\n";
131		String found = execParser("T.g", grammar, "TParser", "TLexer",
132				    "a", "a", debug);
133		assertEquals("a<V>\n", found);
134	}
135
136	@Test public void testTokenRoot() throws Exception {
137		String grammar =
138			"grammar T;\n" +
139			"options {output=AST;}\n" +
140			"@members {static class V extends CommonTree {\n" +
141			"  public V(Token t) { token=t;}\n" +
142			"  public String toString() { return token.getText()+\"<V>\";}\n" +
143			"}\n" +
144			"}\n"+
145			"a : ID<V>^ ;\n"+
146			"ID : 'a'..'z'+ ;\n" +
147			"WS : (' '|'\\n') {$channel=HIDDEN;} ;\n";
148		String found = execParser("T.g", grammar, "TParser", "TLexer",
149				    "a", "a", debug);
150		assertEquals("a<V>\n", found);
151	}
152
153	@Test public void testTokenRootWithListLabel() throws Exception {
154		String grammar =
155			"grammar T;\n" +
156			"options {output=AST;}\n" +
157			"@members {static class V extends CommonTree {\n" +
158			"  public V(Token t) { token=t;}\n" +
159			"  public String toString() { return token.getText()+\"<V>\";}\n" +
160			"}\n" +
161			"}\n"+
162			"a : x+=ID<V>^ ;\n"+
163			"ID : 'a'..'z'+ ;\n" +
164			"WS : (' '|'\\n') {$channel=HIDDEN;} ;\n";
165		String found = execParser("T.g", grammar, "TParser", "TLexer",
166				    "a", "a", debug);
167		assertEquals("a<V>\n", found);
168	}
169
170	@Test public void testString() throws Exception {
171		String grammar =
172			"grammar T;\n" +
173			"options {output=AST;}\n" +
174			"@members {static class V extends CommonTree {\n" +
175			"  public V(Token t) { token=t;}\n" +
176			"  public String toString() { return token.getText()+\"<V>\";}\n" +
177			"}\n" +
178			"}\n"+
179			"a : 'begin'<V> ;\n"+
180			"ID : 'a'..'z'+ ;\n" +
181			"WS : (' '|'\\n') {$channel=HIDDEN;} ;\n";
182		String found = execParser("T.g", grammar, "TParser", "TLexer",
183				    "a", "begin", debug);
184		assertEquals("begin<V>\n", found);
185	}
186
187	@Test public void testStringRoot() throws Exception {
188		String grammar =
189			"grammar T;\n" +
190			"options {output=AST;}\n" +
191			"@members {static class V extends CommonTree {\n" +
192			"  public V(Token t) { token=t;}\n" +
193			"  public String toString() { return token.getText()+\"<V>\";}\n" +
194			"}\n" +
195			"}\n"+
196			"a : 'begin'<V>^ ;\n"+
197			"ID : 'a'..'z'+ ;\n" +
198			"WS : (' '|'\\n') {$channel=HIDDEN;} ;\n";
199		String found = execParser("T.g", grammar, "TParser", "TLexer",
200				    "a", "begin", debug);
201		assertEquals("begin<V>\n", found);
202	}
203
204	// PARSERS -- REWRITE AST
205
206	@Test public void testRewriteToken() throws Exception {
207		String grammar =
208			"grammar T;\n" +
209			"options {output=AST;}\n" +
210			"@members {static class V extends CommonTree {\n" +
211			"  public V(Token t) { token=t;}\n" +
212			"  public String toString() { return token.getText()+\"<V>\";}\n" +
213			"}\n" +
214			"}\n"+
215			"a : ID -> ID<V> ;\n"+
216			"ID : 'a'..'z'+ ;\n" +
217			"WS : (' '|'\\n') {$channel=HIDDEN;} ;\n";
218		String found = execParser("T.g", grammar, "TParser", "TLexer",
219				    "a", "a", debug);
220		assertEquals("a<V>\n", found);
221	}
222
223	@Test public void testRewriteTokenWithArgs() throws Exception {
224		// arg to ID<V>[42,19,30] means you're constructing node not associated with ID
225		// so must pass in token manually
226		String grammar =
227			"grammar T;\n" +
228			"options {output=AST;}\n" +
229			"@members {\n" +
230			"static class V extends CommonTree {\n" +
231			"  public int x,y,z;\n"+
232			"  public V(int ttype, int x, int y, int z) { this.x=x; this.y=y; this.z=z; token=new CommonToken(ttype,\"\"); }\n" +
233			"  public V(int ttype, Token t, int x) { token=t; this.x=x;}\n" +
234			"  public String toString() { return (token!=null?token.getText():\"\")+\"<V>;\"+x+y+z;}\n" +
235			"}\n" +
236			"}\n"+
237			"a : ID -> ID<V>[42,19,30] ID<V>[$ID,99] ;\n"+
238			"ID : 'a'..'z'+ ;\n" +
239			"WS : (' '|'\\n') {$channel=HIDDEN;} ;\n";
240		String found = execParser("T.g", grammar, "TParser", "TLexer",
241				    "a", "a", debug);
242		assertEquals("<V>;421930 a<V>;9900\n", found);
243	}
244
245	@Test public void testRewriteTokenRoot() throws Exception {
246		String grammar =
247			"grammar T;\n" +
248			"options {output=AST;}\n" +
249			"@members {static class V extends CommonTree {\n" +
250			"  public V(Token t) { token=t;}\n" +
251			"  public String toString() { return token.getText()+\"<V>\";}\n" +
252			"}\n" +
253			"}\n"+
254			"a : ID INT -> ^(ID<V> INT) ;\n"+
255			"ID : 'a'..'z'+ ;\n" +
256			"INT : '0'..'9'+ ;\n" +
257			"WS : (' '|'\\n') {$channel=HIDDEN;} ;\n";
258		String found = execParser("T.g", grammar, "TParser", "TLexer",
259				    "a", "a 2", debug);
260		assertEquals("(a<V> 2)\n", found);
261	}
262
263	@Test public void testRewriteString() throws Exception {
264		String grammar =
265			"grammar T;\n" +
266			"options {output=AST;}\n" +
267			"@members {static class V extends CommonTree {\n" +
268			"  public V(Token t) { token=t;}\n" +
269			"  public String toString() { return token.getText()+\"<V>\";}\n" +
270			"}\n" +
271			"}\n"+
272			"a : 'begin' -> 'begin'<V> ;\n"+
273			"ID : 'a'..'z'+ ;\n" +
274			"WS : (' '|'\\n') {$channel=HIDDEN;} ;\n";
275		String found = execParser("T.g", grammar, "TParser", "TLexer",
276				    "a", "begin", debug);
277		assertEquals("begin<V>\n", found);
278	}
279
280	@Test public void testRewriteStringRoot() throws Exception {
281		String grammar =
282			"grammar T;\n" +
283			"options {output=AST;}\n" +
284			"@members {static class V extends CommonTree {\n" +
285			"  public V(Token t) { token=t;}\n" +
286			"  public String toString() { return token.getText()+\"<V>\";}\n" +
287			"}\n" +
288			"}\n"+
289			"a : 'begin' INT -> ^('begin'<V> INT) ;\n"+
290			"ID : 'a'..'z'+ ;\n" +
291			"INT : '0'..'9'+ ;\n" +
292			"WS : (' '|'\\n') {$channel=HIDDEN;} ;\n";
293		String found = execParser("T.g", grammar, "TParser", "TLexer",
294				    "a", "begin 2", debug);
295		assertEquals("(begin<V> 2)\n", found);
296	}
297
298    @Test public void testRewriteRuleResults() throws Exception {
299        String grammar =
300            "grammar T;\n" +
301            "options {output=AST;}\n" +
302            "tokens {LIST;}\n" +
303            "@members {\n" +
304            "static class V extends CommonTree {\n" +
305            "  public V(Token t) { token=t;}\n" +
306            "  public String toString() { return token.getText()+\"<V>\";}\n" +
307            "}\n" +
308            "static class W extends CommonTree {\n" +
309            "  public W(int tokenType, String txt) { super(new CommonToken(tokenType,txt)); }\n" +
310            "  public W(Token t) { token=t;}\n" +
311            "  public String toString() { return token.getText()+\"<W>\";}\n" +
312            "}\n" +
313            "}\n"+
314            "a : id (',' id)* -> ^(LIST<W>[\"LIST\"] id+);\n" +
315            "id : ID -> ID<V>;\n"+
316            "ID : 'a'..'z'+ ;\n" +
317            "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n";
318        String found = execParser("T.g", grammar, "TParser", "TLexer",
319                    "a", "a,b,c", debug);
320        assertEquals("(LIST<W> a<V> b<V> c<V>)\n", found);
321    }
322
323    @Test public void testCopySemanticsWithHetero() throws Exception {
324        String grammar =
325            "grammar T;\n" +
326            "options {output=AST;}\n" +
327            "@members {\n" +
328            "static class V extends CommonTree {\n" +
329            "  public V(Token t) { token=t;}\n" +  // for 'int'<V>
330            "  public V(V node) { super(node); }\n\n" + // for dupNode
331            "  public Tree dupNode() { return new V(this); }\n" + // for dup'ing type
332            "  public String toString() { return token.getText()+\"<V>\";}\n" +
333            "}\n" +
334            "}\n" +
335            "a : type ID (',' ID)* ';' -> ^(type ID)+;\n" +
336            "type : 'int'<V> ;\n" +
337            "ID : 'a'..'z'+ ;\n" +
338            "INT : '0'..'9'+;\n" +
339            "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n";
340        String found = execParser("T.g", grammar, "TParser", "TLexer",
341                    "a", "int a, b, c;", debug);
342        assertEquals("(int<V> a) (int<V> b) (int<V> c)\n", found);
343    }
344
345    // TREE PARSERS -- REWRITE AST
346
347	@Test public void testTreeParserRewriteFlatList() throws Exception {
348		String grammar =
349			"grammar T;\n" +
350			"options {output=AST;}\n" +
351			"a : ID INT;\n" +
352			"ID : 'a'..'z'+ ;\n" +
353			"INT : '0'..'9'+;\n" +
354			"WS : (' '|'\\n') {$channel=HIDDEN;} ;\n";
355
356		String treeGrammar =
357			"tree grammar TP;\n"+
358			"options {output=AST; ASTLabelType=CommonTree; tokenVocab=T;}\n" +
359			"@members {\n" +
360			"static class V extends CommonTree {\n" +
361			"  public V(Object t) { super((CommonTree)t); }\n" +
362			"  public String toString() { return token.getText()+\"<V>\";}\n" +
363			"}\n" +
364			"static class W extends CommonTree {\n" +
365			"  public W(Object t) { super((CommonTree)t); }\n" +
366			"  public String toString() { return token.getText()+\"<W>\";}\n" +
367			"}\n" +
368			"}\n"+
369			"a : ID INT -> INT<V> ID<W>\n" +
370			"  ;\n";
371
372		String found = execTreeParser("T.g", grammar, "TParser", "TP.g",
373				    treeGrammar, "TP", "TLexer", "a", "a", "abc 34");
374		assertEquals("34<V> abc<W>\n", found);
375	}
376
377	@Test public void testTreeParserRewriteTree() throws Exception {
378		String grammar =
379			"grammar T;\n" +
380			"options {output=AST;}\n" +
381			"a : ID INT;\n" +
382			"ID : 'a'..'z'+ ;\n" +
383			"INT : '0'..'9'+;\n" +
384			"WS : (' '|'\\n') {$channel=HIDDEN;} ;\n";
385
386		String treeGrammar =
387			"tree grammar TP;\n"+
388			"options {output=AST; ASTLabelType=CommonTree; tokenVocab=T;}\n" +
389			"@members {\n" +
390			"static class V extends CommonTree {\n" +
391			"  public V(Object t) { super((CommonTree)t); }\n" +
392			"  public String toString() { return token.getText()+\"<V>\";}\n" +
393			"}\n" +
394			"static class W extends CommonTree {\n" +
395			"  public W(Object t) { super((CommonTree)t); }\n" +
396			"  public String toString() { return token.getText()+\"<W>\";}\n" +
397			"}\n" +
398			"}\n"+
399			"a : ID INT -> ^(INT<V> ID<W>)\n" +
400			"  ;\n";
401
402		String found = execTreeParser("T.g", grammar, "TParser", "TP.g",
403				    treeGrammar, "TP", "TLexer", "a", "a", "abc 34");
404		assertEquals("(34<V> abc<W>)\n", found);
405	}
406
407	@Test public void testTreeParserRewriteImaginary() throws Exception {
408		String grammar =
409			"grammar T;\n" +
410			"options {output=AST;}\n" +
411			"a : ID ;\n" +
412			"ID : 'a'..'z'+ ;\n" +
413			"INT : '0'..'9'+;\n" +
414			"WS : (' '|'\\n') {$channel=HIDDEN;} ;\n";
415
416		String treeGrammar =
417			"tree grammar TP;\n"+
418			"options {output=AST; ASTLabelType=CommonTree; tokenVocab=T;}\n" +
419			"tokens { ROOT; }\n" +
420			"@members {\n" +
421			"class V extends CommonTree {\n" +
422			"  public V(int tokenType) { super(new CommonToken(tokenType)); }\n" +
423			"  public String toString() { return tokenNames[token.getType()]+\"<V>\";}\n" +
424			"}\n" +
425			"}\n"+
426			"a : ID -> ROOT<V> ID\n" +
427			"  ;\n";
428
429		String found = execTreeParser("T.g", grammar, "TParser", "TP.g",
430				    treeGrammar, "TP", "TLexer", "a", "a", "abc");
431		assertEquals("ROOT<V> abc\n", found);
432	}
433
434	@Test public void testTreeParserRewriteImaginaryWithArgs() throws Exception {
435		String grammar =
436			"grammar T;\n" +
437			"options {output=AST;}\n" +
438			"a : ID ;\n" +
439			"ID : 'a'..'z'+ ;\n" +
440			"INT : '0'..'9'+;\n" +
441			"WS : (' '|'\\n') {$channel=HIDDEN;} ;\n";
442
443		String treeGrammar =
444			"tree grammar TP;\n"+
445			"options {output=AST; ASTLabelType=CommonTree; tokenVocab=T;}\n" +
446			"tokens { ROOT; }\n" +
447			"@members {\n" +
448			"class V extends CommonTree {\n" +
449			"  public int x;\n" +
450			"  public V(int tokenType, int x) { super(new CommonToken(tokenType)); this.x=x;}\n" +
451			"  public String toString() { return tokenNames[token.getType()]+\"<V>;\"+x;}\n" +
452			"}\n" +
453			"}\n"+
454			"a : ID -> ROOT<V>[42] ID\n" +
455			"  ;\n";
456
457		String found = execTreeParser("T.g", grammar, "TParser", "TP.g",
458				    treeGrammar, "TP", "TLexer", "a", "a", "abc");
459		assertEquals("ROOT<V>;42 abc\n", found);
460	}
461
462	@Test public void testTreeParserRewriteImaginaryRoot() throws Exception {
463		String grammar =
464			"grammar T;\n" +
465			"options {output=AST;}\n" +
466			"a : ID ;\n" +
467			"ID : 'a'..'z'+ ;\n" +
468			"INT : '0'..'9'+;\n" +
469			"WS : (' '|'\\n') {$channel=HIDDEN;} ;\n";
470
471		String treeGrammar =
472			"tree grammar TP;\n"+
473			"options {output=AST; ASTLabelType=CommonTree; tokenVocab=T;}\n" +
474			"tokens { ROOT; }\n" +
475			"@members {\n" +
476			"class V extends CommonTree {\n" +
477			"  public V(int tokenType) { super(new CommonToken(tokenType)); }\n" +
478			"  public String toString() { return tokenNames[token.getType()]+\"<V>\";}\n" +
479			"}\n" +
480			"}\n"+
481			"a : ID -> ^(ROOT<V> ID)\n" +
482			"  ;\n";
483
484		String found = execTreeParser("T.g", grammar, "TParser", "TP.g",
485				    treeGrammar, "TP", "TLexer", "a", "a", "abc");
486		assertEquals("(ROOT<V> abc)\n", found);
487	}
488
489	@Test public void testTreeParserRewriteImaginaryFromReal() throws Exception {
490		String grammar =
491			"grammar T;\n" +
492			"options {output=AST;}\n" +
493			"a : ID ;\n" +
494			"ID : 'a'..'z'+ ;\n" +
495			"INT : '0'..'9'+;\n" +
496			"WS : (' '|'\\n') {$channel=HIDDEN;} ;\n";
497
498		String treeGrammar =
499			"tree grammar TP;\n"+
500			"options {output=AST; ASTLabelType=CommonTree; tokenVocab=T;}\n" +
501			"tokens { ROOT; }\n" +
502			"@members {\n" +
503			"class V extends CommonTree {\n" +
504			"  public V(int tokenType) { super(new CommonToken(tokenType)); }\n" +
505			"  public V(int tokenType, Object tree) { super((CommonTree)tree); token.setType(tokenType); }\n" +
506			"  public String toString() { return tokenNames[token.getType()]+\"<V>@\"+token.getLine();}\n" +
507			"}\n" +
508			"}\n"+
509			"a : ID -> ROOT<V>[$ID]\n" +
510			"  ;\n";
511
512		String found = execTreeParser("T.g", grammar, "TParser", "TP.g",
513				    treeGrammar, "TP", "TLexer", "a", "a", "abc");
514		assertEquals("ROOT<V>@1\n", found); // at line 1; shows copy of ID's stuff
515	}
516
517	@Test public void testTreeParserAutoHeteroAST() throws Exception {
518		String grammar =
519			"grammar T;\n" +
520			"options {output=AST;}\n" +
521			"a : ID ';' ;\n" +
522			"ID : 'a'..'z'+ ;\n" +
523			"INT : '0'..'9'+;\n" +
524			"WS : (' '|'\\n') {$channel=HIDDEN;} ;\n";
525
526		String treeGrammar =
527			"tree grammar TP;\n"+
528			"options {output=AST; ASTLabelType=CommonTree; tokenVocab=T;}\n" +
529			"tokens { ROOT; }\n" +
530			"@members {\n" +
531			"class V extends CommonTree {\n" +
532			"  public V(CommonTree t) { super(t); }\n" + // NEEDS SPECIAL CTOR
533			"  public String toString() { return super.toString()+\"<V>\";}\n" +
534			"}\n" +
535			"}\n"+
536			"a : ID<V> ';'<V>\n" +
537			"  ;\n";
538
539		String found = execTreeParser("T.g", grammar, "TParser", "TP.g",
540				    treeGrammar, "TP", "TLexer", "a", "a", "abc;");
541		assertEquals("abc<V> ;<V>\n", found);
542	}
543
544}
545