1import unittest
2import textwrap
3import antlr3
4import antlr3.tree
5import testbase
6
7class T(testbase.ANTLRTest):
8    def walkerClass(self, base):
9        class TWalker(base):
10            def __init__(self, *args, **kwargs):
11                base.__init__(self, *args, **kwargs)
12                self.buf = ""
13
14            def traceIn(self, ruleName, ruleIndex):
15                self.traces.append('>'+ruleName)
16
17
18            def traceOut(self, ruleName, ruleIndex):
19                self.traces.append('<'+ruleName)
20
21
22            def recover(self, input, re):
23                # no error recovery yet, just crash!
24                raise
25
26        return TWalker
27
28
29    def execTreeParser(self, grammar, grammarEntry, treeGrammar, treeEntry, input):
30        lexerCls, parserCls = self.compileInlineGrammar(grammar)
31        walkerCls = self.compileInlineGrammar(treeGrammar)
32
33        cStream = antlr3.StringStream(input)
34        lexer = lexerCls(cStream)
35        tStream = antlr3.CommonTokenStream(lexer)
36        parser = parserCls(tStream)
37        r = getattr(parser, grammarEntry)()
38        nodes = antlr3.tree.CommonTreeNodeStream(r.tree)
39        nodes.setTokenStream(tStream)
40        walker = walkerCls(nodes)
41        r = getattr(walker, treeEntry)()
42
43        if r.tree is not None:
44            return r.tree.toStringTree()
45
46        return ""
47
48
49    def testFlatList(self):
50        grammar = textwrap.dedent(
51        r'''
52        grammar T1;
53        options {
54            language=Python;
55            output=AST;
56        }
57        a : ID INT;
58        ID : 'a'..'z'+ ;
59        INT : '0'..'9'+;
60        WS : (' '|'\\n') {$channel=HIDDEN;} ;
61        ''')
62
63        treeGrammar = textwrap.dedent(
64        r'''
65        tree grammar TP1;
66        options {
67            language=Python;
68            output=AST;
69            ASTLabelType=CommonTree;
70            tokenVocab=T1;
71        }
72
73        a : ID INT -> INT ID;
74        ''')
75
76        found = self.execTreeParser(
77            grammar, 'a',
78            treeGrammar, 'a',
79            "abc 34"
80            )
81
82        self.failUnlessEqual("34 abc", found)
83
84
85    def testSimpleTree(self):
86        grammar = textwrap.dedent(
87        r'''
88        grammar T2;
89        options {
90            language=Python;
91            output=AST;
92        }
93        a : ID INT -> ^(ID INT);
94        ID : 'a'..'z'+ ;
95        INT : '0'..'9'+;
96        WS : (' '|'\\n') {$channel=HIDDEN;} ;
97        ''')
98
99        treeGrammar = textwrap.dedent(
100        r'''
101        tree grammar TP2;
102        options {
103            language=Python;
104            output=AST;
105            ASTLabelType=CommonTree;
106            tokenVocab=T2;
107        }
108        a : ^(ID INT) -> ^(INT ID);
109        ''')
110
111        found = self.execTreeParser(
112            grammar, 'a',
113            treeGrammar, 'a',
114            "abc 34"
115            )
116
117        self.failUnlessEqual("(34 abc)", found)
118
119
120    def testCombinedRewriteAndAuto(self):
121        grammar = textwrap.dedent(
122        r'''
123        grammar T3;
124        options {
125            language=Python;
126            output=AST;
127        }
128        a : ID INT -> ^(ID INT) | INT ;
129        ID : 'a'..'z'+ ;
130        INT : '0'..'9'+;
131        WS : (' '|'\\n') {$channel=HIDDEN;} ;
132        ''')
133
134        treeGrammar = textwrap.dedent(
135        r'''
136        tree grammar TP3;
137        options {
138            language=Python;
139            output=AST;
140            ASTLabelType=CommonTree;
141            tokenVocab=T3;
142        }
143        a : ^(ID INT) -> ^(INT ID) | INT;
144        ''')
145
146        found = self.execTreeParser(
147            grammar, 'a',
148            treeGrammar, 'a',
149            "abc 34"
150            )
151
152        self.failUnlessEqual("(34 abc)", found)
153
154
155        found = self.execTreeParser(
156            grammar, 'a',
157            treeGrammar, 'a',
158            "34"
159            )
160
161        self.failUnlessEqual("34", found)
162
163
164    def testAvoidDup(self):
165        grammar = textwrap.dedent(
166        r'''
167        grammar T4;
168        options {
169            language=Python;
170            output=AST;
171        }
172        a : ID ;
173        ID : 'a'..'z'+ ;
174        INT : '0'..'9'+;
175        WS : (' '|'\\n') {$channel=HIDDEN;} ;
176        ''')
177
178        treeGrammar = textwrap.dedent(
179        r'''
180        tree grammar TP4;
181        options {
182            language=Python;
183            output=AST;
184            ASTLabelType=CommonTree;
185            tokenVocab=T4;
186        }
187        a : ID -> ^(ID ID);
188        ''')
189
190        found = self.execTreeParser(
191            grammar, 'a',
192            treeGrammar, 'a',
193            "abc"
194            )
195
196        self.failUnlessEqual("(abc abc)", found)
197
198
199    def testLoop(self):
200        grammar = textwrap.dedent(
201        r'''
202        grammar T5;
203        options {
204            language=Python;
205            output=AST;
206        }
207        a : ID+ INT+ -> (^(ID INT))+ ;
208        ID : 'a'..'z'+ ;
209        INT : '0'..'9'+;
210        WS : (' '|'\\n') {$channel=HIDDEN;} ;
211        ''')
212
213        treeGrammar = textwrap.dedent(
214        r'''
215        tree grammar TP5;
216        options {
217            language=Python;
218            output=AST;
219            ASTLabelType=CommonTree;
220            tokenVocab=T5;
221        }
222        a : (^(ID INT))+ -> INT+ ID+;
223        ''')
224
225        found = self.execTreeParser(
226            grammar, 'a',
227            treeGrammar, 'a',
228            "a b c 3 4 5"
229            )
230
231        self.failUnlessEqual("3 4 5 a b c", found)
232
233
234    def testAutoDup(self):
235        grammar = textwrap.dedent(
236        r'''
237        grammar T6;
238        options {
239            language=Python;
240            output=AST;
241        }
242        a : ID ;
243        ID : 'a'..'z'+ ;
244        INT : '0'..'9'+;
245        WS : (' '|'\\n') {$channel=HIDDEN;} ;
246        ''')
247
248        treeGrammar = textwrap.dedent(
249        r'''
250        tree grammar TP6;
251        options {
252            language=Python;
253            output=AST;
254            ASTLabelType=CommonTree;
255            tokenVocab=T6;
256        }
257        a : ID;
258        ''')
259
260        found = self.execTreeParser(
261            grammar, 'a',
262            treeGrammar, 'a',
263            "abc"
264            )
265
266        self.failUnlessEqual("abc", found)
267
268
269    def testAutoDupRule(self):
270        grammar = textwrap.dedent(
271        r'''
272        grammar T7;
273        options {
274            language=Python;
275            output=AST;
276        }
277        a : ID INT ;
278        ID : 'a'..'z'+ ;
279        INT : '0'..'9'+;
280        WS : (' '|'\\n') {$channel=HIDDEN;} ;
281        ''')
282
283        treeGrammar = textwrap.dedent(
284        r'''
285        tree grammar TP7;
286        options {
287            language=Python;
288            output=AST;
289            ASTLabelType=CommonTree;
290            tokenVocab=T7;
291        }
292        a : b c ;
293        b : ID ;
294        c : INT ;
295        ''')
296
297        found = self.execTreeParser(
298            grammar, 'a',
299            treeGrammar, 'a',
300            "a 1"
301            )
302
303        self.failUnlessEqual("a 1", found)
304
305
306    def testAutoWildcard(self):
307        grammar = textwrap.dedent(
308            r'''
309            grammar T;
310            options {language=Python;output=AST;}
311            a : ID INT ;
312            ID : 'a'..'z'+ ;
313            INT : '0'..'9'+;
314            WS : (' '|'\n') {$channel=HIDDEN;} ;
315            ''')
316
317        treeGrammar = textwrap.dedent(
318            r'''
319            tree grammar TP;
320            options {language=Python;output=AST; ASTLabelType=CommonTree; tokenVocab=T;}
321            a : ID .
322              ;
323            ''')
324
325        found = self.execTreeParser(
326            grammar, 'a',
327            treeGrammar, 'a',
328            "abc 34")
329        self.assertEquals("abc 34", found)
330
331
332#     def testNoWildcardAsRootError(self):
333#         ErrorQueue equeue = new ErrorQueue();
334#         ErrorManager.setErrorListener(equeue);
335# >
336#         String treeGrammar =
337#             "tree grammar TP;\n"+
338#             "options {language=Python;output=AST;}
339#             "a : ^(. INT)
340#             "  ;\n";
341# >
342#         Grammar g = new Grammar(treeGrammar);
343#         Tool antlr = newTool();
344#         antlr.setOutputDirectory(null); // write to /dev/null
345#         CodeGenerator generator = new CodeGenerator(antlr, g, "Java");
346#         g.setCodeGenerator(generator);
347#         generator.genRecognizer();
348# >
349#         assertEquals("unexpected errors: "+equeue, 1, equeue.errors.size());
350# >
351#         int expectedMsgID = ErrorManager.MSG_WILDCARD_AS_ROOT;
352#         Object expectedArg = null;
353#         antlr.RecognitionException expectedExc = null;
354#         GrammarSyntaxMessage expectedMessage =
355#             new GrammarSyntaxMessage(expectedMsgID, g, null, expectedArg, expectedExc);
356# >
357#         checkError(equeue, expectedMessage);
358#     }
359
360    def testAutoWildcard2(self):
361        grammar = textwrap.dedent(
362            r'''
363            grammar T;
364            options {language=Python;output=AST;}
365            a : ID INT -> ^(ID INT);
366            ID : 'a'..'z'+ ;
367            INT : '0'..'9'+;
368            WS : (' '|'\n') {$channel=HIDDEN;} ;
369            ''')
370
371        treeGrammar = textwrap.dedent(
372            r'''
373            tree grammar TP;
374            options {language=Python;output=AST; ASTLabelType=CommonTree; tokenVocab=T;}
375            a : ^(ID .)
376              ;
377            ''')
378
379        found = self.execTreeParser(
380            grammar, 'a',
381            treeGrammar, 'a',
382            "abc 34")
383        self.assertEquals("(abc 34)", found)
384
385
386    def testAutoWildcardWithLabel(self):
387        grammar = textwrap.dedent(
388            r'''
389            grammar T;
390            options {language=Python;output=AST;}
391            a : ID INT ;
392            ID : 'a'..'z'+ ;
393            INT : '0'..'9'+;
394            WS : (' '|'\n') {$channel=HIDDEN;} ;
395            ''')
396
397        treeGrammar = textwrap.dedent(
398            r'''
399            tree grammar TP;
400            options {language=Python;output=AST; ASTLabelType=CommonTree; tokenVocab=T;}
401            a : ID c=.
402              ;
403            ''')
404
405        found = self.execTreeParser(
406            grammar, 'a',
407            treeGrammar, 'a',
408            "abc 34")
409        self.assertEquals("abc 34", found)
410
411
412    def testAutoWildcardWithListLabel(self):
413        grammar = textwrap.dedent(
414            r'''
415            grammar T;
416            options {language=Python;output=AST;}
417            a : ID INT ;
418            ID : 'a'..'z'+ ;
419            INT : '0'..'9'+;
420            WS : (' '|'\n') {$channel=HIDDEN;} ;
421            ''')
422
423        treeGrammar = textwrap.dedent(
424            r'''
425            tree grammar TP;
426            options {language=Python;output=AST; ASTLabelType=CommonTree; tokenVocab=T;}
427            a : ID c+=.
428              ;
429            ''')
430
431        found = self.execTreeParser(
432            grammar, 'a',
433            treeGrammar, 'a',
434            "abc 34")
435        self.assertEquals("abc 34", found)
436
437
438    def testAutoDupMultiple(self):
439        grammar = textwrap.dedent(
440        r'''
441        grammar T8;
442        options {
443            language=Python;
444            output=AST;
445        }
446        a : ID ID INT;
447        ID : 'a'..'z'+ ;
448        INT : '0'..'9'+;
449        WS : (' '|'\\n') {$channel=HIDDEN;} ;
450        ''')
451
452        treeGrammar = textwrap.dedent(
453        r'''
454        tree grammar TP8;
455        options {
456            language=Python;
457            output=AST;
458            ASTLabelType=CommonTree;
459            tokenVocab=T8;
460        }
461        a : ID ID INT
462          ;
463        ''')
464
465        found = self.execTreeParser(
466            grammar, 'a',
467            treeGrammar, 'a',
468            "a b 3"
469            )
470
471        self.failUnlessEqual("a b 3", found)
472
473
474    def testAutoDupTree(self):
475        grammar = textwrap.dedent(
476        r'''
477        grammar T9;
478        options {
479            language=Python;
480            output=AST;
481        }
482        a : ID INT -> ^(ID INT);
483        ID : 'a'..'z'+ ;
484        INT : '0'..'9'+;
485        WS : (' '|'\\n') {$channel=HIDDEN;} ;
486        ''')
487
488        treeGrammar = textwrap.dedent(
489        r'''
490        tree grammar TP9;
491        options {
492            language=Python;
493            output=AST;
494            ASTLabelType=CommonTree;
495            tokenVocab=T9;
496        }
497        a : ^(ID INT)
498          ;
499        ''')
500
501        found = self.execTreeParser(
502            grammar, 'a',
503            treeGrammar, 'a',
504            "a 3"
505            )
506
507        self.failUnlessEqual("(a 3)", found)
508
509
510    def testAutoDupTreeWithLabels(self):
511        grammar = textwrap.dedent(
512        r'''
513        grammar T10;
514        options {
515            language=Python;
516            output=AST;
517        }
518        a : ID INT -> ^(ID INT);
519        ID : 'a'..'z'+ ;
520        INT : '0'..'9'+;
521        WS : (' '|'\\n') {$channel=HIDDEN;} ;
522        ''')
523
524        treeGrammar = textwrap.dedent(
525        r'''
526        tree grammar TP10;
527        options {
528            language=Python;
529            output=AST;
530            ASTLabelType=CommonTree;
531            tokenVocab=T10;
532        }
533        a : ^(x=ID y=INT)
534          ;
535        ''')
536
537        found = self.execTreeParser(
538            grammar, 'a',
539            treeGrammar, 'a',
540            "a 3"
541            )
542
543        self.failUnlessEqual("(a 3)", found)
544
545
546    def testAutoDupTreeWithListLabels(self):
547        grammar = textwrap.dedent(
548        r'''
549        grammar T11;
550        options {
551            language=Python;
552            output=AST;
553        }
554        a : ID INT -> ^(ID INT);
555        ID : 'a'..'z'+ ;
556        INT : '0'..'9'+;
557        WS : (' '|'\\n') {$channel=HIDDEN;} ;
558        ''')
559
560        treeGrammar = textwrap.dedent(
561        r'''
562        tree grammar TP11;
563        options {
564            language=Python;
565            output=AST;
566            ASTLabelType=CommonTree;
567            tokenVocab=T11;
568        }
569        a : ^(x+=ID y+=INT)
570          ;
571        ''')
572
573        found = self.execTreeParser(
574            grammar, 'a',
575            treeGrammar, 'a',
576            "a 3"
577            )
578
579        self.failUnlessEqual("(a 3)", found)
580
581
582    def testAutoDupTreeWithRuleRoot(self):
583        grammar = textwrap.dedent(
584        r'''
585        grammar T12;
586        options {
587            language=Python;
588            output=AST;
589        }
590        a : ID INT -> ^(ID INT);
591        ID : 'a'..'z'+ ;
592        INT : '0'..'9'+;
593        WS : (' '|'\\n') {$channel=HIDDEN;} ;
594        ''')
595
596        treeGrammar = textwrap.dedent(
597        r'''
598        tree grammar TP12;
599        options {
600            language=Python;
601            output=AST;
602            ASTLabelType=CommonTree;
603            tokenVocab=T12;
604        }
605        a : ^(b INT) ;
606        b : ID ;
607        ''')
608
609        found = self.execTreeParser(
610            grammar, 'a',
611            treeGrammar, 'a',
612            "a 3"
613            )
614
615        self.failUnlessEqual("(a 3)", found)
616
617
618    def testAutoDupTreeWithRuleRootAndLabels(self):
619        grammar = textwrap.dedent(
620        r'''
621        grammar T13;
622        options {
623            language=Python;
624            output=AST;
625        }
626        a : ID INT -> ^(ID INT);
627        ID : 'a'..'z'+ ;
628        INT : '0'..'9'+;
629        WS : (' '|'\\n') {$channel=HIDDEN;} ;
630        ''')
631
632        treeGrammar = textwrap.dedent(
633        r'''
634        tree grammar TP13;
635        options {
636            language=Python;
637            output=AST;
638            ASTLabelType=CommonTree;
639            tokenVocab=T13;
640        }
641        a : ^(x=b INT) ;
642        b : ID ;
643        ''')
644
645        found = self.execTreeParser(
646            grammar, 'a',
647            treeGrammar, 'a',
648            "a 3"
649            )
650
651        self.failUnlessEqual("(a 3)", found)
652
653
654    def testAutoDupTreeWithRuleRootAndListLabels(self):
655        grammar = textwrap.dedent(
656        r'''
657        grammar T14;
658        options {
659            language=Python;
660            output=AST;
661        }
662        a : ID INT -> ^(ID INT);
663        ID : 'a'..'z'+ ;
664        INT : '0'..'9'+;
665        WS : (' '|'\\n') {$channel=HIDDEN;} ;
666        ''')
667
668        treeGrammar = textwrap.dedent(
669        r'''
670        tree grammar TP14;
671        options {
672            language=Python;
673            output=AST;
674            ASTLabelType=CommonTree;
675            tokenVocab=T14;
676        }
677        a : ^(x+=b y+=c) ;
678        b : ID ;
679        c : INT ;
680        ''')
681
682        found = self.execTreeParser(
683            grammar, 'a',
684            treeGrammar, 'a',
685            "a 3"
686            )
687
688        self.failUnlessEqual("(a 3)", found)
689
690
691    def testAutoDupNestedTree(self):
692        grammar = textwrap.dedent(
693        r'''
694        grammar T15;
695        options {
696            language=Python;
697            output=AST;
698        }
699        a : x=ID y=ID INT -> ^($x ^($y INT));
700        ID : 'a'..'z'+ ;
701        INT : '0'..'9'+;
702        WS : (' '|'\\n') {$channel=HIDDEN;} ;
703        ''')
704
705        treeGrammar = textwrap.dedent(
706        r'''
707        tree grammar TP15;
708        options {
709            language=Python;
710            output=AST;
711            ASTLabelType=CommonTree;
712            tokenVocab=T15;
713        }
714        a : ^(ID ^(ID INT))
715          ;
716        ''')
717
718        found = self.execTreeParser(
719            grammar, 'a',
720            treeGrammar, 'a',
721            "a b 3"
722            )
723
724        self.failUnlessEqual("(a (b 3))", found)
725
726
727    def testDelete(self):
728        grammar = textwrap.dedent(
729        r'''
730        grammar T16;
731        options {
732            language=Python;
733            output=AST;
734        }
735        a : ID ;
736        ID : 'a'..'z'+ ;
737        INT : '0'..'9'+;
738        WS : (' '|'\\n') {$channel=HIDDEN;} ;
739        ''')
740
741        treeGrammar = textwrap.dedent(
742        r'''
743        tree grammar TP16;
744        options {
745            language=Python;
746            output=AST;
747            ASTLabelType=CommonTree;
748            tokenVocab=T16;
749        }
750        a : ID ->
751          ;
752        ''')
753
754        found = self.execTreeParser(
755            grammar, 'a',
756            treeGrammar, 'a',
757            "abc"
758            )
759
760        self.failUnlessEqual("", found)
761
762    def testSetMatchNoRewrite(self):
763        grammar = textwrap.dedent(
764            r'''
765            grammar T;
766            options {
767                language=Python;
768                output=AST;
769            }
770            a : ID INT ;
771            ID : 'a'..'z'+ ;
772            INT : '0'..'9'+;
773            WS : (' '|'\n') {$channel=HIDDEN;} ;
774            ''')
775
776        treeGrammar = textwrap.dedent(
777            r'''
778            tree grammar TP;
779            options {
780                language=Python;
781                output=AST;
782                ASTLabelType=CommonTree;
783                tokenVocab=T;
784            }
785            a : b INT;
786            b : ID | INT;
787            ''')
788
789        found = self.execTreeParser(
790            grammar, 'a',
791            treeGrammar, 'a',
792            "abc 34"
793            )
794
795        self.failUnlessEqual("abc 34", found)
796
797
798    def testSetOptionalMatchNoRewrite(self):
799        grammar = textwrap.dedent(
800            r'''
801            grammar T;
802            options {
803                language=Python;
804                output=AST;
805            }
806            a : ID INT ;
807            ID : 'a'..'z'+ ;
808            INT : '0'..'9'+;
809            WS : (' '|'\n') {$channel=HIDDEN;} ;
810            ''')
811
812        treeGrammar = textwrap.dedent(
813            r'''
814            tree grammar TP;
815            options {
816                language=Python;
817                output=AST;
818                ASTLabelType=CommonTree;
819                tokenVocab=T;
820            }
821            a : (ID|INT)? INT ;
822            ''')
823
824        found = self.execTreeParser(
825            grammar, 'a',
826            treeGrammar, 'a',
827            "abc 34")
828
829        self.failUnlessEqual("abc 34", found)
830
831
832    def testSetMatchNoRewriteLevel2(self):
833        grammar = textwrap.dedent(
834            r'''
835            grammar T;
836            options {
837                language=Python;
838                output=AST;
839            }
840            a : x=ID INT -> ^($x INT);
841            ID : 'a'..'z'+ ;
842            INT : '0'..'9'+;
843            WS : (' '|'\n') {$channel=HIDDEN;} ;
844            ''')
845
846        treeGrammar = textwrap.dedent(
847            r'''
848            tree grammar TP;
849            options {
850                language=Python;
851                output=AST;
852                ASTLabelType=CommonTree;
853                tokenVocab=T;
854            }
855            a : ^(ID (ID | INT) ) ;
856            ''')
857
858        found = self.execTreeParser(
859            grammar, 'a',
860            treeGrammar, 'a',
861            "abc 34"
862            )
863
864        self.failUnlessEqual("(abc 34)", found)
865
866
867    def testSetMatchNoRewriteLevel2Root(self):
868        grammar = textwrap.dedent(
869            r'''
870            grammar T;
871            options {
872                language=Python;
873                output=AST;
874            }
875            a : x=ID INT -> ^($x INT);
876            ID : 'a'..'z'+ ;
877            INT : '0'..'9'+;
878            WS : (' '|'\n') {$channel=HIDDEN;} ;
879            ''')
880
881        treeGrammar = textwrap.dedent(
882            r'''
883            tree grammar TP;
884            options {
885                language=Python;
886                output=AST;
887                ASTLabelType=CommonTree;
888                tokenVocab=T;
889            }
890            a : ^((ID | INT) INT) ;
891            ''')
892
893        found = self.execTreeParser(
894            grammar, 'a',
895            treeGrammar, 'a',
896            "abc 34"
897            )
898
899        self.failUnlessEqual("(abc 34)", found)
900
901
902    ## REWRITE MODE
903
904    def testRewriteModeCombinedRewriteAndAuto(self):
905        grammar = textwrap.dedent(
906        r'''
907        grammar T17;
908        options {
909            language=Python;
910            output=AST;
911        }
912        a : ID INT -> ^(ID INT) | INT ;
913        ID : 'a'..'z'+ ;
914        INT : '0'..'9'+;
915        WS : (' '|'\\n') {$channel=HIDDEN;} ;
916        ''')
917
918        treeGrammar = textwrap.dedent(
919        r'''
920        tree grammar TP17;
921        options {
922            language=Python;
923            output=AST;
924            ASTLabelType=CommonTree;
925            tokenVocab=T17;
926            rewrite=true;
927        }
928        a : ^(ID INT) -> ^(ID["ick"] INT)
929          | INT // leaves it alone, returning $a.start
930          ;
931        ''')
932
933        found = self.execTreeParser(
934            grammar, 'a',
935            treeGrammar, 'a',
936            "abc 34"
937            )
938
939        self.failUnlessEqual("(ick 34)", found)
940
941
942        found = self.execTreeParser(
943            grammar, 'a',
944            treeGrammar, 'a',
945            "34"
946            )
947
948        self.failUnlessEqual("34", found)
949
950
951    def testRewriteModeFlatTree(self):
952        grammar = textwrap.dedent(
953            r'''
954            grammar T18;
955            options {
956              language=Python;
957              output=AST;
958            }
959            a : ID INT -> ID INT | INT ;
960            ID : 'a'..'z'+ ;
961            INT : '0'..'9'+;
962            WS : (' '|'\n') {$channel=HIDDEN;} ;
963            ''')
964
965        treeGrammar = textwrap.dedent(
966            r'''
967            tree grammar TP18;
968            options {
969              language=Python;
970              output=AST;
971              ASTLabelType=CommonTree;
972              tokenVocab=T18;
973              rewrite=true;
974            }
975            s : ID a ;
976            a : INT -> INT["1"]
977              ;
978            ''')
979
980        found = self.execTreeParser(
981            grammar, 'a',
982            treeGrammar, 's',
983            "abc 34"
984            )
985        self.assertEquals("abc 1", found)
986
987
988    def testRewriteModeChainRuleFlatTree(self):
989        grammar = textwrap.dedent(
990            r'''
991            grammar T;
992            options {language=Python; output=AST;}
993            a : ID INT -> ID INT | INT ;
994            ID : 'a'..'z'+ ;
995            INT : '0'..'9'+;
996            WS : (' '|'\n') {$channel=HIDDEN;} ;
997            ''')
998
999        treeGrammar = textwrap.dedent(
1000            r'''
1001            tree grammar TP;
1002            options {language=Python; output=AST; ASTLabelType=CommonTree; tokenVocab=T; rewrite=true;}
1003            s : a ;
1004            a : b ;
1005            b : ID INT -> INT ID
1006              ;
1007            ''')
1008
1009        found = self.execTreeParser(
1010            grammar, 'a',
1011            treeGrammar, 's',
1012            "abc 34")
1013        self.assertEquals("34 abc", found)
1014
1015
1016    def testRewriteModeChainRuleTree(self):
1017        grammar = textwrap.dedent(
1018            r'''
1019            grammar T;
1020            options {language=Python; output=AST;}
1021            a : ID INT -> ^(ID INT) ;
1022            ID : 'a'..'z'+ ;
1023            INT : '0'..'9'+;
1024            WS : (' '|'\n') {$channel=HIDDEN;} ;
1025            ''')
1026
1027        treeGrammar = textwrap.dedent(
1028            r'''
1029            tree grammar TP;
1030            options {language=Python; output=AST; ASTLabelType=CommonTree; tokenVocab=T; rewrite=true;}
1031            s : a ;
1032            a : b ; // a.tree must become b.tree
1033            b : ^(ID INT) -> INT
1034              ;
1035            ''')
1036
1037        found = self.execTreeParser(
1038            grammar, 'a',
1039            treeGrammar, 's',
1040            "abc 34")
1041        self.assertEquals("34", found)
1042
1043
1044    def testRewriteModeChainRuleTree2(self):
1045        grammar = textwrap.dedent(
1046            r'''
1047            grammar T;
1048            options {language=Python; output=AST;}
1049            a : ID INT -> ^(ID INT) ;
1050            ID : 'a'..'z'+ ;
1051            INT : '0'..'9'+;
1052            WS : (' '|'\n') {$channel=HIDDEN;} ;
1053            ''')
1054
1055        treeGrammar = textwrap.dedent(
1056            r"""
1057            tree grammar TP;
1058            options {language=Python; output=AST; ASTLabelType=CommonTree; tokenVocab=T; rewrite=true;}
1059            tokens { X; }
1060            s : a* b ; // only b contributes to tree, but it's after a*; s.tree = b.tree
1061            a : X ;
1062            b : ^(ID INT) -> INT
1063              ;
1064            """)
1065
1066        found = self.execTreeParser(
1067            grammar, 'a',
1068            treeGrammar, 's',
1069            "abc 34")
1070        self.assertEquals("34", found)
1071
1072
1073    def testRewriteModeChainRuleTree3(self):
1074        grammar = textwrap.dedent(
1075            r'''
1076            grammar T;
1077            options {language=Python; output=AST;}
1078            a : 'boo' ID INT -> 'boo' ^(ID INT) ;
1079            ID : 'a'..'z'+ ;
1080            INT : '0'..'9'+;
1081            WS : (' '|'\n') {$channel=HIDDEN;} ;
1082            ''')
1083
1084        treeGrammar = textwrap.dedent(
1085            r"""
1086            tree grammar TP;
1087            options {language=Python; output=AST; ASTLabelType=CommonTree; tokenVocab=T; rewrite=true;}
1088            tokens { X; }
1089            s : 'boo' a* b ; // don't reset s.tree to b.tree due to 'boo'
1090            a : X ;
1091            b : ^(ID INT) -> INT
1092              ;
1093            """)
1094
1095        found = self.execTreeParser(
1096            grammar, 'a',
1097            treeGrammar, 's',
1098            "boo abc 34")
1099        self.assertEquals("boo 34", found)
1100
1101
1102    def testRewriteModeChainRuleTree4(self):
1103        grammar = textwrap.dedent(
1104            r"""
1105            grammar T;
1106            options {language=Python; output=AST;}
1107            a : 'boo' ID INT -> ^('boo' ^(ID INT)) ;
1108            ID : 'a'..'z'+ ;
1109            INT : '0'..'9'+;
1110            WS : (' '|'\n') {$channel=HIDDEN;} ;
1111            """)
1112
1113        treeGrammar = textwrap.dedent(
1114            r"""
1115            tree grammar TP;
1116            options {language=Python; output=AST; ASTLabelType=CommonTree; tokenVocab=T; rewrite=true;}
1117            tokens { X; }
1118            s : ^('boo' a* b) ; // don't reset s.tree to b.tree due to 'boo'
1119            a : X ;
1120            b : ^(ID INT) -> INT
1121              ;
1122            """)
1123
1124        found = self.execTreeParser(
1125            grammar, 'a',
1126            treeGrammar, 's',
1127            "boo abc 34")
1128        self.assertEquals("(boo 34)", found)
1129
1130
1131    def testRewriteModeChainRuleTree5(self):
1132        grammar = textwrap.dedent(
1133            r"""
1134            grammar T;
1135            options {language=Python; output=AST;}
1136            a : 'boo' ID INT -> ^('boo' ^(ID INT)) ;
1137            ID : 'a'..'z'+ ;
1138            INT : '0'..'9'+;
1139            WS : (' '|'\n') {$channel=HIDDEN;} ;
1140            """)
1141
1142        treeGrammar = textwrap.dedent(
1143            r"""
1144            tree grammar TP;
1145            options {language=Python; output=AST; ASTLabelType=CommonTree; tokenVocab=T; rewrite=true;}
1146            tokens { X; }
1147            s : ^(a b) ; // s.tree is a.tree
1148            a : 'boo' ;
1149            b : ^(ID INT) -> INT
1150              ;
1151            """)
1152
1153        found = self.execTreeParser(
1154            grammar, 'a',
1155            treeGrammar, 's',
1156            "boo abc 34")
1157        self.assertEquals("(boo 34)", found)
1158
1159
1160    def testRewriteOfRuleRef(self):
1161        grammar = textwrap.dedent(
1162            r"""
1163            grammar T;
1164            options {language=Python; output=AST;}
1165            a : ID INT -> ID INT | INT ;
1166            ID : 'a'..'z'+ ;
1167            INT : '0'..'9'+;
1168            WS : (' '|'\n') {$channel=HIDDEN;} ;
1169            """)
1170
1171        treeGrammar = textwrap.dedent(
1172            r"""
1173            tree grammar TP;
1174            options {language=Python; output=AST; ASTLabelType=CommonTree; tokenVocab=T; rewrite=true;}
1175            s : a -> a ;
1176            a : ID INT -> ID INT ;
1177            """)
1178
1179        found = self.execTreeParser(
1180            grammar, 'a',
1181            treeGrammar, 's',
1182            "abc 34")
1183        self.failUnlessEqual("abc 34", found)
1184
1185
1186    def testRewriteOfRuleRefRoot(self):
1187        grammar = textwrap.dedent(
1188            r"""
1189            grammar T;
1190            options {language=Python; output=AST;}
1191            a : ID INT INT -> ^(INT ^(ID INT));
1192            ID : 'a'..'z'+ ;
1193            INT : '0'..'9'+;
1194            WS : (' '|'\n') {$channel=HIDDEN;} ;
1195            """)
1196
1197        treeGrammar = textwrap.dedent(
1198            r"""
1199            tree grammar TP;
1200            options {language=Python; output=AST; ASTLabelType=CommonTree; tokenVocab=T; rewrite=true;}
1201            s : ^(a ^(ID INT)) -> a ;
1202            a : INT ;
1203            """)
1204
1205        found = self.execTreeParser(
1206            grammar, 'a',
1207            treeGrammar, 's',
1208            "abc 12 34")
1209        # emits whole tree when you ref the root since I can't know whether
1210        # you want the children or not.  You might be returning a whole new
1211        # tree.  Hmm...still seems weird.  oh well.
1212        self.failUnlessEqual("(12 (abc 34))", found)
1213
1214
1215    def testRewriteOfRuleRefRootLabeled(self):
1216        grammar = textwrap.dedent(
1217            r"""
1218            grammar T;
1219            options {language=Python; output=AST;}
1220            a : ID INT INT -> ^(INT ^(ID INT));
1221            ID : 'a'..'z'+ ;
1222            INT : '0'..'9'+;
1223            WS : (' '|'\n') {$channel=HIDDEN;} ;
1224            """)
1225
1226        treeGrammar = textwrap.dedent(
1227            r"""
1228            tree grammar TP;
1229            options {language=Python; output=AST; ASTLabelType=CommonTree; tokenVocab=T; rewrite=true;}
1230            s : ^(label=a ^(ID INT)) -> a ;
1231            a : INT ;
1232            """)
1233
1234        found = self.execTreeParser(
1235            grammar, 'a',
1236            treeGrammar, 's',
1237            "abc 12 34")
1238        # emits whole tree when you ref the root since I can't know whether
1239        # you want the children or not.  You might be returning a whole new
1240        # tree.  Hmm...still seems weird.  oh well.
1241        self.failUnlessEqual("(12 (abc 34))", found)
1242
1243
1244    def testRewriteOfRuleRefRootListLabeled(self):
1245        grammar = textwrap.dedent(
1246            r"""
1247            grammar T;
1248            options {language=Python; output=AST;}
1249            a : ID INT INT -> ^(INT ^(ID INT));
1250            ID : 'a'..'z'+ ;
1251            INT : '0'..'9'+;
1252            WS : (' '|'\n') {$channel=HIDDEN;} ;
1253            """)
1254
1255        treeGrammar = textwrap.dedent(
1256            r"""
1257            tree grammar TP;
1258            options {language=Python; output=AST; ASTLabelType=CommonTree; tokenVocab=T; rewrite=true;}
1259            s : ^(label+=a ^(ID INT)) -> a ;
1260            a : INT ;
1261            """)
1262
1263        found = self.execTreeParser(
1264            grammar, 'a',
1265            treeGrammar, 's',
1266            "abc 12 34")
1267        # emits whole tree when you ref the root since I can't know whether
1268        # you want the children or not.  You might be returning a whole new
1269        # tree.  Hmm...still seems weird.  oh well.
1270        self.failUnlessEqual("(12 (abc 34))", found)
1271
1272
1273    def testRewriteOfRuleRefChild(self):
1274        grammar = textwrap.dedent(
1275            r"""
1276            grammar T;
1277            options {language=Python; output=AST;}
1278            a : ID INT -> ^(ID ^(INT INT));
1279            ID : 'a'..'z'+ ;
1280            INT : '0'..'9'+;
1281            WS : (' '|'\n') {$channel=HIDDEN;} ;
1282            """)
1283
1284        treeGrammar = textwrap.dedent(
1285            r"""
1286            tree grammar TP;
1287            options {language=Python; output=AST; ASTLabelType=CommonTree; tokenVocab=T; rewrite=true;}
1288            s : ^(ID a) -> a ;
1289            a : ^(INT INT) ;
1290            """)
1291
1292        found = self.execTreeParser(
1293            grammar, 'a',
1294            treeGrammar, 's',
1295            "abc 34")
1296        self.failUnlessEqual("(34 34)", found)
1297
1298
1299    def testRewriteOfRuleRefLabel(self):
1300        grammar = textwrap.dedent(
1301            r"""
1302            grammar T;
1303            options {language=Python; output=AST;}
1304            a : ID INT -> ^(ID ^(INT INT));
1305            ID : 'a'..'z'+ ;
1306            INT : '0'..'9'+;
1307            WS : (' '|'\n') {$channel=HIDDEN;} ;
1308            """)
1309
1310        treeGrammar = textwrap.dedent(
1311            r"""
1312            tree grammar TP;
1313            options {language=Python; output=AST; ASTLabelType=CommonTree; tokenVocab=T; rewrite=true;}
1314            s : ^(ID label=a) -> a ;
1315            a : ^(INT INT) ;
1316            """)
1317
1318        found = self.execTreeParser(
1319            grammar, 'a',
1320            treeGrammar, 's',
1321            "abc 34")
1322        self.failUnlessEqual("(34 34)", found)
1323
1324
1325    def testRewriteOfRuleRefListLabel(self):
1326        grammar = textwrap.dedent(
1327            r"""
1328            grammar T;
1329            options {language=Python; output=AST;}
1330            a : ID INT -> ^(ID ^(INT INT));
1331            ID : 'a'..'z'+ ;
1332            INT : '0'..'9'+;
1333            WS : (' '|'\n') {$channel=HIDDEN;} ;
1334            """)
1335
1336        treeGrammar = textwrap.dedent(
1337            r"""
1338            tree grammar TP;
1339            options {language=Python; output=AST; ASTLabelType=CommonTree; tokenVocab=T; rewrite=true;}
1340            s : ^(ID label+=a) -> a ;
1341            a : ^(INT INT) ;
1342            """)
1343
1344        found = self.execTreeParser(
1345            grammar, 'a',
1346            treeGrammar, 's',
1347            "abc 34")
1348        self.failUnlessEqual("(34 34)", found)
1349
1350
1351
1352    def testRewriteModeWithPredicatedRewrites(self):
1353        grammar = textwrap.dedent(
1354            r'''
1355            grammar T19;
1356            options {
1357              language=Python;
1358              output=AST;
1359            }
1360            a : ID INT -> ^(ID["root"] ^(ID INT)) | INT -> ^(ID["root"] INT) ;
1361            ID : 'a'..'z'+ ;
1362            INT : '0'..'9'+;
1363            WS : (' '|'\n') {$channel=HIDDEN;} ;
1364            ''')
1365
1366        treeGrammar = textwrap.dedent(
1367            r'''
1368            tree grammar TP19;
1369            options {
1370              language=Python;
1371              output=AST;
1372              ASTLabelType=CommonTree;
1373              tokenVocab=T19;
1374              rewrite=true;
1375            }
1376            s : ^(ID a) { self.buf += $s.start.toStringTree() };
1377            a : ^(ID INT) -> {True}? ^(ID["ick"] INT)
1378                          -> INT
1379              ;
1380            ''')
1381
1382        found = self.execTreeParser(
1383            grammar, 'a',
1384            treeGrammar, 's',
1385            "abc 34"
1386            )
1387
1388        self.failUnlessEqual("(root (ick 34))", found)
1389
1390
1391    def testWildcardSingleNode(self):
1392        grammar = textwrap.dedent(
1393            r'''
1394            grammar T;
1395            options {
1396                language=Python;
1397                output=AST;
1398            }
1399            a : ID INT -> ^(ID["root"] INT);
1400            ID : 'a'..'z'+ ;
1401            INT : '0'..'9'+;
1402            WS : (' '|'\n') {$channel=HIDDEN;} ;
1403            ''')
1404
1405        treeGrammar = textwrap.dedent(
1406            r'''
1407            tree grammar TP;
1408            options {
1409                language=Python;
1410                output=AST;
1411                ASTLabelType=CommonTree;
1412                tokenVocab=T;
1413            }
1414            s : ^(ID c=.) -> $c
1415            ;
1416            ''')
1417
1418        found = self.execTreeParser(
1419            grammar, 'a',
1420            treeGrammar, 's',
1421            "abc 34"
1422            )
1423
1424        self.failUnlessEqual("34", found)
1425
1426    def testWildcardUnlabeledSingleNode(self):
1427        grammar = textwrap.dedent(
1428            r'''
1429            grammar T;
1430            options {language=Python; output=AST;}
1431            a : ID INT -> ^(ID INT);
1432            ID : 'a'..'z'+ ;
1433            INT : '0'..'9'+;
1434            WS : (' '|'\n') {$channel=HIDDEN;} ;
1435            ''')
1436
1437        treeGrammar = textwrap.dedent(
1438            r'''
1439            tree grammar TP;
1440            options {language=Python; output=AST; ASTLabelType=CommonTree; tokenVocab=T;}
1441            s : ^(ID .) -> ID
1442              ;
1443            ''')
1444
1445        found = self.execTreeParser(
1446            grammar, 'a',
1447            treeGrammar, 's',
1448            "abc 34")
1449        self.assertEquals("abc", found)
1450
1451
1452    def testWildcardGrabsSubtree(self):
1453        grammar = textwrap.dedent(
1454            r'''
1455            grammar T;
1456            options {language=Python; output=AST;}
1457            a : ID x=INT y=INT z=INT -> ^(ID[\"root\"] ^($x $y $z));
1458            ID : 'a'..'z'+ ;
1459            INT : '0'..'9'+;
1460            WS : (' '|'\n') {$channel=HIDDEN;} ;
1461            ''')
1462
1463        treeGrammar = textwrap.dedent(
1464            r'''
1465            tree grammar TP;
1466            options {language=Python; output=AST; ASTLabelType=CommonTree; tokenVocab=T;}
1467            s : ^(ID c=.) -> $c
1468              ;
1469            ''')
1470
1471        found = self.execTreeParser(
1472            grammar, 'a',
1473            treeGrammar, 's',
1474            "abc 1 2 3")
1475        self.assertEquals("(1 2 3)", found)
1476
1477
1478    def testWildcardGrabsSubtree2(self):
1479        grammar = textwrap.dedent(
1480            r'''
1481            grammar T;
1482            options {language=Python; output=AST;}
1483            a : ID x=INT y=INT z=INT -> ID ^($x $y $z);
1484            ID : 'a'..'z'+ ;
1485            INT : '0'..'9'+;
1486            WS : (' '|'\n') {$channel=HIDDEN;} ;
1487            ''')
1488
1489        treeGrammar = textwrap.dedent(
1490            r'''
1491            tree grammar TP;
1492            options {language=Python; output=AST; ASTLabelType=CommonTree; tokenVocab=T;}
1493            s : ID c=. -> $c
1494              ;
1495            ''')
1496
1497        found = self.execTreeParser(
1498            grammar, 'a',
1499            treeGrammar, 's',
1500            "abc 1 2 3")
1501        self.assertEquals("(1 2 3)", found)
1502
1503
1504    def testWildcardListLabel(self):
1505        grammar = textwrap.dedent(
1506            r'''
1507            grammar T;
1508            options {language=Python; output=AST;}
1509            a : INT INT INT ;
1510            ID : 'a'..'z'+ ;
1511            INT : '0'..'9'+;
1512            WS : (' '|'\n') {$channel=HIDDEN;} ;
1513            ''')
1514
1515        treeGrammar = textwrap.dedent(
1516            r'''
1517            tree grammar TP;
1518            options {language=Python; output=AST; ASTLabelType=CommonTree; tokenVocab=T;}
1519            s : (c+=.)+ -> $c+
1520              ;
1521            ''')
1522
1523        found = self.execTreeParser(
1524            grammar, 'a',
1525            treeGrammar, 's',
1526            "1 2 3")
1527        self.assertEquals("1 2 3", found)
1528
1529
1530    def testWildcardListLabel2(self):
1531        grammar = textwrap.dedent(
1532            r'''
1533            grammar T;
1534            options {language=Python; output=AST; ASTLabelType=CommonTree;}
1535            a  : x=INT y=INT z=INT -> ^($x ^($y $z) ^($y $z));
1536            ID : 'a'..'z'+ ;
1537            INT : '0'..'9'+;
1538            WS : (' '|'\n') {$channel=HIDDEN;} ;
1539            ''')
1540
1541        treeGrammar = textwrap.dedent(
1542            r'''
1543            tree grammar TP;
1544            options {language=Python; output=AST; ASTLabelType=CommonTree; tokenVocab=T; rewrite=true;}
1545            s : ^(INT (c+=.)+) -> $c+
1546              ;
1547            ''')
1548
1549        found = self.execTreeParser(
1550            grammar, 'a',
1551            treeGrammar, 's',
1552            "1 2 3")
1553        self.assertEquals("(2 3) (2 3)", found)
1554
1555
1556    def testRuleResultAsRoot(self):
1557        grammar = textwrap.dedent(
1558            r'''
1559            grammar T;
1560            options {
1561                language=Python;
1562                output=AST;
1563            }
1564            a : ID '=' INT -> ^('=' ID INT);
1565            ID : 'a'..'z'+ ;
1566            INT : '0'..'9'+;
1567            COLON : ':' ;
1568            WS : (' '|'\n') {$channel=HIDDEN;} ;
1569            ''')
1570
1571        treeGrammar = textwrap.dedent(
1572            r'''
1573            tree grammar TP;
1574            options {
1575                language=Python;
1576                output=AST;
1577                rewrite=true;
1578                ASTLabelType=CommonTree;
1579                tokenVocab=T;
1580            }
1581            a : ^(eq e1=ID e2=.) -> ^(eq $e2 $e1) ;
1582            eq : '=' | ':' {pass} ;  // bug in set match, doesn't add to tree!! booh. force nonset.
1583            ''')
1584
1585        found = self.execTreeParser(
1586            grammar, 'a',
1587            treeGrammar, 'a',
1588            "abc = 34")
1589        self.assertEquals("(= 34 abc)", found)
1590
1591
1592if __name__ == '__main__':
1593    unittest.main()
1594