1import unittest
2import textwrap
3import antlr3
4import antlr3.tree
5import testbase
6import sys
7
8class TestAutoAST(testbase.ANTLRTest):
9    def parserClass(self, base):
10        class TParser(base):
11            def __init__(self, *args, **kwargs):
12                base.__init__(self, *args, **kwargs)
13
14                self._errors = []
15                self._output = ""
16
17
18            def capture(self, t):
19                self._output += t
20
21
22            def traceIn(self, ruleName, ruleIndex):
23                self.traces.append('>'+ruleName)
24
25
26            def traceOut(self, ruleName, ruleIndex):
27                self.traces.append('<'+ruleName)
28
29
30            def emitErrorMessage(self, msg):
31                self._errors.append(msg)
32
33
34        return TParser
35
36
37    def lexerClass(self, base):
38        class TLexer(base):
39            def __init__(self, *args, **kwargs):
40                base.__init__(self, *args, **kwargs)
41
42                self._output = ""
43
44
45            def capture(self, t):
46                self._output += t
47
48
49            def traceIn(self, ruleName, ruleIndex):
50                self.traces.append('>'+ruleName)
51
52
53            def traceOut(self, ruleName, ruleIndex):
54                self.traces.append('<'+ruleName)
55
56
57            def recover(self, input, re):
58                # no error recovery yet, just crash!
59                raise
60
61        return TLexer
62
63
64    def execParser(self, grammar, grammarEntry, input, expectErrors=False):
65        lexerCls, parserCls = self.compileInlineGrammar(grammar)
66
67        cStream = antlr3.StringStream(input)
68        lexer = lexerCls(cStream)
69        tStream = antlr3.CommonTokenStream(lexer)
70        parser = parserCls(tStream)
71        r = getattr(parser, grammarEntry)()
72
73        if not expectErrors:
74            self.assertEquals(len(parser._errors), 0, parser._errors)
75
76        result = ""
77
78        if r is not None:
79            if hasattr(r, 'result'):
80                result += r.result
81
82            if r.tree is not None:
83                result += r.tree.toStringTree()
84
85        if not expectErrors:
86            return result
87
88        else:
89            return result, parser._errors
90
91
92    def execTreeParser(self, grammar, grammarEntry, treeGrammar, treeEntry, input):
93        lexerCls, parserCls = self.compileInlineGrammar(grammar)
94        walkerCls = self.compileInlineGrammar(treeGrammar)
95
96        cStream = antlr3.StringStream(input)
97        lexer = lexerCls(cStream)
98        tStream = antlr3.CommonTokenStream(lexer)
99        parser = parserCls(tStream)
100        r = getattr(parser, grammarEntry)()
101        nodes = antlr3.tree.CommonTreeNodeStream(r.tree)
102        nodes.setTokenStream(tStream)
103        walker = walkerCls(nodes)
104        r = getattr(walker, treeEntry)()
105
106        if r is not None:
107            return r.tree.toStringTree()
108
109        return ""
110
111
112    def testTokenList(self):
113        grammar = textwrap.dedent(
114            r'''
115            grammar foo;
116            options {language=Python;output=AST;}
117            a : ID INT ;
118            ID : 'a'..'z'+ ;
119            INT : '0'..'9'+;
120            WS : (' '|'\n') {$channel=HIDDEN;};
121            ''')
122
123        found = self.execParser(grammar, "a", "abc 34")
124        self.assertEquals("abc 34", found);
125
126
127    def testTokenListInSingleAltBlock(self):
128        grammar = textwrap.dedent(
129            r'''
130            grammar foo;
131            options {language=Python;output=AST;}
132            a : (ID INT) ;
133            ID : 'a'..'z'+ ;
134            INT : '0'..'9'+;
135            WS : (' '|'\n') {$channel=HIDDEN;} ;
136            ''')
137
138        found = self.execParser(grammar,"a", "abc 34")
139        self.assertEquals("abc 34", found)
140
141
142    def testSimpleRootAtOuterLevel(self):
143        grammar = textwrap.dedent(
144            r'''
145            grammar foo;
146            options {language=Python;output=AST;}
147            a : ID^ INT ;
148            ID : 'a'..'z'+ ;
149            INT : '0'..'9'+;
150            WS : (' '|'\n') {$channel=HIDDEN;} ;
151            ''')
152
153        found = self.execParser(grammar, "a", "abc 34")
154        self.assertEquals("(abc 34)", found)
155
156
157    def testSimpleRootAtOuterLevelReverse(self):
158        grammar = textwrap.dedent(
159            r'''
160            grammar T;
161            options {language=Python;output=AST;}
162            a : INT ID^ ;
163            ID : 'a'..'z'+ ;
164            INT : '0'..'9'+;
165            WS : (' '|'\n') {$channel=HIDDEN;} ;
166            ''')
167
168        found = self.execParser(grammar, "a", "34 abc")
169        self.assertEquals("(abc 34)", found)
170
171
172    def testBang(self):
173        grammar = textwrap.dedent(
174            r'''
175            grammar T;
176            options {language=Python;output=AST;}
177            a : ID INT! ID! INT ;
178            ID : 'a'..'z'+ ;
179            INT : '0'..'9'+;
180            WS : (' '|'\n') {$channel=HIDDEN;} ;
181            ''')
182
183        found = self.execParser(grammar, "a", "abc 34 dag 4532")
184        self.assertEquals("abc 4532", found)
185
186
187    def testOptionalThenRoot(self):
188        grammar = textwrap.dedent(
189            r'''
190            grammar T;
191            options {language=Python;output=AST;}
192            a : ( ID INT )? ID^ ;
193            ID : 'a'..'z'+ ;
194            INT : '0'..'9'+;
195            WS : (' '|'\n') {$channel=HIDDEN;} ;
196            ''')
197
198        found = self.execParser(grammar, "a", "a 1 b")
199        self.assertEquals("(b a 1)", found)
200
201
202    def testLabeledStringRoot(self):
203        grammar = textwrap.dedent(
204            r'''
205            grammar T;
206            options {language=Python;output=AST;}
207            a : v='void'^ ID ';' ;
208            ID : 'a'..'z'+ ;
209            INT : '0'..'9'+;
210            WS : (' '|'\n') {$channel=HIDDEN;} ;
211            ''')
212
213        found = self.execParser(grammar, "a", "void foo;")
214        self.assertEquals("(void foo ;)", found)
215
216
217    def testWildcard(self):
218        grammar = textwrap.dedent(
219            r'''
220            grammar T;
221            options {language=Python;output=AST;}
222            a : v='void'^ . ';' ;
223            ID : 'a'..'z'+ ;
224            INT : '0'..'9'+;
225            WS : (' '|'\n') {$channel=HIDDEN;} ;
226            ''')
227
228        found = self.execParser(grammar, "a", "void foo;")
229        self.assertEquals("(void foo ;)", found)
230
231
232    def testWildcardRoot(self):
233        grammar = textwrap.dedent(
234            r'''
235            grammar T;
236            options {language=Python;output=AST;}
237            a : v='void' .^ ';' ;
238            ID : 'a'..'z'+ ;
239            INT : '0'..'9'+;
240            WS : (' '|'\n') {$channel=HIDDEN;} ;
241            ''')
242
243        found = self.execParser(grammar, "a", "void foo;")
244        self.assertEquals("(foo void ;)", found)
245
246
247    def testWildcardRootWithLabel(self):
248        grammar = textwrap.dedent(
249            r'''
250            grammar T;
251            options {language=Python;output=AST;}
252            a : v='void' x=.^ ';' ;
253            ID : 'a'..'z'+ ;
254            INT : '0'..'9'+;
255            WS : (' '|'\n') {$channel=HIDDEN;} ;
256            ''')
257
258        found = self.execParser(grammar, "a", "void foo;")
259        self.assertEquals("(foo void ;)", found)
260
261
262    def testWildcardRootWithListLabel(self):
263        grammar = textwrap.dedent(
264            r'''
265            grammar T;
266            options {language=Python;output=AST;}
267            a : v='void' x=.^ ';' ;
268            ID : 'a'..'z'+ ;
269            INT : '0'..'9'+;
270            WS : (' '|'\n') {$channel=HIDDEN;} ;
271            ''')
272
273        found = self.execParser(grammar, "a", "void foo;")
274        self.assertEquals("(foo void ;)", found)
275
276
277    def testWildcardBangWithListLabel(self):
278        grammar = textwrap.dedent(
279            r'''
280            grammar T;
281            options {language=Python;output=AST;}
282            a : v='void' x=.! ';' ;
283            ID : 'a'..'z'+ ;
284            INT : '0'..'9'+;
285            WS : (' '|'\n') {$channel=HIDDEN;} ;
286            ''')
287
288        found = self.execParser(grammar, "a", "void foo;")
289        self.assertEquals("void ;", found)
290
291
292    def testRootRoot(self):
293        grammar = textwrap.dedent(
294            r'''
295            grammar T;
296            options {language=Python;output=AST;}
297            a : ID^ INT^ ID ;
298            ID : 'a'..'z'+ ;
299            INT : '0'..'9'+;
300            WS : (' '|'\n') {$channel=HIDDEN;} ;
301            ''')
302
303        found = self.execParser(grammar, "a", "a 34 c")
304        self.assertEquals("(34 a c)", found)
305
306
307    def testRootRoot2(self):
308        grammar = textwrap.dedent(
309            r'''
310            grammar T;
311            options {language=Python;output=AST;}
312            a : ID INT^ ID^ ;
313            ID : 'a'..'z'+ ;
314            INT : '0'..'9'+;
315            WS : (' '|'\n') {$channel=HIDDEN;} ;
316            ''')
317
318        found = self.execParser(grammar, "a", "a 34 c")
319        self.assertEquals("(c (34 a))", found)
320
321
322    def testRootThenRootInLoop(self):
323        grammar = textwrap.dedent(
324            r'''
325            grammar T;
326            options {language=Python;output=AST;}
327            a : ID^ (INT '*'^ ID)+ ;
328            ID  : 'a'..'z'+ ;
329            INT : '0'..'9'+;
330            WS : (' '|'\n') {$channel=HIDDEN;} ;
331            ''')
332
333        found = self.execParser(grammar, "a", "a 34 * b 9 * c")
334        self.assertEquals("(* (* (a 34) b 9) c)", found)
335
336
337    def testNestedSubrule(self):
338        grammar = textwrap.dedent(
339            r'''
340            grammar T;
341            options {language=Python;output=AST;}
342            a : 'void' (({pass}ID|INT) ID | 'null' ) ';' ;
343            ID : 'a'..'z'+ ;
344            INT : '0'..'9'+;
345            WS : (' '|'\n') {$channel=HIDDEN;} ;
346            ''')
347
348        found = self.execParser(grammar, "a", "void a b;")
349        self.assertEquals("void a b ;", found)
350
351
352    def testInvokeRule(self):
353        grammar = textwrap.dedent(
354            r'''
355            grammar T;
356            options {language=Python;output=AST;}
357            a  : type ID ;
358            type : {pass}'int' | 'float' ;
359            ID : 'a'..'z'+ ;
360            INT : '0'..'9'+;
361            WS : (' '|'\n') {$channel=HIDDEN;} ;
362            ''')
363
364        found = self.execParser(grammar, "a", "int a")
365        self.assertEquals("int a", found)
366
367
368    def testInvokeRuleAsRoot(self):
369        grammar = textwrap.dedent(
370            r'''
371            grammar T;
372            options {language=Python;output=AST;}
373            a  : type^ ID ;
374            type : {pass}'int' | 'float' ;
375            ID : 'a'..'z'+ ;
376            INT : '0'..'9'+;
377            WS : (' '|'\n') {$channel=HIDDEN;} ;
378            ''')
379
380        found = self.execParser(grammar, "a", "int a")
381        self.assertEquals("(int a)", found)
382
383
384    def testInvokeRuleAsRootWithLabel(self):
385        grammar = textwrap.dedent(
386            r'''
387            grammar T;
388            options {language=Python;output=AST;}
389            a  : x=type^ ID ;
390            type : {pass}'int' | 'float' ;
391            ID : 'a'..'z'+ ;
392            INT : '0'..'9'+;
393            WS : (' '|'\n') {$channel=HIDDEN;} ;
394            ''')
395
396        found = self.execParser(grammar, "a", "int a")
397        self.assertEquals("(int a)", found)
398
399
400    def testInvokeRuleAsRootWithListLabel(self):
401        grammar = textwrap.dedent(
402            r'''
403            grammar T;
404            options {language=Python;output=AST;}
405            a  : x+=type^ ID ;
406            type : {pass}'int' | 'float' ;
407            ID : 'a'..'z'+ ;
408            INT : '0'..'9'+;
409            WS : (' '|'\n') {$channel=HIDDEN;} ;
410            ''')
411
412        found = self.execParser(grammar, "a", "int a")
413        self.assertEquals("(int a)", found)
414
415
416    def testRuleRootInLoop(self):
417        grammar = textwrap.dedent(
418            r'''
419            grammar T;
420            options {language=Python;output=AST;}
421            a : ID ('+'^ ID)* ;
422            ID : 'a'..'z'+ ;
423            INT : '0'..'9'+;
424            WS : (' '|'\n') {$channel=HIDDEN;} ;
425            ''')
426
427        found = self.execParser(grammar, "a", "a+b+c+d")
428        self.assertEquals("(+ (+ (+ a b) c) d)", found)
429
430
431    def testRuleInvocationRuleRootInLoop(self):
432        grammar = textwrap.dedent(
433            r'''
434            grammar T;
435            options {language=Python;output=AST;}
436            a : ID (op^ ID)* ;
437            op : {pass}'+' | '-' ;
438            ID : 'a'..'z'+ ;
439            INT : '0'..'9'+;
440            WS : (' '|'\n') {$channel=HIDDEN;} ;
441            ''')
442
443        found = self.execParser(grammar, "a", "a+b+c-d")
444        self.assertEquals("(- (+ (+ a b) c) d)", found)
445
446
447    def testTailRecursion(self):
448        grammar = textwrap.dedent(
449            r'''
450            grammar T;
451            options {language=Python;output=AST;}
452            s : a ;
453            a : atom ('exp'^ a)? ;
454            atom : INT ;
455            ID : 'a'..'z'+ ;
456            INT : '0'..'9'+;
457            WS : (' '|'\n') {$channel=HIDDEN;} ;
458            ''')
459
460        found = self.execParser(grammar, "s", "3 exp 4 exp 5")
461        self.assertEquals("(exp 3 (exp 4 5))", found)
462
463
464    def testSet(self):
465        grammar = textwrap.dedent(
466            r'''
467            grammar T;
468            options {language=Python;output=AST;}
469            a : ID|INT ;
470            ID : 'a'..'z'+ ;
471            INT : '0'..'9'+;
472            WS : (' '|'\n') {$channel=HIDDEN;} ;
473            ''')
474
475        found = self.execParser(grammar, "a", "abc")
476        self.assertEquals("abc", found)
477
478
479    def testSetRoot(self):
480        grammar = textwrap.dedent(
481        r'''
482            grammar T;
483            options {language=Python;output=AST;}
484            a : ('+' | '-')^ ID ;
485            ID : 'a'..'z'+ ;
486            INT : '0'..'9'+;
487            WS : (' '|'\n') {$channel=HIDDEN;} ;
488            ''')
489
490        found = self.execParser(grammar, "a", "+abc")
491        self.assertEquals("(+ abc)", found)
492
493
494    @testbase.broken(
495        "FAILS until antlr.g rebuilt in v3", testbase.GrammarCompileError)
496    def testSetRootWithLabel(self):
497        grammar = textwrap.dedent(
498            r'''
499            grammar T;
500            options {language=Python;output=AST;}
501            a : x=('+' | '-')^ ID ;
502            ID : 'a'..'z'+ ;
503            INT : '0'..'9'+;
504            WS : (' '|'\n') {$channel=HIDDEN;} ;
505            ''')
506
507        found = self.execParser(grammar, "a", "+abc")
508        self.assertEquals("(+ abc)", found)
509
510
511    def testSetAsRuleRootInLoop(self):
512        grammar = textwrap.dedent(
513            r'''
514            grammar T;
515            options {language=Python;output=AST;}
516            a : ID (('+'|'-')^ ID)* ;
517            ID : 'a'..'z'+ ;
518            INT : '0'..'9'+;
519            WS : (' '|'\n') {$channel=HIDDEN;} ;
520            ''')
521
522        found = self.execParser(grammar, "a", "a+b-c")
523        self.assertEquals("(- (+ a b) c)", found)
524
525
526    def testNotSet(self):
527        grammar = textwrap.dedent(
528            r'''
529            grammar T;
530            options {language=Python;output=AST;}
531            a : ~ID '+' INT ;
532            ID : 'a'..'z'+ ;
533            INT : '0'..'9'+;
534            WS : (' '|'\n') {$channel=HIDDEN;} ;
535            ''')
536
537        found = self.execParser(grammar, "a", "34+2")
538        self.assertEquals("34 + 2", found)
539
540
541    def testNotSetWithLabel(self):
542        grammar = textwrap.dedent(
543            r'''
544            grammar T;
545            options {language=Python;output=AST;}
546            a : x=~ID '+' INT ;
547            ID : 'a'..'z'+ ;
548            INT : '0'..'9'+;
549            WS : (' '|'\n') {$channel=HIDDEN;} ;
550            ''')
551
552        found = self.execParser(grammar, "a", "34+2")
553        self.assertEquals("34 + 2", found)
554
555
556    def testNotSetWithListLabel(self):
557        grammar = textwrap.dedent(
558            r'''
559            grammar T;
560            options {language=Python;output=AST;}
561            a : x=~ID '+' INT ;
562            ID : 'a'..'z'+ ;
563            INT : '0'..'9'+;
564            WS : (' '|'\n') {$channel=HIDDEN;} ;
565            ''')
566
567        found = self.execParser(grammar, "a", "34+2")
568        self.assertEquals("34 + 2", found)
569
570
571    def testNotSetRoot(self):
572        grammar = textwrap.dedent(
573            r'''
574            grammar T;
575            options {language=Python;output=AST;}
576            a : ~'+'^ INT ;
577            ID : 'a'..'z'+ ;
578            INT : '0'..'9'+;
579            WS : (' '|'\n') {$channel=HIDDEN;} ;
580            ''')
581
582        found = self.execParser(grammar, "a", "34 55")
583        self.assertEquals("(34 55)", found)
584
585
586    def testNotSetRootWithLabel(self):
587        grammar = textwrap.dedent(
588            r'''
589            grammar T;
590            options {language=Python;output=AST;}
591            a : ~'+'^ INT ;
592            ID : 'a'..'z'+ ;
593            INT : '0'..'9'+;
594            WS : (' '|'\n') {$channel=HIDDEN;} ;
595            ''')
596
597        found = self.execParser(grammar, "a", "34 55")
598        self.assertEquals("(34 55)", found)
599
600
601    def testNotSetRootWithListLabel(self):
602        grammar = textwrap.dedent(
603            r'''
604            grammar T;
605            options {language=Python;output=AST;}
606            a : ~'+'^ INT ;
607            ID : 'a'..'z'+ ;
608            INT : '0'..'9'+;
609            WS : (' '|'\n') {$channel=HIDDEN;} ;
610            ''')
611
612        found = self.execParser(grammar, "a", "34 55")
613        self.assertEquals("(34 55)", found)
614
615
616    def testNotSetRuleRootInLoop(self):
617        grammar = textwrap.dedent(
618            r'''
619            grammar T;
620            options {language=Python;output=AST;}
621            a : INT (~INT^ INT)* ;
622            blort : '+' ;
623            ID : 'a'..'z'+ ;
624            INT : '0'..'9'+;
625            WS : (' '|'\n') {$channel=HIDDEN;} ;
626            ''')
627
628        found = self.execParser(grammar, "a", "3+4+5")
629        self.assertEquals("(+ (+ 3 4) 5)", found)
630
631
632    @testbase.broken("FIXME: What happened to the semicolon?", AssertionError)
633    def testTokenLabelReuse(self):
634        # check for compilation problem due to multiple defines
635        grammar = textwrap.dedent(
636            r'''
637            grammar T;
638            options {language=Python;output=AST;}
639            a returns [result] : id=ID id=ID {$result = "2nd id="+$id.text+";";} ;
640            ID : 'a'..'z'+ ;
641            INT : '0'..'9'+;
642            WS : (' '|'\n') {$channel=HIDDEN;} ;
643            ''')
644
645        found = self.execParser(grammar, "a", "a b")
646        self.assertEquals("2nd id=b;a b", found)
647
648
649    def testTokenLabelReuse2(self):
650        # check for compilation problem due to multiple defines
651        grammar = textwrap.dedent(
652            r'''
653            grammar T;
654            options {language=Python;output=AST;}
655            a returns [result]: id=ID id=ID^ {$result = "2nd id="+$id.text+',';} ;
656            ID : 'a'..'z'+ ;
657            INT : '0'..'9'+;
658            WS : (' '|'\n') {$channel=HIDDEN;} ;
659            ''')
660
661        found = self.execParser(grammar, "a", "a b")
662        self.assertEquals("2nd id=b,(b a)", found)
663
664
665    def testTokenListLabelReuse(self):
666        # check for compilation problem due to multiple defines
667        # make sure ids has both ID tokens
668        grammar = textwrap.dedent(
669            r'''
670            grammar T;
671            options {language=Python;output=AST;}
672            a returns [result] : ids+=ID ids+=ID {$result = "id list=["+",".join([t.text for t in $ids])+'],';} ;
673            ID : 'a'..'z'+ ;
674            INT : '0'..'9'+;
675            WS : (' '|'\n') {$channel=HIDDEN;} ;
676            ''')
677
678        found = self.execParser(grammar, "a", "a b")
679        expecting = "id list=[a,b],a b"
680        self.assertEquals(expecting, found)
681
682
683    def testTokenListLabelReuse2(self):
684        # check for compilation problem due to multiple defines
685        # make sure ids has both ID tokens
686        grammar = textwrap.dedent(
687            r'''
688            grammar T;
689            options {language=Python;output=AST;}
690            a returns [result] : ids+=ID^ ids+=ID {$result = "id list=["+",".join([t.text for t in $ids])+'],';} ;
691            ID : 'a'..'z'+ ;
692            INT : '0'..'9'+;
693            WS : (' '|'\n') {$channel=HIDDEN;} ;
694            ''')
695
696        found = self.execParser(grammar, "a", "a b")
697        expecting = "id list=[a,b],(a b)"
698        self.assertEquals(expecting, found)
699
700
701    def testTokenListLabelRuleRoot(self):
702        grammar = textwrap.dedent(
703            r'''
704            grammar T;
705            options {language=Python;output=AST;}
706            a : id+=ID^ ;
707            ID : 'a'..'z'+ ;
708            INT : '0'..'9'+;
709            WS : (' '|'\n') {$channel=HIDDEN;} ;
710            ''')
711
712        found = self.execParser(grammar, "a", "a")
713        self.assertEquals("a", found)
714
715
716    def testTokenListLabelBang(self):
717        grammar = textwrap.dedent(
718            r'''
719            grammar T;
720            options {language=Python;output=AST;}
721            a : id+=ID! ;
722            ID : 'a'..'z'+ ;
723            INT : '0'..'9'+;
724            WS : (' '|'\n') {$channel=HIDDEN;} ;
725            ''')
726
727        found = self.execParser(grammar, "a", "a")
728        self.assertEquals("", found)
729
730
731    def testRuleListLabel(self):
732        grammar = textwrap.dedent(
733            r'''
734            grammar T;
735            options {language=Python;output=AST;}
736            a returns [result]: x+=b x+=b {
737            t=$x[1]
738            $result = "2nd x="+t.toStringTree()+',';
739            };
740            b : ID;
741            ID : 'a'..'z'+ ;
742            INT : '0'..'9'+;
743            WS : (' '|'\n') {$channel=HIDDEN;} ;
744            ''')
745
746        found = self.execParser(grammar, "a", "a b")
747        self.assertEquals("2nd x=b,a b", found)
748
749
750    def testRuleListLabelRuleRoot(self):
751        grammar = textwrap.dedent(
752            r'''
753            grammar T;
754            options {language=Python;output=AST;}
755            a returns [result] : ( x+=b^ )+ {
756            $result = "x="+$x[1].toStringTree()+',';
757            } ;
758            b : ID;
759            ID : 'a'..'z'+ ;
760            INT : '0'..'9'+;
761            WS : (' '|'\n') {$channel=HIDDEN;} ;
762            ''')
763
764        found = self.execParser(grammar, "a", "a b")
765        self.assertEquals("x=(b a),(b a)", found)
766
767
768    def testRuleListLabelBang(self):
769        grammar = textwrap.dedent(
770            r'''
771            grammar T;
772            options {language=Python;output=AST;}
773            a returns [result] : x+=b! x+=b {
774            $result = "1st x="+$x[0].toStringTree()+',';
775            } ;
776            b : ID;
777            ID : 'a'..'z'+ ;
778            INT : '0'..'9'+;
779            WS : (' '|'\n') {$channel=HIDDEN;} ;
780            ''')
781
782        found = self.execParser(grammar, "a", "a b")
783        self.assertEquals("1st x=a,b", found)
784
785
786    def testComplicatedMelange(self):
787        # check for compilation problem
788        grammar = textwrap.dedent(
789            r'''
790            grammar T;
791            options {language=Python;output=AST;}
792            a : A b=B b=B c+=C c+=C D {s = $D.text} ;
793            A : 'a' ;
794            B : 'b' ;
795            C : 'c' ;
796            D : 'd' ;
797            WS : (' '|'\n') {$channel=HIDDEN;} ;
798            ''')
799
800        found = self.execParser(grammar, "a", "a b b c c d")
801        self.assertEquals("a b b c c d", found)
802
803
804    def testReturnValueWithAST(self):
805        grammar = textwrap.dedent(
806            r'''
807            grammar foo;
808            options {language=Python;output=AST;}
809            a returns [result] : ID b { $result = str($b.i) + '\n';} ;
810            b returns [i] : INT {$i=int($INT.text);} ;
811            ID : 'a'..'z'+ ;
812            INT : '0'..'9'+;
813            WS : (' '|'\n') {$channel=HIDDEN;} ;
814            ''')
815
816        found = self.execParser(grammar, "a", "abc 34")
817        self.assertEquals("34\nabc 34", found)
818
819
820    def testSetLoop(self):
821        grammar = textwrap.dedent(
822            r'''
823            grammar T;
824            options { language=Python;output=AST; }
825            r : (INT|ID)+ ;
826            ID : 'a'..'z' + ;
827            INT : '0'..'9' +;
828            WS: (' ' | '\n' | '\\t')+ {$channel = HIDDEN;};
829            ''')
830
831        found = self.execParser(grammar, "r", "abc 34 d")
832        self.assertEquals("abc 34 d", found)
833
834
835    def testExtraTokenInSimpleDecl(self):
836        grammar = textwrap.dedent(
837            r'''
838            grammar foo;
839            options {language=Python;output=AST;}
840            decl : type^ ID '='! INT ';'! ;
841            type : 'int' | 'float' ;
842            ID : 'a'..'z'+ ;
843            INT : '0'..'9'+;
844            WS : (' '|'\n') {$channel=HIDDEN;} ;
845            ''')
846
847        found, errors = self.execParser(grammar, "decl", "int 34 x=1;",
848                                        expectErrors=True)
849        self.assertEquals(["line 1:4 extraneous input u'34' expecting ID"],
850                          errors)
851        self.assertEquals("(int x 1)", found) # tree gets correct x and 1 tokens
852
853
854    def testMissingIDInSimpleDecl(self):
855        grammar = textwrap.dedent(
856            r'''
857            grammar foo;
858            options {language=Python;output=AST;}
859            tokens {EXPR;}
860            decl : type^ ID '='! INT ';'! ;
861            type : 'int' | 'float' ;
862            ID : 'a'..'z'+ ;
863            INT : '0'..'9'+;
864            WS : (' '|'\n') {$channel=HIDDEN;} ;
865            ''')
866
867        found, errors = self.execParser(grammar, "decl", "int =1;",
868                                        expectErrors=True)
869        self.assertEquals(["line 1:4 missing ID at u'='"], errors)
870        self.assertEquals("(int <missing ID> 1)", found) # tree gets invented ID token
871
872
873    def testMissingSetInSimpleDecl(self):
874        grammar = textwrap.dedent(
875            r'''
876            grammar foo;
877            options {language=Python;output=AST;}
878            tokens {EXPR;}
879            decl : type^ ID '='! INT ';'! ;
880            type : 'int' | 'float' ;
881            ID : 'a'..'z'+ ;
882            INT : '0'..'9'+;
883            WS : (' '|'\n') {$channel=HIDDEN;} ;
884            ''')
885
886        found, errors = self.execParser(grammar, "decl", "x=1;",
887                                        expectErrors=True)
888        self.assertEquals(["line 1:0 mismatched input u'x' expecting set None"], errors)
889        self.assertEquals("(<error: x> x 1)", found) # tree gets invented ID token
890
891
892    def testMissingTokenGivesErrorNode(self):
893        grammar = textwrap.dedent(
894            r'''
895            grammar foo;
896            options {language=Python;output=AST;}
897            a : ID INT ; // follow is EOF
898            ID : 'a'..'z'+ ;
899            INT : '0'..'9'+;
900            WS : (' '|'\n') {$channel=HIDDEN;} ;
901            ''')
902
903        found, errors = self.execParser(grammar, "a", "abc", expectErrors=True)
904        self.assertEquals(["line 1:3 missing INT at '<EOF>'"], errors)
905        self.assertEquals("abc <missing INT>", found)
906
907
908    def testMissingTokenGivesErrorNodeInInvokedRule(self):
909        grammar = textwrap.dedent(
910            r'''
911            grammar foo;
912            options {language=Python;output=AST;}
913            a : b ;
914            b : ID INT ; // follow should see EOF
915            ID : 'a'..'z'+ ;
916            INT : '0'..'9'+;
917            WS : (' '|'\n') {$channel=HIDDEN;} ;
918            ''')
919
920        found, errors = self.execParser(grammar, "a", "abc", expectErrors=True)
921        self.assertEquals(["line 1:3 mismatched input '<EOF>' expecting INT"], errors)
922        self.assertEquals("<mismatched token: <EOF>, resync=abc>", found)
923
924
925    def testExtraTokenGivesErrorNode(self):
926        grammar = textwrap.dedent(
927            r'''
928            grammar foo;
929            options {language=Python;output=AST;}
930            a : b c ;
931            b : ID ;
932            c : INT ;
933            ID : 'a'..'z'+ ;
934            INT : '0'..'9'+;
935            WS : (' '|'\n') {$channel=HIDDEN;} ;
936            ''')
937
938        found, errors = self.execParser(grammar, "a", "abc ick 34",
939                                        expectErrors=True)
940        self.assertEquals(["line 1:4 extraneous input u'ick' expecting INT"],
941                          errors)
942        self.assertEquals("abc 34", found)
943
944
945    def testMissingFirstTokenGivesErrorNode(self):
946        grammar = textwrap.dedent(
947            r'''
948            grammar foo;
949            options {language=Python;output=AST;}
950            a : ID INT ;
951            ID : 'a'..'z'+ ;
952            INT : '0'..'9'+;
953            WS : (' '|'\n') {$channel=HIDDEN;} ;
954            ''')
955
956        found, errors = self.execParser(grammar, "a", "34", expectErrors=True)
957        self.assertEquals(["line 1:0 missing ID at u'34'"], errors)
958        self.assertEquals("<missing ID> 34", found)
959
960
961    def testMissingFirstTokenGivesErrorNode2(self):
962        grammar = textwrap.dedent(
963            r'''
964            grammar foo;
965            options {language=Python;output=AST;}
966            a : b c ;
967            b : ID ;
968            c : INT ;
969            ID : 'a'..'z'+ ;
970            INT : '0'..'9'+;
971            WS : (' '|'\n') {$channel=HIDDEN;} ;
972            ''')
973
974        found, errors = self.execParser(grammar, "a", "34", expectErrors=True)
975
976        # finds an error at the first token, 34, and re-syncs.
977        # re-synchronizing does not consume a token because 34 follows
978        # ref to rule b (start of c). It then matches 34 in c.
979        self.assertEquals(["line 1:0 missing ID at u'34'"], errors)
980        self.assertEquals("<missing ID> 34", found)
981
982
983    def testNoViableAltGivesErrorNode(self):
984        grammar = textwrap.dedent(
985            r'''
986            grammar foo;
987            options {language=Python;output=AST;}
988            a : b | c ;
989            b : ID ;
990            c : INT ;
991            ID : 'a'..'z'+ ;
992            S : '*' ;
993            INT : '0'..'9'+;
994            WS : (' '|'\n') {$channel=HIDDEN;} ;
995            ''')
996
997        found, errors = self.execParser(grammar, "a", "*", expectErrors=True)
998        self.assertEquals(["line 1:0 no viable alternative at input u'*'"],
999                          errors)
1000        self.assertEquals("<unexpected: [@0,0:0=u'*',<6>,1:0], resync=*>",
1001                          found)
1002
1003
1004if __name__ == '__main__':
1005    unittest.main()
1006