1import parser
2import unittest
3import sys
4import struct
5from test import test_support as support
6from test.script_helper import assert_python_failure
7
8#
9#  First, we test that we can generate trees from valid source fragments,
10#  and that these valid trees are indeed allowed by the tree-loading side
11#  of the parser module.
12#
13
14class RoundtripLegalSyntaxTestCase(unittest.TestCase):
15
16    def roundtrip(self, f, s):
17        st1 = f(s)
18        t = st1.totuple()
19        try:
20            st2 = parser.sequence2st(t)
21        except parser.ParserError, why:
22            self.fail("could not roundtrip %r: %s" % (s, why))
23
24        self.assertEqual(t, st2.totuple(),
25                         "could not re-generate syntax tree")
26
27    def check_expr(self, s):
28        self.roundtrip(parser.expr, s)
29
30    def test_flags_passed(self):
31        # The unicode literals flags has to be passed from the paser to AST
32        # generation.
33        suite = parser.suite("from __future__ import unicode_literals; x = ''")
34        code = suite.compile()
35        scope = {}
36        exec code in scope
37        self.assertIsInstance(scope["x"], unicode)
38
39    def check_suite(self, s):
40        self.roundtrip(parser.suite, s)
41
42    def test_yield_statement(self):
43        self.check_suite("def f(): yield 1")
44        self.check_suite("def f(): yield")
45        self.check_suite("def f(): x += yield")
46        self.check_suite("def f(): x = yield 1")
47        self.check_suite("def f(): x = y = yield 1")
48        self.check_suite("def f(): x = yield")
49        self.check_suite("def f(): x = y = yield")
50        self.check_suite("def f(): 1 + (yield)*2")
51        self.check_suite("def f(): (yield 1)*2")
52        self.check_suite("def f(): return; yield 1")
53        self.check_suite("def f(): yield 1; return")
54        self.check_suite("def f():\n"
55                         "    for x in range(30):\n"
56                         "        yield x\n")
57        self.check_suite("def f():\n"
58                         "    if (yield):\n"
59                         "        yield x\n")
60
61    def test_expressions(self):
62        self.check_expr("foo(1)")
63        self.check_expr("{1:1}")
64        self.check_expr("{1:1, 2:2, 3:3}")
65        self.check_expr("{1:1, 2:2, 3:3,}")
66        self.check_expr("{1}")
67        self.check_expr("{1, 2, 3}")
68        self.check_expr("{1, 2, 3,}")
69        self.check_expr("[]")
70        self.check_expr("[1]")
71        self.check_expr("[1, 2, 3]")
72        self.check_expr("[1, 2, 3,]")
73        self.check_expr("()")
74        self.check_expr("(1,)")
75        self.check_expr("(1, 2, 3)")
76        self.check_expr("(1, 2, 3,)")
77        self.check_expr("[x**3 for x in range(20)]")
78        self.check_expr("[x**3 for x in range(20) if x % 3]")
79        self.check_expr("[x**3 for x in range(20) if x % 2 if x % 3]")
80        self.check_expr("[x+y for x in range(30) for y in range(20) if x % 2 if y % 3]")
81        #self.check_expr("[x for x in lambda: True, lambda: False if x()]")
82        self.check_expr("list(x**3 for x in range(20))")
83        self.check_expr("list(x**3 for x in range(20) if x % 3)")
84        self.check_expr("list(x**3 for x in range(20) if x % 2 if x % 3)")
85        self.check_expr("list(x+y for x in range(30) for y in range(20) if x % 2 if y % 3)")
86        self.check_expr("{x**3 for x in range(30)}")
87        self.check_expr("{x**3 for x in range(30) if x % 3}")
88        self.check_expr("{x**3 for x in range(30) if x % 2 if x % 3}")
89        self.check_expr("{x+y for x in range(30) for y in range(20) if x % 2 if y % 3}")
90        self.check_expr("{x**3: y**2 for x, y in zip(range(30), range(30))}")
91        self.check_expr("{x**3: y**2 for x, y in zip(range(30), range(30)) if x % 3}")
92        self.check_expr("{x**3: y**2 for x, y in zip(range(30), range(30)) if x % 3 if y % 3}")
93        self.check_expr("{x:y for x in range(30) for y in range(20) if x % 2 if y % 3}")
94        self.check_expr("foo(*args)")
95        self.check_expr("foo(*args, **kw)")
96        self.check_expr("foo(**kw)")
97        self.check_expr("foo(key=value)")
98        self.check_expr("foo(key=value, *args)")
99        self.check_expr("foo(key=value, *args, **kw)")
100        self.check_expr("foo(key=value, **kw)")
101        self.check_expr("foo(a, b, c, *args)")
102        self.check_expr("foo(a, b, c, *args, **kw)")
103        self.check_expr("foo(a, b, c, **kw)")
104        self.check_expr("foo(a, *args, keyword=23)")
105        self.check_expr("foo + bar")
106        self.check_expr("foo - bar")
107        self.check_expr("foo * bar")
108        self.check_expr("foo / bar")
109        self.check_expr("foo // bar")
110        self.check_expr("lambda: 0")
111        self.check_expr("lambda x: 0")
112        self.check_expr("lambda *y: 0")
113        self.check_expr("lambda *y, **z: 0")
114        self.check_expr("lambda **z: 0")
115        self.check_expr("lambda x, y: 0")
116        self.check_expr("lambda foo=bar: 0")
117        self.check_expr("lambda foo=bar, spaz=nifty+spit: 0")
118        self.check_expr("lambda foo=bar, **z: 0")
119        self.check_expr("lambda foo=bar, blaz=blat+2, **z: 0")
120        self.check_expr("lambda foo=bar, blaz=blat+2, *y, **z: 0")
121        self.check_expr("lambda x, *y, **z: 0")
122        self.check_expr("lambda x: 5 if x else 2")
123        self.check_expr("(x for x in range(10))")
124        self.check_expr("foo(x for x in range(10))")
125
126    def test_print(self):
127        self.check_suite("print")
128        self.check_suite("print 1")
129        self.check_suite("print 1,")
130        self.check_suite("print >>fp")
131        self.check_suite("print >>fp, 1")
132        self.check_suite("print >>fp, 1,")
133
134    def test_simple_expression(self):
135        # expr_stmt
136        self.check_suite("a")
137
138    def test_simple_assignments(self):
139        self.check_suite("a = b")
140        self.check_suite("a = b = c = d = e")
141
142    def test_simple_augmented_assignments(self):
143        self.check_suite("a += b")
144        self.check_suite("a -= b")
145        self.check_suite("a *= b")
146        self.check_suite("a /= b")
147        self.check_suite("a //= b")
148        self.check_suite("a %= b")
149        self.check_suite("a &= b")
150        self.check_suite("a |= b")
151        self.check_suite("a ^= b")
152        self.check_suite("a <<= b")
153        self.check_suite("a >>= b")
154        self.check_suite("a **= b")
155
156    def test_function_defs(self):
157        self.check_suite("def f(): pass")
158        self.check_suite("def f(*args): pass")
159        self.check_suite("def f(*args, **kw): pass")
160        self.check_suite("def f(**kw): pass")
161        self.check_suite("def f(foo=bar): pass")
162        self.check_suite("def f(foo=bar, *args): pass")
163        self.check_suite("def f(foo=bar, *args, **kw): pass")
164        self.check_suite("def f(foo=bar, **kw): pass")
165
166        self.check_suite("def f(a, b): pass")
167        self.check_suite("def f(a, b, *args): pass")
168        self.check_suite("def f(a, b, *args, **kw): pass")
169        self.check_suite("def f(a, b, **kw): pass")
170        self.check_suite("def f(a, b, foo=bar): pass")
171        self.check_suite("def f(a, b, foo=bar, *args): pass")
172        self.check_suite("def f(a, b, foo=bar, *args, **kw): pass")
173        self.check_suite("def f(a, b, foo=bar, **kw): pass")
174
175        self.check_suite("@staticmethod\n"
176                         "def f(): pass")
177        self.check_suite("@staticmethod\n"
178                         "@funcattrs(x, y)\n"
179                         "def f(): pass")
180        self.check_suite("@funcattrs()\n"
181                         "def f(): pass")
182
183    def test_class_defs(self):
184        self.check_suite("class foo():pass")
185        self.check_suite("@class_decorator\n"
186                         "class foo():pass")
187        self.check_suite("@class_decorator(arg)\n"
188                         "class foo():pass")
189        self.check_suite("@decorator1\n"
190                         "@decorator2\n"
191                         "class foo():pass")
192
193
194    def test_import_from_statement(self):
195        self.check_suite("from sys.path import *")
196        self.check_suite("from sys.path import dirname")
197        self.check_suite("from sys.path import (dirname)")
198        self.check_suite("from sys.path import (dirname,)")
199        self.check_suite("from sys.path import dirname as my_dirname")
200        self.check_suite("from sys.path import (dirname as my_dirname)")
201        self.check_suite("from sys.path import (dirname as my_dirname,)")
202        self.check_suite("from sys.path import dirname, basename")
203        self.check_suite("from sys.path import (dirname, basename)")
204        self.check_suite("from sys.path import (dirname, basename,)")
205        self.check_suite(
206            "from sys.path import dirname as my_dirname, basename")
207        self.check_suite(
208            "from sys.path import (dirname as my_dirname, basename)")
209        self.check_suite(
210            "from sys.path import (dirname as my_dirname, basename,)")
211        self.check_suite(
212            "from sys.path import dirname, basename as my_basename")
213        self.check_suite(
214            "from sys.path import (dirname, basename as my_basename)")
215        self.check_suite(
216            "from sys.path import (dirname, basename as my_basename,)")
217        self.check_suite("from .bogus import x")
218
219    def test_basic_import_statement(self):
220        self.check_suite("import sys")
221        self.check_suite("import sys as system")
222        self.check_suite("import sys, math")
223        self.check_suite("import sys as system, math")
224        self.check_suite("import sys, math as my_math")
225
226    def test_relative_imports(self):
227        self.check_suite("from . import name")
228        self.check_suite("from .. import name")
229        self.check_suite("from .pkg import name")
230        self.check_suite("from ..pkg import name")
231
232    def test_pep263(self):
233        self.check_suite("# -*- coding: iso-8859-1 -*-\n"
234                         "pass\n")
235
236    def test_assert(self):
237        self.check_suite("assert alo < ahi and blo < bhi\n")
238
239    def test_with(self):
240        self.check_suite("with open('x'): pass\n")
241        self.check_suite("with open('x') as f: pass\n")
242        self.check_suite("with open('x') as f, open('y') as g: pass\n")
243
244    def test_try_stmt(self):
245        self.check_suite("try: pass\nexcept: pass\n")
246        self.check_suite("try: pass\nfinally: pass\n")
247        self.check_suite("try: pass\nexcept A: pass\nfinally: pass\n")
248        self.check_suite("try: pass\nexcept A: pass\nexcept: pass\n"
249                         "finally: pass\n")
250        self.check_suite("try: pass\nexcept: pass\nelse: pass\n")
251        self.check_suite("try: pass\nexcept: pass\nelse: pass\n"
252                         "finally: pass\n")
253
254    def test_except_clause(self):
255        self.check_suite("try: pass\nexcept: pass\n")
256        self.check_suite("try: pass\nexcept A: pass\n")
257        self.check_suite("try: pass\nexcept A, e: pass\n")
258        self.check_suite("try: pass\nexcept A as e: pass\n")
259
260    def test_position(self):
261        # An absolutely minimal test of position information.  Better
262        # tests would be a big project.
263        code = "def f(x):\n    return x + 1"
264        st1 = parser.suite(code)
265        st2 = st1.totuple(line_info=1, col_info=1)
266
267        def walk(tree):
268            node_type = tree[0]
269            next = tree[1]
270            if isinstance(next, tuple):
271                for elt in tree[1:]:
272                    for x in walk(elt):
273                        yield x
274            else:
275                yield tree
276
277        terminals = list(walk(st2))
278        self.assertEqual([
279            (1, 'def', 1, 0),
280            (1, 'f', 1, 4),
281            (7, '(', 1, 5),
282            (1, 'x', 1, 6),
283            (8, ')', 1, 7),
284            (11, ':', 1, 8),
285            (4, '', 1, 9),
286            (5, '', 2, -1),
287            (1, 'return', 2, 4),
288            (1, 'x', 2, 11),
289            (14, '+', 2, 13),
290            (2, '1', 2, 15),
291            (4, '', 2, 16),
292            (6, '', 2, -1),
293            (4, '', 2, -1),
294            (0, '', 2, -1)],
295                         terminals)
296
297
298#
299#  Second, we take *invalid* trees and make sure we get ParserError
300#  rejections for them.
301#
302
303class IllegalSyntaxTestCase(unittest.TestCase):
304
305    def check_bad_tree(self, tree, label):
306        try:
307            parser.sequence2st(tree)
308        except parser.ParserError:
309            pass
310        else:
311            self.fail("did not detect invalid tree for %r" % label)
312
313    def test_junk(self):
314        # not even remotely valid:
315        self.check_bad_tree((1, 2, 3), "<junk>")
316
317    def test_illegal_yield_1(self):
318        # Illegal yield statement: def f(): return 1; yield 1
319        tree = \
320        (257,
321         (264,
322          (285,
323           (259,
324            (1, 'def'),
325            (1, 'f'),
326            (260, (7, '('), (8, ')')),
327            (11, ':'),
328            (291,
329             (4, ''),
330             (5, ''),
331             (264,
332              (265,
333               (266,
334                (272,
335                 (275,
336                  (1, 'return'),
337                  (313,
338                   (292,
339                    (293,
340                     (294,
341                      (295,
342                       (297,
343                        (298,
344                         (299,
345                          (300,
346                           (301,
347                            (302, (303, (304, (305, (2, '1')))))))))))))))))),
348               (264,
349                (265,
350                 (266,
351                  (272,
352                   (276,
353                    (1, 'yield'),
354                    (313,
355                     (292,
356                      (293,
357                       (294,
358                        (295,
359                         (297,
360                          (298,
361                           (299,
362                            (300,
363                             (301,
364                              (302,
365                               (303, (304, (305, (2, '1')))))))))))))))))),
366                 (4, ''))),
367               (6, ''))))),
368           (4, ''),
369           (0, ''))))
370        self.check_bad_tree(tree, "def f():\n  return 1\n  yield 1")
371
372    def test_illegal_yield_2(self):
373        # Illegal return in generator: def f(): return 1; yield 1
374        tree = \
375        (257,
376         (264,
377          (265,
378           (266,
379            (278,
380             (1, 'from'),
381             (281, (1, '__future__')),
382             (1, 'import'),
383             (279, (1, 'generators')))),
384           (4, ''))),
385         (264,
386          (285,
387           (259,
388            (1, 'def'),
389            (1, 'f'),
390            (260, (7, '('), (8, ')')),
391            (11, ':'),
392            (291,
393             (4, ''),
394             (5, ''),
395             (264,
396              (265,
397               (266,
398                (272,
399                 (275,
400                  (1, 'return'),
401                  (313,
402                   (292,
403                    (293,
404                     (294,
405                      (295,
406                       (297,
407                        (298,
408                         (299,
409                          (300,
410                           (301,
411                            (302, (303, (304, (305, (2, '1')))))))))))))))))),
412               (264,
413                (265,
414                 (266,
415                  (272,
416                   (276,
417                    (1, 'yield'),
418                    (313,
419                     (292,
420                      (293,
421                       (294,
422                        (295,
423                         (297,
424                          (298,
425                           (299,
426                            (300,
427                             (301,
428                              (302,
429                               (303, (304, (305, (2, '1')))))))))))))))))),
430                 (4, ''))),
431               (6, ''))))),
432           (4, ''),
433           (0, ''))))
434        self.check_bad_tree(tree, "def f():\n  return 1\n  yield 1")
435
436    def test_print_chevron_comma(self):
437        # Illegal input: print >>fp,
438        tree = \
439        (257,
440         (264,
441          (265,
442           (266,
443            (268,
444             (1, 'print'),
445             (35, '>>'),
446             (290,
447              (291,
448               (292,
449                (293,
450                 (295,
451                  (296,
452                   (297,
453                    (298, (299, (300, (301, (302, (303, (1, 'fp')))))))))))))),
454             (12, ','))),
455           (4, ''))),
456         (0, ''))
457        self.check_bad_tree(tree, "print >>fp,")
458
459    def test_a_comma_comma_c(self):
460        # Illegal input: a,,c
461        tree = \
462        (258,
463         (311,
464          (290,
465           (291,
466            (292,
467             (293,
468              (295,
469               (296,
470                (297,
471                 (298, (299, (300, (301, (302, (303, (1, 'a')))))))))))))),
472          (12, ','),
473          (12, ','),
474          (290,
475           (291,
476            (292,
477             (293,
478              (295,
479               (296,
480                (297,
481                 (298, (299, (300, (301, (302, (303, (1, 'c'))))))))))))))),
482         (4, ''),
483         (0, ''))
484        self.check_bad_tree(tree, "a,,c")
485
486    def test_illegal_operator(self):
487        # Illegal input: a $= b
488        tree = \
489        (257,
490         (264,
491          (265,
492           (266,
493            (267,
494             (312,
495              (291,
496               (292,
497                (293,
498                 (294,
499                  (296,
500                   (297,
501                    (298,
502                     (299,
503                      (300, (301, (302, (303, (304, (1, 'a'))))))))))))))),
504             (268, (37, '$=')),
505             (312,
506              (291,
507               (292,
508                (293,
509                 (294,
510                  (296,
511                   (297,
512                    (298,
513                     (299,
514                      (300, (301, (302, (303, (304, (1, 'b'))))))))))))))))),
515           (4, ''))),
516         (0, ''))
517        self.check_bad_tree(tree, "a $= b")
518
519    def test_malformed_global(self):
520        #doesn't have global keyword in ast
521        tree = (257,
522                (264,
523                 (265,
524                  (266,
525                   (282, (1, 'foo'))), (4, ''))),
526                (4, ''),
527                (0, ''))
528        self.check_bad_tree(tree, "malformed global ast")
529
530    def test_missing_import_source(self):
531        # from import a
532        tree = \
533            (257,
534             (267,
535              (268,
536               (269,
537                (281,
538                 (283, (1, 'from'), (1, 'import'),
539                  (286, (284, (1, 'fred')))))),
540               (4, ''))),
541             (4, ''), (0, ''))
542        self.check_bad_tree(tree, "from import a")
543
544
545class CompileTestCase(unittest.TestCase):
546
547    # These tests are very minimal. :-(
548
549    def test_compile_expr(self):
550        st = parser.expr('2 + 3')
551        code = parser.compilest(st)
552        self.assertEqual(eval(code), 5)
553
554    def test_compile_suite(self):
555        st = parser.suite('x = 2; y = x + 3')
556        code = parser.compilest(st)
557        globs = {}
558        exec code in globs
559        self.assertEqual(globs['y'], 5)
560
561    def test_compile_error(self):
562        st = parser.suite('1 = 3 + 4')
563        self.assertRaises(SyntaxError, parser.compilest, st)
564
565    def test_compile_badunicode(self):
566        st = parser.suite('a = u"\U12345678"')
567        self.assertRaises(SyntaxError, parser.compilest, st)
568        st = parser.suite('a = u"\u1"')
569        self.assertRaises(SyntaxError, parser.compilest, st)
570
571    def test_issue_9011(self):
572        # Issue 9011: compilation of an unary minus expression changed
573        # the meaning of the ST, so that a second compilation produced
574        # incorrect results.
575        st = parser.expr('-3')
576        code1 = parser.compilest(st)
577        self.assertEqual(eval(code1), -3)
578        code2 = parser.compilest(st)
579        self.assertEqual(eval(code2), -3)
580
581
582class ParserStackLimitTestCase(unittest.TestCase):
583    """try to push the parser to/over its limits.
584    see http://bugs.python.org/issue1881 for a discussion
585    """
586    def _nested_expression(self, level):
587        return "["*level+"]"*level
588
589    def test_deeply_nested_list(self):
590        e = self._nested_expression(99)
591        st = parser.expr(e)
592        st.compile()
593
594    def test_trigger_memory_error(self):
595        e = self._nested_expression(100)
596        rc, out, err = assert_python_failure('-c', e)
597        # parsing the expression will result in an error message
598        # followed by a MemoryError (see #11963)
599        self.assertIn(b's_push: parser stack overflow', err)
600        self.assertIn(b'MemoryError', err)
601
602class STObjectTestCase(unittest.TestCase):
603    """Test operations on ST objects themselves"""
604
605    check_sizeof = support.check_sizeof
606
607    @support.cpython_only
608    def test_sizeof(self):
609        def XXXROUNDUP(n):
610            if n <= 1:
611                return n
612            if n <= 128:
613                return (n + 3) & ~3
614            return 1 << (n - 1).bit_length()
615
616        basesize = support.calcobjsize('Pii')
617        nodesize = struct.calcsize('hP3iP0h')
618        def sizeofchildren(node):
619            if node is None:
620                return 0
621            res = 0
622            hasstr = len(node) > 1 and isinstance(node[-1], str)
623            if hasstr:
624                res += len(node[-1]) + 1
625            children = node[1:-1] if hasstr else node[1:]
626            if children:
627                res += XXXROUNDUP(len(children)) * nodesize
628                for child in children:
629                    res += sizeofchildren(child)
630            return res
631
632        def check_st_sizeof(st):
633            self.check_sizeof(st, basesize + nodesize +
634                                  sizeofchildren(st.totuple()))
635
636        check_st_sizeof(parser.expr('2 + 3'))
637        check_st_sizeof(parser.expr('2 + 3 + 4'))
638        check_st_sizeof(parser.suite('x = 2 + 3'))
639        check_st_sizeof(parser.suite(''))
640        check_st_sizeof(parser.suite('# -*- coding: utf-8 -*-'))
641        check_st_sizeof(parser.expr('[' + '2,' * 1000 + ']'))
642
643
644    # XXX tests for pickling and unpickling of ST objects should go here
645
646def test_main():
647    support.run_unittest(
648        RoundtripLegalSyntaxTestCase,
649        IllegalSyntaxTestCase,
650        CompileTestCase,
651        ParserStackLimitTestCase,
652        STObjectTestCase,
653    )
654
655
656if __name__ == "__main__":
657    test_main()
658