1"""Test suite for 2to3's parser and grammar files.
2
3This is the place to add tests for changes to 2to3's grammar, such as those
4merging the grammars for Python 2 and 3. In addition to specific tests for
5parts of the grammar we've changed, we also make sure we can parse the
6test_grammar.py files from both Python 2 and Python 3.
7"""
8
9# Testing imports
10from . import support
11from .support import driver, driver_no_print_statement
12
13# Python imports
14import difflib
15import importlib
16import operator
17import os
18import pickle
19import shutil
20import subprocess
21import sys
22import tempfile
23import unittest
24
25# Local imports
26from lib2to3.pgen2 import driver as pgen2_driver
27from lib2to3.pgen2 import tokenize
28from ..pgen2.parse import ParseError
29from lib2to3.pygram import python_symbols as syms
30
31
32class TestDriver(support.TestCase):
33
34    def test_formfeed(self):
35        s = """print 1\n\x0Cprint 2\n"""
36        t = driver.parse_string(s)
37        self.assertEqual(t.children[0].children[0].type, syms.print_stmt)
38        self.assertEqual(t.children[1].children[0].type, syms.print_stmt)
39
40
41class TestPgen2Caching(support.TestCase):
42    def test_load_grammar_from_txt_file(self):
43        pgen2_driver.load_grammar(support.grammar_path, save=False, force=True)
44
45    def test_load_grammar_from_pickle(self):
46        # Make a copy of the grammar file in a temp directory we are
47        # guaranteed to be able to write to.
48        tmpdir = tempfile.mkdtemp()
49        try:
50            grammar_copy = os.path.join(
51                    tmpdir, os.path.basename(support.grammar_path))
52            shutil.copy(support.grammar_path, grammar_copy)
53            pickle_name = pgen2_driver._generate_pickle_name(grammar_copy)
54
55            pgen2_driver.load_grammar(grammar_copy, save=True, force=True)
56            self.assertTrue(os.path.exists(pickle_name))
57
58            os.unlink(grammar_copy)  # Only the pickle remains...
59            pgen2_driver.load_grammar(grammar_copy, save=False, force=False)
60        finally:
61            shutil.rmtree(tmpdir)
62
63    @unittest.skipIf(sys.executable is None, 'sys.executable required')
64    def test_load_grammar_from_subprocess(self):
65        tmpdir = tempfile.mkdtemp()
66        tmpsubdir = os.path.join(tmpdir, 'subdir')
67        try:
68            os.mkdir(tmpsubdir)
69            grammar_base = os.path.basename(support.grammar_path)
70            grammar_copy = os.path.join(tmpdir, grammar_base)
71            grammar_sub_copy = os.path.join(tmpsubdir, grammar_base)
72            shutil.copy(support.grammar_path, grammar_copy)
73            shutil.copy(support.grammar_path, grammar_sub_copy)
74            pickle_name = pgen2_driver._generate_pickle_name(grammar_copy)
75            pickle_sub_name = pgen2_driver._generate_pickle_name(
76                     grammar_sub_copy)
77            self.assertNotEqual(pickle_name, pickle_sub_name)
78
79            # Generate a pickle file from this process.
80            pgen2_driver.load_grammar(grammar_copy, save=True, force=True)
81            self.assertTrue(os.path.exists(pickle_name))
82
83            # Generate a new pickle file in a subprocess with a most likely
84            # different hash randomization seed.
85            sub_env = dict(os.environ)
86            sub_env['PYTHONHASHSEED'] = 'random'
87            subprocess.check_call(
88                    [sys.executable, '-c', """
89from lib2to3.pgen2 import driver as pgen2_driver
90pgen2_driver.load_grammar(%r, save=True, force=True)
91                    """ % (grammar_sub_copy,)],
92                    env=sub_env)
93            self.assertTrue(os.path.exists(pickle_sub_name))
94
95            with open(pickle_name, 'rb') as pickle_f_1, \
96                    open(pickle_sub_name, 'rb') as pickle_f_2:
97                self.assertEqual(
98                    pickle_f_1.read(), pickle_f_2.read(),
99                    msg='Grammar caches generated using different hash seeds'
100                    ' were not identical.')
101        finally:
102            shutil.rmtree(tmpdir)
103
104    def test_load_packaged_grammar(self):
105        modname = __name__ + '.load_test'
106        class MyLoader:
107            def get_data(self, where):
108                return pickle.dumps({'elephant': 19})
109        class MyModule:
110            __file__ = 'parsertestmodule'
111            __spec__ = importlib.util.spec_from_loader(modname, MyLoader())
112        sys.modules[modname] = MyModule()
113        self.addCleanup(operator.delitem, sys.modules, modname)
114        g = pgen2_driver.load_packaged_grammar(modname, 'Grammar.txt')
115        self.assertEqual(g.elephant, 19)
116
117
118class GrammarTest(support.TestCase):
119    def validate(self, code):
120        support.parse_string(code)
121
122    def invalid_syntax(self, code):
123        try:
124            self.validate(code)
125        except ParseError:
126            pass
127        else:
128            raise AssertionError("Syntax shouldn't have been valid")
129
130
131class TestMatrixMultiplication(GrammarTest):
132    def test_matrix_multiplication_operator(self):
133        self.validate("a @ b")
134        self.validate("a @= b")
135
136
137class TestYieldFrom(GrammarTest):
138    def test_yield_from(self):
139        self.validate("yield from x")
140        self.validate("(yield from x) + y")
141        self.invalid_syntax("yield from")
142
143
144class TestAsyncAwait(GrammarTest):
145    def test_await_expr(self):
146        self.validate("""async def foo():
147                             await x
148                      """)
149
150        self.validate("""async def foo():
151                             [i async for i in b]
152                      """)
153
154        self.validate("""async def foo():
155                             {i for i in b
156                                async for i in a if await i
157                                  for b in i}
158                      """)
159
160        self.validate("""async def foo():
161                             [await i for i in b if await c]
162                      """)
163
164        self.validate("""async def foo():
165                             [ i for i in b if c]
166                      """)
167
168        self.validate("""async def foo():
169
170            def foo(): pass
171
172            def foo(): pass
173
174            await x
175        """)
176
177        self.validate("""async def foo(): return await a""")
178
179        self.validate("""def foo():
180            def foo(): pass
181            async def foo(): await x
182        """)
183
184        self.invalid_syntax("await x")
185        self.invalid_syntax("""def foo():
186                                   await x""")
187
188        self.invalid_syntax("""def foo():
189            def foo(): pass
190            async def foo(): pass
191            await x
192        """)
193
194    def test_async_var(self):
195        self.validate("""async = 1""")
196        self.validate("""await = 1""")
197        self.validate("""def async(): pass""")
198
199    def test_async_with(self):
200        self.validate("""async def foo():
201                             async for a in b: pass""")
202
203        self.invalid_syntax("""def foo():
204                                   async for a in b: pass""")
205
206    def test_async_for(self):
207        self.validate("""async def foo():
208                             async with a: pass""")
209
210        self.invalid_syntax("""def foo():
211                                   async with a: pass""")
212
213
214class TestRaiseChanges(GrammarTest):
215    def test_2x_style_1(self):
216        self.validate("raise")
217
218    def test_2x_style_2(self):
219        self.validate("raise E, V")
220
221    def test_2x_style_3(self):
222        self.validate("raise E, V, T")
223
224    def test_2x_style_invalid_1(self):
225        self.invalid_syntax("raise E, V, T, Z")
226
227    def test_3x_style(self):
228        self.validate("raise E1 from E2")
229
230    def test_3x_style_invalid_1(self):
231        self.invalid_syntax("raise E, V from E1")
232
233    def test_3x_style_invalid_2(self):
234        self.invalid_syntax("raise E from E1, E2")
235
236    def test_3x_style_invalid_3(self):
237        self.invalid_syntax("raise from E1, E2")
238
239    def test_3x_style_invalid_4(self):
240        self.invalid_syntax("raise E from")
241
242
243# Modelled after Lib/test/test_grammar.py:TokenTests.test_funcdef issue2292
244# and Lib/test/text_parser.py test_list_displays, test_set_displays,
245# test_dict_displays, test_argument_unpacking, ... changes.
246class TestUnpackingGeneralizations(GrammarTest):
247    def test_mid_positional_star(self):
248        self.validate("""func(1, *(2, 3), 4)""")
249
250    def test_double_star_dict_literal(self):
251        self.validate("""func(**{'eggs':'scrambled', 'spam':'fried'})""")
252
253    def test_double_star_dict_literal_after_keywords(self):
254        self.validate("""func(spam='fried', **{'eggs':'scrambled'})""")
255
256    def test_list_display(self):
257        self.validate("""[*{2}, 3, *[4]]""")
258
259    def test_set_display(self):
260        self.validate("""{*{2}, 3, *[4]}""")
261
262    def test_dict_display_1(self):
263        self.validate("""{**{}}""")
264
265    def test_dict_display_2(self):
266        self.validate("""{**{}, 3:4, **{5:6, 7:8}}""")
267
268    def test_argument_unpacking_1(self):
269        self.validate("""f(a, *b, *c, d)""")
270
271    def test_argument_unpacking_2(self):
272        self.validate("""f(**a, **b)""")
273
274    def test_argument_unpacking_3(self):
275        self.validate("""f(2, *a, *b, **b, **c, **d)""")
276
277    def test_trailing_commas_1(self):
278        self.validate("def f(a, b): call(a, b)")
279        self.validate("def f(a, b,): call(a, b,)")
280
281    def test_trailing_commas_2(self):
282        self.validate("def f(a, *b): call(a, *b)")
283        self.validate("def f(a, *b,): call(a, *b,)")
284
285    def test_trailing_commas_3(self):
286        self.validate("def f(a, b=1): call(a, b=1)")
287        self.validate("def f(a, b=1,): call(a, b=1,)")
288
289    def test_trailing_commas_4(self):
290        self.validate("def f(a, **b): call(a, **b)")
291        self.validate("def f(a, **b,): call(a, **b,)")
292
293    def test_trailing_commas_5(self):
294        self.validate("def f(*a, b=1): call(*a, b=1)")
295        self.validate("def f(*a, b=1,): call(*a, b=1,)")
296
297    def test_trailing_commas_6(self):
298        self.validate("def f(*a, **b): call(*a, **b)")
299        self.validate("def f(*a, **b,): call(*a, **b,)")
300
301    def test_trailing_commas_7(self):
302        self.validate("def f(*, b=1): call(*b)")
303        self.validate("def f(*, b=1,): call(*b,)")
304
305    def test_trailing_commas_8(self):
306        self.validate("def f(a=1, b=2): call(a=1, b=2)")
307        self.validate("def f(a=1, b=2,): call(a=1, b=2,)")
308
309    def test_trailing_commas_9(self):
310        self.validate("def f(a=1, **b): call(a=1, **b)")
311        self.validate("def f(a=1, **b,): call(a=1, **b,)")
312
313    def test_trailing_commas_lambda_1(self):
314        self.validate("f = lambda a, b: call(a, b)")
315        self.validate("f = lambda a, b,: call(a, b,)")
316
317    def test_trailing_commas_lambda_2(self):
318        self.validate("f = lambda a, *b: call(a, *b)")
319        self.validate("f = lambda a, *b,: call(a, *b,)")
320
321    def test_trailing_commas_lambda_3(self):
322        self.validate("f = lambda a, b=1: call(a, b=1)")
323        self.validate("f = lambda a, b=1,: call(a, b=1,)")
324
325    def test_trailing_commas_lambda_4(self):
326        self.validate("f = lambda a, **b: call(a, **b)")
327        self.validate("f = lambda a, **b,: call(a, **b,)")
328
329    def test_trailing_commas_lambda_5(self):
330        self.validate("f = lambda *a, b=1: call(*a, b=1)")
331        self.validate("f = lambda *a, b=1,: call(*a, b=1,)")
332
333    def test_trailing_commas_lambda_6(self):
334        self.validate("f = lambda *a, **b: call(*a, **b)")
335        self.validate("f = lambda *a, **b,: call(*a, **b,)")
336
337    def test_trailing_commas_lambda_7(self):
338        self.validate("f = lambda *, b=1: call(*b)")
339        self.validate("f = lambda *, b=1,: call(*b,)")
340
341    def test_trailing_commas_lambda_8(self):
342        self.validate("f = lambda a=1, b=2: call(a=1, b=2)")
343        self.validate("f = lambda a=1, b=2,: call(a=1, b=2,)")
344
345    def test_trailing_commas_lambda_9(self):
346        self.validate("f = lambda a=1, **b: call(a=1, **b)")
347        self.validate("f = lambda a=1, **b,: call(a=1, **b,)")
348
349
350# Adapted from Python 3's Lib/test/test_grammar.py:GrammarTests.testFuncdef
351class TestFunctionAnnotations(GrammarTest):
352    def test_1(self):
353        self.validate("""def f(x) -> list: pass""")
354
355    def test_2(self):
356        self.validate("""def f(x:int): pass""")
357
358    def test_3(self):
359        self.validate("""def f(*x:str): pass""")
360
361    def test_4(self):
362        self.validate("""def f(**x:float): pass""")
363
364    def test_5(self):
365        self.validate("""def f(x, y:1+2): pass""")
366
367    def test_6(self):
368        self.validate("""def f(a, (b:1, c:2, d)): pass""")
369
370    def test_7(self):
371        self.validate("""def f(a, (b:1, c:2, d), e:3=4, f=5, *g:6): pass""")
372
373    def test_8(self):
374        s = """def f(a, (b:1, c:2, d), e:3=4, f=5,
375                        *g:6, h:7, i=8, j:9=10, **k:11) -> 12: pass"""
376        self.validate(s)
377
378    def test_9(self):
379        s = """def f(
380          a: str,
381          b: int,
382          *,
383          c: bool = False,
384          **kwargs,
385        ) -> None:
386            call(c=c, **kwargs,)"""
387        self.validate(s)
388
389    def test_10(self):
390        s = """def f(
391          a: str,
392        ) -> None:
393            call(a,)"""
394        self.validate(s)
395
396    def test_11(self):
397        s = """def f(
398          a: str = '',
399        ) -> None:
400            call(a=a,)"""
401        self.validate(s)
402
403    def test_12(self):
404        s = """def f(
405          *args: str,
406        ) -> None:
407            call(*args,)"""
408        self.validate(s)
409
410    def test_13(self):
411        self.validate("def f(a: str, b: int) -> None: call(a, b)")
412        self.validate("def f(a: str, b: int,) -> None: call(a, b,)")
413
414    def test_14(self):
415        self.validate("def f(a: str, *b: int) -> None: call(a, *b)")
416        self.validate("def f(a: str, *b: int,) -> None: call(a, *b,)")
417
418    def test_15(self):
419        self.validate("def f(a: str, b: int=1) -> None: call(a, b=1)")
420        self.validate("def f(a: str, b: int=1,) -> None: call(a, b=1,)")
421
422    def test_16(self):
423        self.validate("def f(a: str, **b: int) -> None: call(a, **b)")
424        self.validate("def f(a: str, **b: int,) -> None: call(a, **b,)")
425
426    def test_17(self):
427        self.validate("def f(*a: str, b: int=1) -> None: call(*a, b=1)")
428        self.validate("def f(*a: str, b: int=1,) -> None: call(*a, b=1,)")
429
430    def test_18(self):
431        self.validate("def f(*a: str, **b: int) -> None: call(*a, **b)")
432        self.validate("def f(*a: str, **b: int,) -> None: call(*a, **b,)")
433
434    def test_19(self):
435        self.validate("def f(*, b: int=1) -> None: call(*b)")
436        self.validate("def f(*, b: int=1,) -> None: call(*b,)")
437
438    def test_20(self):
439        self.validate("def f(a: str='', b: int=2) -> None: call(a=a, b=2)")
440        self.validate("def f(a: str='', b: int=2,) -> None: call(a=a, b=2,)")
441
442    def test_21(self):
443        self.validate("def f(a: str='', **b: int) -> None: call(a=a, **b)")
444        self.validate("def f(a: str='', **b: int,) -> None: call(a=a, **b,)")
445
446
447# Adapted from Python 3's Lib/test/test_grammar.py:GrammarTests.test_var_annot
448class TestVarAnnotations(GrammarTest):
449    def test_1(self):
450        self.validate("var1: int = 5")
451
452    def test_2(self):
453        self.validate("var2: [int, str]")
454
455    def test_3(self):
456        self.validate("def f():\n"
457                      "    st: str = 'Hello'\n"
458                      "    a.b: int = (1, 2)\n"
459                      "    return st\n")
460
461    def test_4(self):
462        self.validate("def fbad():\n"
463                      "    x: int\n"
464                      "    print(x)\n")
465
466    def test_5(self):
467        self.validate("class C:\n"
468                      "    x: int\n"
469                      "    s: str = 'attr'\n"
470                      "    z = 2\n"
471                      "    def __init__(self, x):\n"
472                      "        self.x: int = x\n")
473
474    def test_6(self):
475        self.validate("lst: List[int] = []")
476
477
478class TestExcept(GrammarTest):
479    def test_new(self):
480        s = """
481            try:
482                x
483            except E as N:
484                y"""
485        self.validate(s)
486
487    def test_old(self):
488        s = """
489            try:
490                x
491            except E, N:
492                y"""
493        self.validate(s)
494
495
496class TestStringLiterals(GrammarTest):
497    prefixes = ("'", '"',
498        "r'", 'r"', "R'", 'R"',
499        "u'", 'u"', "U'", 'U"',
500        "b'", 'b"', "B'", 'B"',
501        "f'", 'f"', "F'", 'F"',
502        "ur'", 'ur"', "Ur'", 'Ur"',
503        "uR'", 'uR"', "UR'", 'UR"',
504        "br'", 'br"', "Br'", 'Br"',
505        "bR'", 'bR"', "BR'", 'BR"',
506        "rb'", 'rb"', "Rb'", 'Rb"',
507        "rB'", 'rB"', "RB'", 'RB"',)
508
509    def test_lit(self):
510        for pre in self.prefixes:
511            single = "{p}spamspamspam{s}".format(p=pre, s=pre[-1])
512            self.validate(single)
513            triple = "{p}{s}{s}eggs{s}{s}{s}".format(p=pre, s=pre[-1])
514            self.validate(triple)
515
516
517# Adapted from Python 3's Lib/test/test_grammar.py:GrammarTests.testAtoms
518class TestSetLiteral(GrammarTest):
519    def test_1(self):
520        self.validate("""x = {'one'}""")
521
522    def test_2(self):
523        self.validate("""x = {'one', 1,}""")
524
525    def test_3(self):
526        self.validate("""x = {'one', 'two', 'three'}""")
527
528    def test_4(self):
529        self.validate("""x = {2, 3, 4,}""")
530
531
532# Adapted from Python 3's Lib/test/test_unicode_identifiers.py and
533# Lib/test/test_tokenize.py:TokenizeTest.test_non_ascii_identifiers
534class TestIdentfier(GrammarTest):
535    def test_non_ascii_identifiers(self):
536        self.validate("Örter = 'places'\ngrün = 'green'")
537        self.validate("蟒 = a蟒 = 锦蛇 = 1")
538        self.validate("µ = aµ = µµ = 1")
539        self.validate("�������������� = a_�������������� = 1")
540
541
542class TestNumericLiterals(GrammarTest):
543    def test_new_octal_notation(self):
544        self.validate("""0o7777777777777""")
545        self.invalid_syntax("""0o7324528887""")
546
547    def test_new_binary_notation(self):
548        self.validate("""0b101010""")
549        self.invalid_syntax("""0b0101021""")
550
551
552class TestClassDef(GrammarTest):
553    def test_new_syntax(self):
554        self.validate("class B(t=7): pass")
555        self.validate("class B(t, *args): pass")
556        self.validate("class B(t, **kwargs): pass")
557        self.validate("class B(t, *args, **kwargs): pass")
558        self.validate("class B(t, y=9, *args, **kwargs,): pass")
559
560
561class TestParserIdempotency(support.TestCase):
562
563    """A cut-down version of pytree_idempotency.py."""
564
565    def test_all_project_files(self):
566        for filepath in support.all_project_files():
567            with open(filepath, "rb") as fp:
568                encoding = tokenize.detect_encoding(fp.readline)[0]
569            self.assertIsNotNone(encoding,
570                                 "can't detect encoding for %s" % filepath)
571            with open(filepath, "r", encoding=encoding) as fp:
572                source = fp.read()
573            try:
574                tree = driver.parse_string(source)
575            except ParseError:
576                try:
577                    tree = driver_no_print_statement.parse_string(source)
578                except ParseError as err:
579                    self.fail('ParseError on file %s (%s)' % (filepath, err))
580            new = str(tree)
581            if new != source:
582                print(diff_texts(source, new, filepath))
583                self.fail("Idempotency failed: %s" % filepath)
584
585    def test_extended_unpacking(self):
586        driver.parse_string("a, *b, c = x\n")
587        driver.parse_string("[*a, b] = x\n")
588        driver.parse_string("(z, *y, w) = m\n")
589        driver.parse_string("for *z, m in d: pass\n")
590
591
592class TestLiterals(GrammarTest):
593
594    def validate(self, s):
595        driver.parse_string(support.dedent(s) + "\n\n")
596
597    def test_multiline_bytes_literals(self):
598        s = """
599            md5test(b"\xaa" * 80,
600                    (b"Test Using Larger Than Block-Size Key "
601                     b"and Larger Than One Block-Size Data"),
602                    "6f630fad67cda0ee1fb1f562db3aa53e")
603            """
604        self.validate(s)
605
606    def test_multiline_bytes_tripquote_literals(self):
607        s = '''
608            b"""
609            <?xml version="1.0" encoding="UTF-8"?>
610            <!DOCTYPE plist PUBLIC "-//Apple Computer//DTD PLIST 1.0//EN">
611            """
612            '''
613        self.validate(s)
614
615    def test_multiline_str_literals(self):
616        s = """
617            md5test("\xaa" * 80,
618                    ("Test Using Larger Than Block-Size Key "
619                     "and Larger Than One Block-Size Data"),
620                    "6f630fad67cda0ee1fb1f562db3aa53e")
621            """
622        self.validate(s)
623
624
625def diff_texts(a, b, filename):
626    a = a.splitlines()
627    b = b.splitlines()
628    return difflib.unified_diff(a, b, filename, filename,
629                                "(original)", "(reserialized)",
630                                lineterm="")
631
632
633if __name__ == '__main__':
634    unittest.main()
635