1"""Test suite for 2to3's parser and grammar files.
2
3This is the place to add tests for changes to 2to3's grammar, such as those
4merging the grammars for Python 2 and 3. In addition to specific tests for
5parts of the grammar we've changed, we also make sure we can parse the
6test_grammar.py files from both Python 2 and Python 3.
7"""
8
9# Testing imports
10from . import support
11from .support import driver, test_dir
12
13# Python imports
14import operator
15import os
16import pickle
17import shutil
18import subprocess
19import sys
20import tempfile
21import types
22import unittest
23
24# Local imports
25from lib2to3.pgen2 import driver as pgen2_driver
26from lib2to3.pgen2 import tokenize
27from ..pgen2.parse import ParseError
28from lib2to3.pygram import python_symbols as syms
29
30
31class TestDriver(support.TestCase):
32
33    def test_formfeed(self):
34        s = """print 1\n\x0Cprint 2\n"""
35        t = driver.parse_string(s)
36        self.assertEqual(t.children[0].children[0].type, syms.print_stmt)
37        self.assertEqual(t.children[1].children[0].type, syms.print_stmt)
38
39
40class TestPgen2Caching(support.TestCase):
41    def test_load_grammar_from_txt_file(self):
42        pgen2_driver.load_grammar(support.grammar_path, save=False, force=True)
43
44    def test_load_grammar_from_pickle(self):
45        # Make a copy of the grammar file in a temp directory we are
46        # guaranteed to be able to write to.
47        tmpdir = tempfile.mkdtemp()
48        try:
49            grammar_copy = os.path.join(
50                    tmpdir, os.path.basename(support.grammar_path))
51            shutil.copy(support.grammar_path, grammar_copy)
52            pickle_name = pgen2_driver._generate_pickle_name(grammar_copy)
53
54            pgen2_driver.load_grammar(grammar_copy, save=True, force=True)
55            self.assertTrue(os.path.exists(pickle_name))
56
57            os.unlink(grammar_copy)  # Only the pickle remains...
58            pgen2_driver.load_grammar(grammar_copy, save=False, force=False)
59        finally:
60            shutil.rmtree(tmpdir)
61
62    @unittest.skipIf(sys.executable is None, 'sys.executable required')
63    def test_load_grammar_from_subprocess(self):
64        tmpdir = tempfile.mkdtemp()
65        tmpsubdir = os.path.join(tmpdir, 'subdir')
66        try:
67            os.mkdir(tmpsubdir)
68            grammar_base = os.path.basename(support.grammar_path)
69            grammar_copy = os.path.join(tmpdir, grammar_base)
70            grammar_sub_copy = os.path.join(tmpsubdir, grammar_base)
71            shutil.copy(support.grammar_path, grammar_copy)
72            shutil.copy(support.grammar_path, grammar_sub_copy)
73            pickle_name = pgen2_driver._generate_pickle_name(grammar_copy)
74            pickle_sub_name = pgen2_driver._generate_pickle_name(
75                     grammar_sub_copy)
76            self.assertNotEqual(pickle_name, pickle_sub_name)
77
78            # Generate a pickle file from this process.
79            pgen2_driver.load_grammar(grammar_copy, save=True, force=True)
80            self.assertTrue(os.path.exists(pickle_name))
81
82            # Generate a new pickle file in a subprocess with a most likely
83            # different hash randomization seed.
84            sub_env = dict(os.environ)
85            sub_env['PYTHONHASHSEED'] = 'random'
86            subprocess.check_call(
87                    [sys.executable, '-c', """
88from lib2to3.pgen2 import driver as pgen2_driver
89pgen2_driver.load_grammar(%r, save=True, force=True)
90                    """ % (grammar_sub_copy,)],
91                    env=sub_env)
92            self.assertTrue(os.path.exists(pickle_sub_name))
93
94            with open(pickle_name, 'rb') as pickle_f_1, \
95                    open(pickle_sub_name, 'rb') as pickle_f_2:
96                self.assertEqual(
97                    pickle_f_1.read(), pickle_f_2.read(),
98                    msg='Grammar caches generated using different hash seeds'
99                    ' were not identical.')
100        finally:
101            shutil.rmtree(tmpdir)
102
103    def test_load_packaged_grammar(self):
104        modname = __name__ + '.load_test'
105        class MyLoader:
106            def get_data(self, where):
107                return pickle.dumps({'elephant': 19})
108        class MyModule(types.ModuleType):
109            __file__ = 'parsertestmodule'
110            __loader__ = MyLoader()
111        sys.modules[modname] = MyModule(modname)
112        self.addCleanup(operator.delitem, sys.modules, modname)
113        g = pgen2_driver.load_packaged_grammar(modname, 'Grammar.txt')
114        self.assertEqual(g.elephant, 19)
115
116
117class GrammarTest(support.TestCase):
118    def validate(self, code):
119        support.parse_string(code)
120
121    def invalid_syntax(self, code):
122        try:
123            self.validate(code)
124        except ParseError:
125            pass
126        else:
127            raise AssertionError("Syntax shouldn't have been valid")
128
129
130class TestMatrixMultiplication(GrammarTest):
131    def test_matrix_multiplication_operator(self):
132        self.validate("a @ b")
133        self.validate("a @= b")
134
135
136class TestYieldFrom(GrammarTest):
137    def test_matrix_multiplication_operator(self):
138        self.validate("yield from x")
139        self.validate("(yield from x) + y")
140        self.invalid_syntax("yield from")
141
142
143class TestRaiseChanges(GrammarTest):
144    def test_2x_style_1(self):
145        self.validate("raise")
146
147    def test_2x_style_2(self):
148        self.validate("raise E, V")
149
150    def test_2x_style_3(self):
151        self.validate("raise E, V, T")
152
153    def test_2x_style_invalid_1(self):
154        self.invalid_syntax("raise E, V, T, Z")
155
156    def test_3x_style(self):
157        self.validate("raise E1 from E2")
158
159    def test_3x_style_invalid_1(self):
160        self.invalid_syntax("raise E, V from E1")
161
162    def test_3x_style_invalid_2(self):
163        self.invalid_syntax("raise E from E1, E2")
164
165    def test_3x_style_invalid_3(self):
166        self.invalid_syntax("raise from E1, E2")
167
168    def test_3x_style_invalid_4(self):
169        self.invalid_syntax("raise E from")
170
171
172# Modelled after Lib/test/test_grammar.py:TokenTests.test_funcdef issue2292
173# and Lib/test/text_parser.py test_list_displays, test_set_displays,
174# test_dict_displays, test_argument_unpacking, ... changes.
175class TestUnpackingGeneralizations(GrammarTest):
176    def test_mid_positional_star(self):
177        self.validate("""func(1, *(2, 3), 4)""")
178
179    def test_double_star_dict_literal(self):
180        self.validate("""func(**{'eggs':'scrambled', 'spam':'fried'})""")
181
182    def test_double_star_dict_literal_after_keywords(self):
183        self.validate("""func(spam='fried', **{'eggs':'scrambled'})""")
184
185    def test_list_display(self):
186        self.validate("""[*{2}, 3, *[4]]""")
187
188    def test_set_display(self):
189        self.validate("""{*{2}, 3, *[4]}""")
190
191    def test_dict_display_1(self):
192        self.validate("""{**{}}""")
193
194    def test_dict_display_2(self):
195        self.validate("""{**{}, 3:4, **{5:6, 7:8}}""")
196
197    def test_argument_unpacking_1(self):
198        self.validate("""f(a, *b, *c, d)""")
199
200    def test_argument_unpacking_2(self):
201        self.validate("""f(**a, **b)""")
202
203    def test_argument_unpacking_3(self):
204        self.validate("""f(2, *a, *b, **b, **c, **d)""")
205
206
207# Adaptated from Python 3's Lib/test/test_grammar.py:GrammarTests.testFuncdef
208class TestFunctionAnnotations(GrammarTest):
209    def test_1(self):
210        self.validate("""def f(x) -> list: pass""")
211
212    def test_2(self):
213        self.validate("""def f(x:int): pass""")
214
215    def test_3(self):
216        self.validate("""def f(*x:str): pass""")
217
218    def test_4(self):
219        self.validate("""def f(**x:float): pass""")
220
221    def test_5(self):
222        self.validate("""def f(x, y:1+2): pass""")
223
224    def test_6(self):
225        self.validate("""def f(a, (b:1, c:2, d)): pass""")
226
227    def test_7(self):
228        self.validate("""def f(a, (b:1, c:2, d), e:3=4, f=5, *g:6): pass""")
229
230    def test_8(self):
231        s = """def f(a, (b:1, c:2, d), e:3=4, f=5,
232                        *g:6, h:7, i=8, j:9=10, **k:11) -> 12: pass"""
233        self.validate(s)
234
235
236class TestExcept(GrammarTest):
237    def test_new(self):
238        s = """
239            try:
240                x
241            except E as N:
242                y"""
243        self.validate(s)
244
245    def test_old(self):
246        s = """
247            try:
248                x
249            except E, N:
250                y"""
251        self.validate(s)
252
253
254# Adapted from Python 3's Lib/test/test_grammar.py:GrammarTests.testAtoms
255class TestSetLiteral(GrammarTest):
256    def test_1(self):
257        self.validate("""x = {'one'}""")
258
259    def test_2(self):
260        self.validate("""x = {'one', 1,}""")
261
262    def test_3(self):
263        self.validate("""x = {'one', 'two', 'three'}""")
264
265    def test_4(self):
266        self.validate("""x = {2, 3, 4,}""")
267
268
269class TestNumericLiterals(GrammarTest):
270    def test_new_octal_notation(self):
271        self.validate("""0o7777777777777""")
272        self.invalid_syntax("""0o7324528887""")
273
274    def test_new_binary_notation(self):
275        self.validate("""0b101010""")
276        self.invalid_syntax("""0b0101021""")
277
278
279class TestClassDef(GrammarTest):
280    def test_new_syntax(self):
281        self.validate("class B(t=7): pass")
282        self.validate("class B(t, *args): pass")
283        self.validate("class B(t, **kwargs): pass")
284        self.validate("class B(t, *args, **kwargs): pass")
285        self.validate("class B(t, y=9, *args, **kwargs): pass")
286
287
288class TestParserIdempotency(support.TestCase):
289
290    """A cut-down version of pytree_idempotency.py."""
291
292    def test_all_project_files(self):
293        if sys.platform.startswith("win"):
294            # XXX something with newlines goes wrong on Windows.
295            return
296        for filepath in support.all_project_files():
297            with open(filepath, "rb") as fp:
298                encoding = tokenize.detect_encoding(fp.readline)[0]
299            self.assertIsNotNone(encoding,
300                                 "can't detect encoding for %s" % filepath)
301            with open(filepath, "r") as fp:
302                source = fp.read()
303                source = source.decode(encoding)
304            tree = driver.parse_string(source)
305            new = unicode(tree)
306            if diff(filepath, new, encoding):
307                self.fail("Idempotency failed: %s" % filepath)
308
309    def test_extended_unpacking(self):
310        driver.parse_string("a, *b, c = x\n")
311        driver.parse_string("[*a, b] = x\n")
312        driver.parse_string("(z, *y, w) = m\n")
313        driver.parse_string("for *z, m in d: pass\n")
314
315class TestLiterals(GrammarTest):
316
317    def validate(self, s):
318        driver.parse_string(support.dedent(s) + "\n\n")
319
320    def test_multiline_bytes_literals(self):
321        s = """
322            md5test(b"\xaa" * 80,
323                    (b"Test Using Larger Than Block-Size Key "
324                     b"and Larger Than One Block-Size Data"),
325                    "6f630fad67cda0ee1fb1f562db3aa53e")
326            """
327        self.validate(s)
328
329    def test_multiline_bytes_tripquote_literals(self):
330        s = '''
331            b"""
332            <?xml version="1.0" encoding="UTF-8"?>
333            <!DOCTYPE plist PUBLIC "-//Apple Computer//DTD PLIST 1.0//EN">
334            """
335            '''
336        self.validate(s)
337
338    def test_multiline_str_literals(self):
339        s = """
340            md5test("\xaa" * 80,
341                    ("Test Using Larger Than Block-Size Key "
342                     "and Larger Than One Block-Size Data"),
343                    "6f630fad67cda0ee1fb1f562db3aa53e")
344            """
345        self.validate(s)
346
347
348def diff(fn, result, encoding):
349    f = open("@", "w")
350    try:
351        f.write(result.encode(encoding))
352    finally:
353        f.close()
354    try:
355        fn = fn.replace('"', '\\"')
356        return os.system('diff -u "%s" @' % fn)
357    finally:
358        os.remove("@")
359