1
2import unittest
3import textwrap
4import antlr3
5import antlr3.tree
6import testbase
7import sys
8from io import StringIO
9
10class T(testbase.ANTLRTest):
11    def setUp(self):
12        self.oldPath = sys.path[:]
13        sys.path.insert(0, self.baseDir)
14
15
16    def tearDown(self):
17        sys.path = self.oldPath
18
19
20    def testOverrideMain(self):
21        grammar = textwrap.dedent(
22            r"""lexer grammar T3;
23            options {
24              language = Python3;
25              }
26
27            @main {
28            def main(argv):
29                raise RuntimeError("no")
30            }
31
32            ID: ('a'..'z' | '\u00c0'..'\u00ff')+;
33            WS: ' '+ { $channel = HIDDEN };
34            """)
35
36
37        stdout = StringIO()
38
39        lexerMod = self.compileInlineGrammar(grammar, returnModule=True)
40        self.assertRaises(RuntimeError, lexerMod.main, ['lexer.py'])
41
42
43    def testLexerFromFile(self):
44        input = "foo bar"
45        inputPath = self.writeFile("input.txt", input)
46
47        grammar = textwrap.dedent(
48            r"""lexer grammar T1;
49            options {
50              language = Python3;
51              }
52
53            ID: 'a'..'z'+;
54            WS: ' '+ { $channel = HIDDEN };
55            """)
56
57
58        stdout = StringIO()
59
60        lexerMod = self.compileInlineGrammar(grammar, returnModule=True)
61        lexerMod.main(
62            ['lexer.py', inputPath],
63            stdout=stdout
64            )
65
66        self.assertEqual(len(stdout.getvalue().splitlines()), 3)
67
68
69    def testLexerFromStdIO(self):
70        input = "foo bar"
71
72        grammar = textwrap.dedent(
73            r"""lexer grammar T2;
74            options {
75              language = Python3;
76              }
77
78            ID: 'a'..'z'+;
79            WS: ' '+ { $channel = HIDDEN };
80            """)
81
82
83        stdout = StringIO()
84
85        lexerMod = self.compileInlineGrammar(grammar, returnModule=True)
86        lexerMod.main(
87            ['lexer.py'],
88            stdin=StringIO(input),
89            stdout=stdout
90            )
91
92        self.assertEqual(len(stdout.getvalue().splitlines()), 3)
93
94
95    def testLexerEncoding(self):
96        input = "föö bär"
97
98        grammar = textwrap.dedent(
99            r"""lexer grammar T3;
100            options {
101              language = Python3;
102              }
103
104            ID: ('a'..'z' | '\u00c0'..'\u00ff')+;
105            WS: ' '+ { $channel = HIDDEN };
106            """)
107
108
109        stdout = StringIO()
110
111        lexerMod = self.compileInlineGrammar(grammar, returnModule=True)
112        lexerMod.main(
113            ['lexer.py'],
114            stdin=StringIO(input),
115            stdout=stdout
116            )
117
118        self.assertEqual(len(stdout.getvalue().splitlines()), 3)
119
120
121    def testCombined(self):
122        input = "foo bar"
123
124        grammar = textwrap.dedent(
125            r"""grammar T4;
126            options {
127              language = Python3;
128              }
129
130            r returns [res]: (ID)+ EOF { $res = $text };
131
132            ID: 'a'..'z'+;
133            WS: ' '+ { $channel = HIDDEN };
134            """)
135
136
137        stdout = StringIO()
138
139        lexerMod, parserMod = self.compileInlineGrammar(grammar, returnModule=True)
140        parserMod.main(
141            ['combined.py', '--rule', 'r'],
142            stdin=StringIO(input),
143            stdout=stdout
144            )
145
146        stdout = stdout.getvalue()
147        self.assertEqual(len(stdout.splitlines()), 1, stdout)
148
149
150    def testCombinedOutputAST(self):
151        input = "foo + bar"
152
153        grammar = textwrap.dedent(
154            r"""grammar T5;
155            options {
156              language = Python3;
157              output = AST;
158            }
159
160            r: ID OP^ ID EOF!;
161
162            ID: 'a'..'z'+;
163            OP: '+';
164            WS: ' '+ { $channel = HIDDEN };
165            """)
166
167
168        stdout = StringIO()
169
170        lexerMod, parserMod = self.compileInlineGrammar(grammar, returnModule=True)
171        parserMod.main(
172            ['combined.py', '--rule', 'r'],
173            stdin=StringIO(input),
174            stdout=stdout
175            )
176
177        stdout = stdout.getvalue().strip()
178        self.assertEqual(stdout, "(+ foo bar)")
179
180
181    def testTreeParser(self):
182        grammar = textwrap.dedent(
183            r'''grammar T6;
184            options {
185              language = Python3;
186              output = AST;
187            }
188
189            r: ID OP^ ID EOF!;
190
191            ID: 'a'..'z'+;
192            OP: '+';
193            WS: ' '+ { $channel = HIDDEN };
194            ''')
195
196        treeGrammar = textwrap.dedent(
197            r'''tree grammar T6Walker;
198            options {
199            language=Python3;
200            ASTLabelType=CommonTree;
201            tokenVocab=T6;
202            }
203            r returns [res]: ^(OP a=ID b=ID)
204              { $res = "{} {} {}".format($a.text, $OP.text, $b.text) }
205              ;
206            ''')
207
208        lexerMod, parserMod = self.compileInlineGrammar(grammar, returnModule=True)
209        walkerMod = self.compileInlineGrammar(treeGrammar, returnModule=True)
210
211        stdout = StringIO()
212        walkerMod.main(
213            ['walker.py', '--rule', 'r', '--parser', 'T6Parser', '--parser-rule', 'r', '--lexer', 'T6Lexer'],
214            stdin=StringIO("a+b"),
215            stdout=stdout
216            )
217
218        stdout = stdout.getvalue().strip()
219        self.assertEqual(stdout, "'a + b'")
220
221
222    def testTreeParserRewrite(self):
223        grammar = textwrap.dedent(
224            r'''grammar T7;
225            options {
226              language = Python3;
227              output = AST;
228            }
229
230            r: ID OP^ ID EOF!;
231
232            ID: 'a'..'z'+;
233            OP: '+';
234            WS: ' '+ { $channel = HIDDEN };
235            ''')
236
237        treeGrammar = textwrap.dedent(
238            r'''tree grammar T7Walker;
239            options {
240              language=Python3;
241              ASTLabelType=CommonTree;
242              tokenVocab=T7;
243              output=AST;
244            }
245            tokens {
246              ARG;
247            }
248            r: ^(OP a=ID b=ID) -> ^(OP ^(ARG ID) ^(ARG ID));
249            ''')
250
251        lexerMod, parserMod = self.compileInlineGrammar(grammar, returnModule=True)
252        walkerMod = self.compileInlineGrammar(treeGrammar, returnModule=True)
253
254        stdout = StringIO()
255        walkerMod.main(
256            ['walker.py', '--rule', 'r', '--parser', 'T7Parser', '--parser-rule', 'r', '--lexer', 'T7Lexer'],
257            stdin=StringIO("a+b"),
258            stdout=stdout
259            )
260
261        stdout = stdout.getvalue().strip()
262        self.assertEqual(stdout, "(+ (ARG a) (ARG b))")
263
264
265
266    def testGrammarImport(self):
267        slave = textwrap.dedent(
268            r'''
269            parser grammar T8S;
270            options {
271              language=Python3;
272            }
273
274            a : B;
275            ''')
276
277        parserName = self.writeInlineGrammar(slave)[0]
278        # slave parsers are imported as normal python modules
279        # to force reloading current version, purge module from sys.modules
280        if parserName + 'Parser' in sys.modules:
281            del sys.modules[parserName+'Parser']
282
283        master = textwrap.dedent(
284            r'''
285            grammar T8M;
286            options {
287              language=Python3;
288            }
289            import T8S;
290            s returns [res]: a { $res = $a.text };
291            B : 'b' ; // defines B from inherited token space
292            WS : (' '|'\n') {self.skip()} ;
293            ''')
294
295        stdout = StringIO()
296
297        lexerMod, parserMod = self.compileInlineGrammar(master, returnModule=True)
298        parserMod.main(
299            ['import.py', '--rule', 's'],
300            stdin=StringIO("b"),
301            stdout=stdout
302            )
303
304        stdout = stdout.getvalue().strip()
305        self.assertEqual(stdout, "'b'")
306
307
308if __name__ == '__main__':
309    unittest.main()
310