1 /* 2 * [The "BSD license"] 3 * Copyright (c) 2010 Terence Parr 4 * All rights reserved. 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 3. The name of the author may not be used to endorse or promote products 15 * derived from this software without specific prior written permission. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 18 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 19 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 20 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 21 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 22 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 26 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 */ 28 package org.antlr.test; 29 30 import org.antlr.Tool; 31 import org.antlr.codegen.CodeGenerator; 32 import org.antlr.tool.ErrorManager; 33 import org.antlr.tool.Grammar; 34 import org.antlr.tool.GrammarSemanticsMessage; 35 import org.junit.Ignore; 36 import org.junit.Test; 37 38 public class TestRewriteAST extends BaseTest { 39 protected boolean debug = false; 40 testDelete()41 @Test public void testDelete() throws Exception { 42 String grammar = 43 "grammar T;\n" + 44 "options {output=AST;}\n" + 45 "a : ID INT -> ;\n" + 46 "ID : 'a'..'z'+ ;\n" + 47 "INT : '0'..'9'+;\n" + 48 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 49 String found = execParser("T.g", grammar, "TParser", "TLexer", 50 "a", "abc 34", debug); 51 assertEquals("", found); 52 } 53 testSingleToken()54 @Test public void testSingleToken() throws Exception { 55 String grammar = 56 "grammar T;\n" + 57 "options {output=AST;}\n" + 58 "a : ID -> ID;\n" + 59 "ID : 'a'..'z'+ ;\n" + 60 "INT : '0'..'9'+;\n" + 61 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 62 String found = execParser("T.g", grammar, "TParser", "TLexer", 63 "a", "abc", debug); 64 assertEquals("abc\n", found); 65 } 66 testSingleTokenToNewNode()67 @Test public void testSingleTokenToNewNode() throws Exception { 68 String grammar = 69 "grammar T;\n" + 70 "options {output=AST;}\n" + 71 "a : ID -> ID[\"x\"];\n" + 72 "ID : 'a'..'z'+ ;\n" + 73 "INT : '0'..'9'+;\n" + 74 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 75 String found = execParser("T.g", grammar, "TParser", "TLexer", 76 "a", "abc", debug); 77 assertEquals("x\n", found); 78 } 79 testSingleTokenToNewNodeRoot()80 @Test public void testSingleTokenToNewNodeRoot() throws Exception { 81 String grammar = 82 "grammar T;\n" + 83 "options {output=AST;}\n" + 84 "a : ID -> ^(ID[\"x\"] INT);\n" + 85 "ID : 'a'..'z'+ ;\n" + 86 "INT : '0'..'9'+;\n" + 87 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 88 String found = execParser("T.g", grammar, "TParser", "TLexer", 89 "a", "abc", debug); 90 assertEquals("(x INT)\n", found); 91 } 92 testSingleTokenToNewNode2()93 @Test public void testSingleTokenToNewNode2() throws Exception { 94 // Allow creation of new nodes w/o args. 95 String grammar = 96 "grammar TT;\n" + 97 "options {output=AST;}\n" + 98 "a : ID -> ID[ ];\n" + 99 "ID : 'a'..'z'+ ;\n" + 100 "INT : '0'..'9'+;\n" + 101 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 102 String found = execParser("TT.g", grammar, "TTParser", "TTLexer", 103 "a", "abc", debug); 104 assertEquals("ID\n", found); 105 } 106 testSingleCharLiteral()107 @Test public void testSingleCharLiteral() throws Exception { 108 String grammar = 109 "grammar T;\n" + 110 "options {output=AST;}\n" + 111 "a : 'c' -> 'c';\n" + 112 "ID : 'a'..'z'+ ;\n" + 113 "INT : '0'..'9'+;\n" + 114 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 115 String found = execParser("T.g", grammar, "TParser", "TLexer", 116 "a", "c", debug); 117 assertEquals("c\n", found); 118 } 119 testSingleStringLiteral()120 @Test public void testSingleStringLiteral() throws Exception { 121 String grammar = 122 "grammar T;\n" + 123 "options {output=AST;}\n" + 124 "a : 'ick' -> 'ick';\n" + 125 "ID : 'a'..'z'+ ;\n" + 126 "INT : '0'..'9'+;\n" + 127 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 128 String found = execParser("T.g", grammar, "TParser", "TLexer", 129 "a", "ick", debug); 130 assertEquals("ick\n", found); 131 } 132 testSingleRule()133 @Test public void testSingleRule() throws Exception { 134 String grammar = 135 "grammar T;\n" + 136 "options {output=AST;}\n" + 137 "a : b -> b;\n" + 138 "b : ID ;\n" + 139 "ID : 'a'..'z'+ ;\n" + 140 "INT : '0'..'9'+;\n" + 141 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 142 String found = execParser("T.g", grammar, "TParser", "TLexer", 143 "a", "abc", debug); 144 assertEquals("abc\n", found); 145 } 146 testReorderTokens()147 @Test public void testReorderTokens() throws Exception { 148 String grammar = 149 "grammar T;\n" + 150 "options {output=AST;}\n" + 151 "a : ID INT -> INT ID;\n" + 152 "ID : 'a'..'z'+ ;\n" + 153 "INT : '0'..'9'+;\n" + 154 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 155 String found = execParser("T.g", grammar, "TParser", "TLexer", 156 "a", "abc 34", debug); 157 assertEquals("34 abc\n", found); 158 } 159 testReorderTokenAndRule()160 @Test public void testReorderTokenAndRule() throws Exception { 161 String grammar = 162 "grammar T;\n" + 163 "options {output=AST;}\n" + 164 "a : b INT -> INT b;\n" + 165 "b : ID ;\n" + 166 "ID : 'a'..'z'+ ;\n" + 167 "INT : '0'..'9'+;\n" + 168 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 169 String found = execParser("T.g", grammar, "TParser", "TLexer", 170 "a", "abc 34", debug); 171 assertEquals("34 abc\n", found); 172 } 173 testTokenTree()174 @Test public void testTokenTree() throws Exception { 175 String grammar = 176 "grammar T;\n" + 177 "options {output=AST;}\n" + 178 "a : ID INT -> ^(INT ID);\n" + 179 "ID : 'a'..'z'+ ;\n" + 180 "INT : '0'..'9'+;\n" + 181 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 182 String found = execParser("T.g", grammar, "TParser", "TLexer", 183 "a", "abc 34", debug); 184 assertEquals("(34 abc)\n", found); 185 } 186 testTokenTreeAfterOtherStuff()187 @Test public void testTokenTreeAfterOtherStuff() throws Exception { 188 String grammar = 189 "grammar T;\n" + 190 "options {output=AST;}\n" + 191 "a : 'void' ID INT -> 'void' ^(INT ID);\n" + 192 "ID : 'a'..'z'+ ;\n" + 193 "INT : '0'..'9'+;\n" + 194 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 195 String found = execParser("T.g", grammar, "TParser", "TLexer", 196 "a", "void abc 34", debug); 197 assertEquals("void (34 abc)\n", found); 198 } 199 testNestedTokenTreeWithOuterLoop()200 @Test public void testNestedTokenTreeWithOuterLoop() throws Exception { 201 // verify that ID and INT both iterate over outer index variable 202 String grammar = 203 "grammar T;\n" + 204 "options {output=AST;}\n" + 205 "tokens {DUH;}\n" + 206 "a : ID INT ID INT -> ^( DUH ID ^( DUH INT) )+ ;\n" + 207 "ID : 'a'..'z'+ ;\n" + 208 "INT : '0'..'9'+;\n" + 209 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 210 String found = execParser("T.g", grammar, "TParser", "TLexer", 211 "a", "a 1 b 2", debug); 212 assertEquals("(DUH a (DUH 1)) (DUH b (DUH 2))\n", found); 213 } 214 testOptionalSingleToken()215 @Test public void testOptionalSingleToken() throws Exception { 216 String grammar = 217 "grammar T;\n" + 218 "options {output=AST;}\n" + 219 "a : ID -> ID? ;\n" + 220 "ID : 'a'..'z'+ ;\n" + 221 "INT : '0'..'9'+;\n" + 222 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 223 String found = execParser("T.g", grammar, "TParser", "TLexer", 224 "a", "abc", debug); 225 assertEquals("abc\n", found); 226 } 227 testClosureSingleToken()228 @Test public void testClosureSingleToken() throws Exception { 229 String grammar = 230 "grammar T;\n" + 231 "options {output=AST;}\n" + 232 "a : ID ID -> ID* ;\n" + 233 "ID : 'a'..'z'+ ;\n" + 234 "INT : '0'..'9'+;\n" + 235 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 236 String found = execParser("T.g", grammar, "TParser", "TLexer", 237 "a", "a b", debug); 238 assertEquals("a b\n", found); 239 } 240 testPositiveClosureSingleToken()241 @Test public void testPositiveClosureSingleToken() throws Exception { 242 String grammar = 243 "grammar T;\n" + 244 "options {output=AST;}\n" + 245 "a : ID ID -> ID+ ;\n" + 246 "ID : 'a'..'z'+ ;\n" + 247 "INT : '0'..'9'+;\n" + 248 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 249 String found = execParser("T.g", grammar, "TParser", "TLexer", 250 "a", "a b", debug); 251 assertEquals("a b\n", found); 252 } 253 testOptionalSingleRule()254 @Test public void testOptionalSingleRule() throws Exception { 255 String grammar = 256 "grammar T;\n" + 257 "options {output=AST;}\n" + 258 "a : b -> b?;\n" + 259 "b : ID ;\n" + 260 "ID : 'a'..'z'+ ;\n" + 261 "INT : '0'..'9'+;\n" + 262 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 263 String found = execParser("T.g", grammar, "TParser", "TLexer", 264 "a", "abc", debug); 265 assertEquals("abc\n", found); 266 } 267 testClosureSingleRule()268 @Test public void testClosureSingleRule() throws Exception { 269 String grammar = 270 "grammar T;\n" + 271 "options {output=AST;}\n" + 272 "a : b b -> b*;\n" + 273 "b : ID ;\n" + 274 "ID : 'a'..'z'+ ;\n" + 275 "INT : '0'..'9'+;\n" + 276 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 277 String found = execParser("T.g", grammar, "TParser", "TLexer", 278 "a", "a b", debug); 279 assertEquals("a b\n", found); 280 } 281 testClosureOfLabel()282 @Test public void testClosureOfLabel() throws Exception { 283 String grammar = 284 "grammar T;\n" + 285 "options {output=AST;}\n" + 286 "a : x+=b x+=b -> $x*;\n" + 287 "b : ID ;\n" + 288 "ID : 'a'..'z'+ ;\n" + 289 "INT : '0'..'9'+;\n" + 290 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 291 String found = execParser("T.g", grammar, "TParser", "TLexer", 292 "a", "a b", debug); 293 assertEquals("a b\n", found); 294 } 295 testOptionalLabelNoListLabel()296 @Test public void testOptionalLabelNoListLabel() throws Exception { 297 String grammar = 298 "grammar T;\n" + 299 "options {output=AST;}\n" + 300 "a : (x=ID)? -> $x?;\n" + 301 "ID : 'a'..'z'+ ;\n" + 302 "INT : '0'..'9'+;\n" + 303 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 304 String found = execParser("T.g", grammar, "TParser", "TLexer", 305 "a", "a", debug); 306 assertEquals("a\n", found); 307 } 308 testPositiveClosureSingleRule()309 @Test public void testPositiveClosureSingleRule() throws Exception { 310 String grammar = 311 "grammar T;\n" + 312 "options {output=AST;}\n" + 313 "a : b b -> b+;\n" + 314 "b : ID ;\n" + 315 "ID : 'a'..'z'+ ;\n" + 316 "INT : '0'..'9'+;\n" + 317 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 318 String found = execParser("T.g", grammar, "TParser", "TLexer", 319 "a", "a b", debug); 320 assertEquals("a b\n", found); 321 } 322 testSinglePredicateT()323 @Test public void testSinglePredicateT() throws Exception { 324 String grammar = 325 "grammar T;\n" + 326 "options {output=AST;}\n" + 327 "a : ID -> {true}? ID -> ;\n" + 328 "ID : 'a'..'z'+ ;\n" + 329 "INT : '0'..'9'+;\n" + 330 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 331 String found = execParser("T.g", grammar, "TParser", "TLexer", 332 "a", "abc", debug); 333 assertEquals("abc\n", found); 334 } 335 testSinglePredicateF()336 @Test public void testSinglePredicateF() throws Exception { 337 String grammar = 338 "grammar T;\n" + 339 "options {output=AST;}\n" + 340 "a : ID -> {false}? ID -> ;\n" + 341 "ID : 'a'..'z'+ ;\n" + 342 "INT : '0'..'9'+;\n" + 343 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 344 String found = execParser("T.g", grammar, "TParser", "TLexer", 345 "a", "abc", debug); 346 assertEquals("", found); 347 } 348 testMultiplePredicate()349 @Test public void testMultiplePredicate() throws Exception { 350 String grammar = 351 "grammar T;\n" + 352 "options {output=AST;}\n" + 353 "a : ID INT -> {false}? ID\n" + 354 " -> {true}? INT\n" + 355 " -> \n" + 356 " ;\n" + 357 "ID : 'a'..'z'+ ;\n" + 358 "INT : '0'..'9'+;\n" + 359 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 360 String found = execParser("T.g", grammar, "TParser", "TLexer", 361 "a", "a 2", debug); 362 assertEquals("2\n", found); 363 } 364 testMultiplePredicateTrees()365 @Test public void testMultiplePredicateTrees() throws Exception { 366 String grammar = 367 "grammar T;\n" + 368 "options {output=AST;}\n" + 369 "a : ID INT -> {false}? ^(ID INT)\n" + 370 " -> {true}? ^(INT ID)\n" + 371 " -> ID\n" + 372 " ;\n" + 373 "ID : 'a'..'z'+ ;\n" + 374 "INT : '0'..'9'+;\n" + 375 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 376 String found = execParser("T.g", grammar, "TParser", "TLexer", 377 "a", "a 2", debug); 378 assertEquals("(2 a)\n", found); 379 } 380 testSimpleTree()381 @Test public void testSimpleTree() throws Exception { 382 String grammar = 383 "grammar T;\n" + 384 "options {output=AST;}\n" + 385 "a : op INT -> ^(op INT);\n" + 386 "op : '+'|'-' ;\n" + 387 "ID : 'a'..'z'+ ;\n" + 388 "INT : '0'..'9'+;\n" + 389 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 390 String found = execParser("T.g", grammar, "TParser", "TLexer", 391 "a", "-34", debug); 392 assertEquals("(- 34)\n", found); 393 } 394 testSimpleTree2()395 @Test public void testSimpleTree2() throws Exception { 396 String grammar = 397 "grammar T;\n" + 398 "options {output=AST;}\n" + 399 "a : op INT -> ^(INT op);\n" + 400 "op : '+'|'-' ;\n" + 401 "ID : 'a'..'z'+ ;\n" + 402 "INT : '0'..'9'+;\n" + 403 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 404 String found = execParser("T.g", grammar, "TParser", "TLexer", 405 "a", "+ 34", debug); 406 assertEquals("(34 +)\n", found); 407 } 408 409 testNestedTrees()410 @Test public void testNestedTrees() throws Exception { 411 String grammar = 412 "grammar T;\n" + 413 "options {output=AST;}\n" + 414 "a : 'var' (ID ':' type ';')+ -> ^('var' ^(':' ID type)+) ;\n" + 415 "type : 'int' | 'float' ;\n" + 416 "ID : 'a'..'z'+ ;\n" + 417 "INT : '0'..'9'+;\n" + 418 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 419 String found = execParser("T.g", grammar, "TParser", "TLexer", 420 "a", "var a:int; b:float;", debug); 421 assertEquals("(var (: a int) (: b float))\n", found); 422 } 423 testImaginaryTokenCopy()424 @Test public void testImaginaryTokenCopy() throws Exception { 425 String grammar = 426 "grammar T;\n" + 427 "options {output=AST;}\n" + 428 "tokens {VAR;}\n" + 429 "a : ID (',' ID)*-> ^(VAR ID)+ ;\n" + 430 "type : 'int' | 'float' ;\n" + 431 "ID : 'a'..'z'+ ;\n" + 432 "INT : '0'..'9'+;\n" + 433 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 434 String found = execParser("T.g", grammar, "TParser", "TLexer", 435 "a", "a,b,c", debug); 436 assertEquals("(VAR a) (VAR b) (VAR c)\n", found); 437 } 438 testTokenUnreferencedOnLeftButDefined()439 @Test public void testTokenUnreferencedOnLeftButDefined() throws Exception { 440 String grammar = 441 "grammar T;\n" + 442 "options {output=AST;}\n" + 443 "tokens {VAR;}\n" + 444 "a : b -> ID ;\n" + 445 "b : ID ;\n"+ 446 "ID : 'a'..'z'+ ;\n" + 447 "INT : '0'..'9'+;\n" + 448 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 449 String found = execParser("T.g", grammar, "TParser", "TLexer", 450 "a", "a", debug); 451 assertEquals("ID\n", found); 452 } 453 testImaginaryTokenCopySetText()454 @Test public void testImaginaryTokenCopySetText() throws Exception { 455 String grammar = 456 "grammar T;\n" + 457 "options {output=AST;}\n" + 458 "tokens {VAR;}\n" + 459 "a : ID (',' ID)*-> ^(VAR[\"var\"] ID)+ ;\n" + 460 "type : 'int' | 'float' ;\n" + 461 "ID : 'a'..'z'+ ;\n" + 462 "INT : '0'..'9'+;\n" + 463 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 464 String found = execParser("T.g", grammar, "TParser", "TLexer", 465 "a", "a,b,c", debug); 466 assertEquals("(var a) (var b) (var c)\n", found); 467 } 468 testImaginaryTokenNoCopyFromToken()469 @Test public void testImaginaryTokenNoCopyFromToken() throws Exception { 470 String grammar = 471 "grammar T;\n" + 472 "options {output=AST;}\n" + 473 "tokens {BLOCK;}\n" + 474 "a : lc='{' ID+ '}' -> ^(BLOCK[$lc] ID+) ;\n" + 475 "type : 'int' | 'float' ;\n" + 476 "ID : 'a'..'z'+ ;\n" + 477 "INT : '0'..'9'+;\n" + 478 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 479 String found = execParser("T.g", grammar, "TParser", "TLexer", 480 "a", "{a b c}", debug); 481 assertEquals("({ a b c)\n", found); 482 } 483 testImaginaryTokenNoCopyFromTokenSetText()484 @Test public void testImaginaryTokenNoCopyFromTokenSetText() throws Exception { 485 String grammar = 486 "grammar T;\n" + 487 "options {output=AST;}\n" + 488 "tokens {BLOCK;}\n" + 489 "a : lc='{' ID+ '}' -> ^(BLOCK[$lc,\"block\"] ID+) ;\n" + 490 "type : 'int' | 'float' ;\n" + 491 "ID : 'a'..'z'+ ;\n" + 492 "INT : '0'..'9'+;\n" + 493 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 494 String found = execParser("T.g", grammar, "TParser", "TLexer", 495 "a", "{a b c}", debug); 496 assertEquals("(block a b c)\n", found); 497 } 498 testMixedRewriteAndAutoAST()499 @Test public void testMixedRewriteAndAutoAST() throws Exception { 500 String grammar = 501 "grammar T;\n" + 502 "options {output=AST;}\n" + 503 "tokens {BLOCK;}\n" + 504 "a : b b^ ;\n" + // 2nd b matches only an INT; can make it root 505 "b : ID INT -> INT ID\n" + 506 " | INT\n" + 507 " ;\n" + 508 "ID : 'a'..'z'+ ;\n" + 509 "INT : '0'..'9'+;\n" + 510 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 511 String found = execParser("T.g", grammar, "TParser", "TLexer", 512 "a", "a 1 2", debug); 513 assertEquals("(2 1 a)\n", found); 514 } 515 testSubruleWithRewrite()516 @Test public void testSubruleWithRewrite() throws Exception { 517 String grammar = 518 "grammar T;\n" + 519 "options {output=AST;}\n" + 520 "tokens {BLOCK;}\n" + 521 "a : b b ;\n" + 522 "b : (ID INT -> INT ID | INT INT -> INT+ )\n" + 523 " ;\n" + 524 "ID : 'a'..'z'+ ;\n" + 525 "INT : '0'..'9'+;\n" + 526 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 527 String found = execParser("T.g", grammar, "TParser", "TLexer", 528 "a", "a 1 2 3", debug); 529 assertEquals("1 a 2 3\n", found); 530 } 531 testSubruleWithRewrite2()532 @Test public void testSubruleWithRewrite2() throws Exception { 533 String grammar = 534 "grammar T;\n" + 535 "options {output=AST;}\n" + 536 "tokens {TYPE;}\n" + 537 "a : b b ;\n" + 538 "b : 'int'\n" + 539 " ( ID -> ^(TYPE 'int' ID)\n" + 540 " | ID '=' INT -> ^(TYPE 'int' ID INT)\n" + 541 " )\n" + 542 " ';'\n" + 543 " ;\n" + 544 "ID : 'a'..'z'+ ;\n" + 545 "INT : '0'..'9'+;\n" + 546 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 547 String found = execParser("T.g", grammar, "TParser", "TLexer", 548 "a", "int a; int b=3;", debug); 549 assertEquals("(TYPE int a) (TYPE int b 3)\n", found); 550 } 551 testNestedRewriteShutsOffAutoAST()552 @Test public void testNestedRewriteShutsOffAutoAST() throws Exception { 553 String grammar = 554 "grammar T;\n" + 555 "options {output=AST;}\n" + 556 "tokens {BLOCK;}\n" + 557 "a : b b ;\n" + 558 "b : ID ( ID (last=ID -> $last)+ ) ';'\n" + // get last ID 559 " | INT\n" + // should still get auto AST construction 560 " ;\n" + 561 "ID : 'a'..'z'+ ;\n" + 562 "INT : '0'..'9'+;\n" + 563 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 564 String found = execParser("T.g", grammar, "TParser", "TLexer", 565 "a", "a b c d; 42", debug); 566 assertEquals("d 42\n", found); 567 } 568 testRewriteActions()569 @Test public void testRewriteActions() throws Exception { 570 String grammar = 571 "grammar T;\n" + 572 "options {output=AST;}\n" + 573 "a : atom -> ^({adaptor.create(INT,\"9\")} atom) ;\n" + 574 "atom : INT ;\n" + 575 "ID : 'a'..'z'+ ;\n" + 576 "INT : '0'..'9'+;\n" + 577 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 578 String found = execParser("T.g", grammar, "TParser", "TLexer", 579 "a", "3", debug); 580 assertEquals("(9 3)\n", found); 581 } 582 testRewriteActions2()583 @Test public void testRewriteActions2() throws Exception { 584 String grammar = 585 "grammar T;\n" + 586 "options {output=AST;}\n" + 587 "a : atom -> {adaptor.create(INT,\"9\")} atom ;\n" + 588 "atom : INT ;\n" + 589 "ID : 'a'..'z'+ ;\n" + 590 "INT : '0'..'9'+;\n" + 591 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 592 String found = execParser("T.g", grammar, "TParser", "TLexer", 593 "a", "3", debug); 594 assertEquals("9 3\n", found); 595 } 596 testRefToOldValue()597 @Test public void testRefToOldValue() throws Exception { 598 String grammar = 599 "grammar T;\n" + 600 "options {output=AST;}\n" + 601 "tokens {BLOCK;}\n" + 602 "a : (atom -> atom) (op='+' r=atom -> ^($op $a $r) )* ;\n" + 603 "atom : INT ;\n" + 604 "ID : 'a'..'z'+ ;\n" + 605 "INT : '0'..'9'+;\n" + 606 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 607 String found = execParser("T.g", grammar, "TParser", "TLexer", 608 "a", "3+4+5", debug); 609 assertEquals("(+ (+ 3 4) 5)\n", found); 610 } 611 testCopySemanticsForRules()612 @Test public void testCopySemanticsForRules() throws Exception { 613 String grammar = 614 "grammar T;\n" + 615 "options {output=AST;}\n" + 616 "tokens {BLOCK;}\n" + 617 "a : atom -> ^(atom atom) ;\n" + // NOT CYCLE! (dup atom) 618 "atom : INT ;\n" + 619 "ID : 'a'..'z'+ ;\n" + 620 "INT : '0'..'9'+;\n" + 621 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 622 String found = execParser("T.g", grammar, "TParser", "TLexer", 623 "a", "3", debug); 624 assertEquals("(3 3)\n", found); 625 } 626 testCopySemanticsForRules2()627 @Test public void testCopySemanticsForRules2() throws Exception { 628 // copy type as a root for each invocation of (...)+ in rewrite 629 String grammar = 630 "grammar T;\n" + 631 "options {output=AST;}\n" + 632 "a : type ID (',' ID)* ';' -> ^(type ID)+ ;\n" + 633 "type : 'int' ;\n" + 634 "ID : 'a'..'z'+ ;\n" + 635 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 636 String found = execParser("T.g", grammar, "TParser", "TLexer", 637 "a", "int a,b,c;", debug); 638 assertEquals("(int a) (int b) (int c)\n", found); 639 } 640 testCopySemanticsForRules3()641 @Test public void testCopySemanticsForRules3() throws Exception { 642 // copy type *and* modifier even though it's optional 643 // for each invocation of (...)+ in rewrite 644 String grammar = 645 "grammar T;\n" + 646 "options {output=AST;}\n" + 647 "a : modifier? type ID (',' ID)* ';' -> ^(type modifier? ID)+ ;\n" + 648 "type : 'int' ;\n" + 649 "modifier : 'public' ;\n" + 650 "ID : 'a'..'z'+ ;\n" + 651 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 652 String found = execParser("T.g", grammar, "TParser", "TLexer", 653 "a", "public int a,b,c;", debug); 654 assertEquals("(int public a) (int public b) (int public c)\n", found); 655 } 656 testCopySemanticsForRules3Double()657 @Test public void testCopySemanticsForRules3Double() throws Exception { 658 // copy type *and* modifier even though it's optional 659 // for each invocation of (...)+ in rewrite 660 String grammar = 661 "grammar T;\n" + 662 "options {output=AST;}\n" + 663 "a : modifier? type ID (',' ID)* ';' -> ^(type modifier? ID)+ ^(type modifier? ID)+ ;\n" + 664 "type : 'int' ;\n" + 665 "modifier : 'public' ;\n" + 666 "ID : 'a'..'z'+ ;\n" + 667 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 668 String found = execParser("T.g", grammar, "TParser", "TLexer", 669 "a", "public int a,b,c;", debug); 670 assertEquals("(int public a) (int public b) (int public c) (int public a) (int public b) (int public c)\n", found); 671 } 672 testCopySemanticsForRules4()673 @Test public void testCopySemanticsForRules4() throws Exception { 674 // copy type *and* modifier even though it's optional 675 // for each invocation of (...)+ in rewrite 676 String grammar = 677 "grammar T;\n" + 678 "options {output=AST;}\n" + 679 "tokens {MOD;}\n" + 680 "a : modifier? type ID (',' ID)* ';' -> ^(type ^(MOD modifier)? ID)+ ;\n" + 681 "type : 'int' ;\n" + 682 "modifier : 'public' ;\n" + 683 "ID : 'a'..'z'+ ;\n" + 684 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 685 String found = execParser("T.g", grammar, "TParser", "TLexer", 686 "a", "public int a,b,c;", debug); 687 assertEquals("(int (MOD public) a) (int (MOD public) b) (int (MOD public) c)\n", found); 688 } 689 testCopySemanticsLists()690 @Test public void testCopySemanticsLists() throws Exception { 691 String grammar = 692 "grammar T;\n" + 693 "options {output=AST;}\n" + 694 "tokens {MOD;}\n" + 695 "a : ID (',' ID)* ';' -> ID+ ID+ ;\n"+ 696 "ID : 'a'..'z'+ ;\n" + 697 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 698 String found = execParser("T.g", grammar, "TParser", "TLexer", 699 "a", "a,b,c;", debug); 700 assertEquals("a b c a b c\n", found); 701 } 702 testCopyRuleLabel()703 @Test public void testCopyRuleLabel() throws Exception { 704 String grammar = 705 "grammar T;\n" + 706 "options {output=AST;}\n" + 707 "tokens {BLOCK;}\n" + 708 "a : x=b -> $x $x;\n"+ 709 "b : ID ;\n"+ 710 "ID : 'a'..'z'+ ;\n" + 711 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 712 String found = execParser("T.g", grammar, "TParser", "TLexer", 713 "a", "a", debug); 714 assertEquals("a a\n", found); 715 } 716 testCopyRuleLabel2()717 @Test public void testCopyRuleLabel2() throws Exception { 718 String grammar = 719 "grammar T;\n" + 720 "options {output=AST;}\n" + 721 "tokens {BLOCK;}\n" + 722 "a : x=b -> ^($x $x);\n"+ 723 "b : ID ;\n"+ 724 "ID : 'a'..'z'+ ;\n" + 725 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 726 String found = execParser("T.g", grammar, "TParser", "TLexer", 727 "a", "a", debug); 728 assertEquals("(a a)\n", found); 729 } 730 testQueueingOfTokens()731 @Test public void testQueueingOfTokens() throws Exception { 732 String grammar = 733 "grammar T;\n" + 734 "options {output=AST;}\n" + 735 "a : 'int' ID (',' ID)* ';' -> ^('int' ID+) ;\n" + 736 "op : '+'|'-' ;\n" + 737 "ID : 'a'..'z'+ ;\n" + 738 "INT : '0'..'9'+;\n" + 739 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 740 String found = execParser("T.g", grammar, "TParser", "TLexer", 741 "a", "int a,b,c;", debug); 742 assertEquals("(int a b c)\n", found); 743 } 744 testCopyOfTokens()745 @Test public void testCopyOfTokens() throws Exception { 746 String grammar = 747 "grammar T;\n" + 748 "options {output=AST;}\n" + 749 "a : 'int' ID ';' -> 'int' ID 'int' ID ;\n" + 750 "op : '+'|'-' ;\n" + 751 "ID : 'a'..'z'+ ;\n" + 752 "INT : '0'..'9'+;\n" + 753 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 754 String found = execParser("T.g", grammar, "TParser", "TLexer", 755 "a", "int a;", debug); 756 assertEquals("int a int a\n", found); 757 } 758 testTokenCopyInLoop()759 @Test public void testTokenCopyInLoop() throws Exception { 760 String grammar = 761 "grammar T;\n" + 762 "options {output=AST;}\n" + 763 "a : 'int' ID (',' ID)* ';' -> ^('int' ID)+ ;\n" + 764 "op : '+'|'-' ;\n" + 765 "ID : 'a'..'z'+ ;\n" + 766 "INT : '0'..'9'+;\n" + 767 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 768 String found = execParser("T.g", grammar, "TParser", "TLexer", 769 "a", "int a,b,c;", debug); 770 assertEquals("(int a) (int b) (int c)\n", found); 771 } 772 testTokenCopyInLoopAgainstTwoOthers()773 @Test public void testTokenCopyInLoopAgainstTwoOthers() throws Exception { 774 // must smear 'int' copies across as root of multiple trees 775 String grammar = 776 "grammar T;\n" + 777 "options {output=AST;}\n" + 778 "a : 'int' ID ':' INT (',' ID ':' INT)* ';' -> ^('int' ID INT)+ ;\n" + 779 "op : '+'|'-' ;\n" + 780 "ID : 'a'..'z'+ ;\n" + 781 "INT : '0'..'9'+;\n" + 782 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 783 String found = execParser("T.g", grammar, "TParser", "TLexer", 784 "a", "int a:1,b:2,c:3;", debug); 785 assertEquals("(int a 1) (int b 2) (int c 3)\n", found); 786 } 787 testListRefdOneAtATime()788 @Test public void testListRefdOneAtATime() throws Exception { 789 String grammar = 790 "grammar T;\n" + 791 "options {output=AST;}\n" + 792 "a : ID+ -> ID ID ID ;\n" + // works if 3 input IDs 793 "op : '+'|'-' ;\n" + 794 "ID : 'a'..'z'+ ;\n" + 795 "INT : '0'..'9'+;\n" + 796 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 797 String found = execParser("T.g", grammar, "TParser", "TLexer", 798 "a", "a b c", debug); 799 assertEquals("a b c\n", found); 800 } 801 testSplitListWithLabels()802 @Test public void testSplitListWithLabels() throws Exception { 803 String grammar = 804 "grammar T;\n" + 805 "options {output=AST;}\n" + 806 "tokens {VAR;}\n"+ 807 "a : first=ID others+=ID* -> $first VAR $others+ ;\n" + 808 "op : '+'|'-' ;\n" + 809 "ID : 'a'..'z'+ ;\n" + 810 "INT : '0'..'9'+;\n" + 811 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 812 String found = execParser("T.g", grammar, "TParser", "TLexer", 813 "a", "a b c", debug); 814 assertEquals("a VAR b c\n", found); 815 } 816 testComplicatedMelange()817 @Test public void testComplicatedMelange() throws Exception { 818 String grammar = 819 "grammar T;\n" + 820 "options {output=AST;}\n" + 821 "tokens {BLOCK;}\n" + 822 "a : A A b=B B b=B c+=C C c+=C D {String s=$D.text;} -> A+ B+ C+ D ;\n" + 823 "type : 'int' | 'float' ;\n" + 824 "A : 'a' ;\n" + 825 "B : 'b' ;\n" + 826 "C : 'c' ;\n" + 827 "D : 'd' ;\n" + 828 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 829 String found = execParser("T.g", grammar, "TParser", "TLexer", 830 "a", "a a b b b c c c d", debug); 831 assertEquals("a a b b b c c c d\n", found); 832 } 833 testRuleLabel()834 @Test public void testRuleLabel() throws Exception { 835 String grammar = 836 "grammar T;\n" + 837 "options {output=AST;}\n" + 838 "tokens {BLOCK;}\n" + 839 "a : x=b -> $x;\n"+ 840 "b : ID ;\n"+ 841 "ID : 'a'..'z'+ ;\n" + 842 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 843 String found = execParser("T.g", grammar, "TParser", "TLexer", 844 "a", "a", debug); 845 assertEquals("a\n", found); 846 } 847 testAmbiguousRule()848 @Test public void testAmbiguousRule() throws Exception { 849 String grammar = 850 "grammar T;\n" + 851 "options {output=AST;}\n" + 852 "a : ID a -> a | INT ;\n"+ 853 "ID : 'a'..'z'+ ;\n" + 854 "INT: '0'..'9'+ ;\n" + 855 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 856 String found = execParser("T.g", grammar, "TParser", "TLexer", 857 "a", "abc 34", debug); 858 assertEquals("34\n", found); 859 } 860 testWeirdRuleRef()861 @Test public void testWeirdRuleRef() throws Exception { 862 ErrorQueue equeue = new ErrorQueue(); 863 ErrorManager.setErrorListener(equeue); 864 String grammar = 865 "grammar T;\n" + 866 "options {output=AST;}\n" + 867 "a : ID a -> $a | INT ;\n"+ 868 "ID : 'a'..'z'+ ;\n" + 869 "INT: '0'..'9'+ ;\n" + 870 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 871 872 Grammar g = new Grammar(grammar); 873 Tool antlr = newTool(); 874 antlr.setOutputDirectory(null); // write to /dev/null 875 CodeGenerator generator = new CodeGenerator(antlr, g, "Java"); 876 g.setCodeGenerator(generator); 877 generator.genRecognizer(); 878 879 // $a is ambig; is it previous root or ref to a ref in alt? 880 assertEquals("unexpected errors: "+equeue, 1, equeue.errors.size()); 881 } 882 testRuleListLabel()883 @Test public void testRuleListLabel() throws Exception { 884 String grammar = 885 "grammar T;\n" + 886 "options {output=AST;}\n" + 887 "tokens {BLOCK;}\n" + 888 "a : x+=b x+=b -> $x+;\n"+ 889 "b : ID ;\n"+ 890 "ID : 'a'..'z'+ ;\n" + 891 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 892 String found = execParser("T.g", grammar, "TParser", "TLexer", 893 "a", "a b", debug); 894 assertEquals("a b\n", found); 895 } 896 testRuleListLabel2()897 @Test public void testRuleListLabel2() throws Exception { 898 String grammar = 899 "grammar T;\n" + 900 "options {output=AST;}\n" + 901 "tokens {BLOCK;}\n" + 902 "a : x+=b x+=b -> $x $x*;\n"+ 903 "b : ID ;\n"+ 904 "ID : 'a'..'z'+ ;\n" + 905 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 906 String found = execParser("T.g", grammar, "TParser", "TLexer", 907 "a", "a b", debug); 908 assertEquals("a b\n", found); 909 } 910 testOptional()911 @Test public void testOptional() throws Exception { 912 String grammar = 913 "grammar T;\n" + 914 "options {output=AST;}\n" + 915 "tokens {BLOCK;}\n" + 916 "a : x=b (y=b)? -> $x $y?;\n"+ 917 "b : ID ;\n"+ 918 "ID : 'a'..'z'+ ;\n" + 919 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 920 String found = execParser("T.g", grammar, "TParser", "TLexer", 921 "a", "a", debug); 922 assertEquals("a\n", found); 923 } 924 testOptional2()925 @Test public void testOptional2() throws Exception { 926 String grammar = 927 "grammar T;\n" + 928 "options {output=AST;}\n" + 929 "tokens {BLOCK;}\n" + 930 "a : x=ID (y=b)? -> $x $y?;\n"+ 931 "b : ID ;\n"+ 932 "ID : 'a'..'z'+ ;\n" + 933 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 934 String found = execParser("T.g", grammar, "TParser", "TLexer", 935 "a", "a b", debug); 936 assertEquals("a b\n", found); 937 } 938 testOptional3()939 @Test public void testOptional3() throws Exception { 940 String grammar = 941 "grammar T;\n" + 942 "options {output=AST;}\n" + 943 "tokens {BLOCK;}\n" + 944 "a : x=ID (y=b)? -> ($x $y)?;\n"+ 945 "b : ID ;\n"+ 946 "ID : 'a'..'z'+ ;\n" + 947 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 948 String found = execParser("T.g", grammar, "TParser", "TLexer", 949 "a", "a b", debug); 950 assertEquals("a b\n", found); 951 } 952 testOptional4()953 @Test public void testOptional4() throws Exception { 954 String grammar = 955 "grammar T;\n" + 956 "options {output=AST;}\n" + 957 "tokens {BLOCK;}\n" + 958 "a : x+=ID (y=b)? -> ($x $y)?;\n"+ 959 "b : ID ;\n"+ 960 "ID : 'a'..'z'+ ;\n" + 961 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 962 String found = execParser("T.g", grammar, "TParser", "TLexer", 963 "a", "a b", debug); 964 assertEquals("a b\n", found); 965 } 966 testOptional5()967 @Test public void testOptional5() throws Exception { 968 String grammar = 969 "grammar T;\n" + 970 "options {output=AST;}\n" + 971 "tokens {BLOCK;}\n" + 972 "a : ID -> ID? ;\n"+ // match an ID to optional ID 973 "b : ID ;\n"+ 974 "ID : 'a'..'z'+ ;\n" + 975 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 976 String found = execParser("T.g", grammar, "TParser", "TLexer", 977 "a", "a", debug); 978 assertEquals("a\n", found); 979 } 980 testArbitraryExprType()981 @Test public void testArbitraryExprType() throws Exception { 982 String grammar = 983 "grammar T;\n" + 984 "options {output=AST;}\n" + 985 "tokens {BLOCK;}\n" + 986 "a : x+=b x+=b -> {new CommonTree()};\n"+ 987 "b : ID ;\n"+ 988 "ID : 'a'..'z'+ ;\n" + 989 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 990 String found = execParser("T.g", grammar, "TParser", "TLexer", 991 "a", "a b", debug); 992 assertEquals("", found); 993 } 994 testSet()995 @Test public void testSet() throws Exception { 996 String grammar = 997 "grammar T;\n" + 998 "options { output = AST; } \n" + 999 "a: (INT|ID)+ -> INT+ ID+ ;\n" + 1000 "INT: '0'..'9'+;\n" + 1001 "ID : 'a'..'z'+;\n" + 1002 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 1003 String found = execParser("T.g", grammar, "TParser", "TLexer", 1004 "a", "2 a 34 de", debug); 1005 assertEquals("2 34 a de\n", found); 1006 } 1007 testSet2()1008 @Test public void testSet2() throws Exception { 1009 String grammar = 1010 "grammar T;\n" + 1011 "options { output = AST; } \n" + 1012 "a: (INT|ID) -> INT? ID? ;\n" + 1013 "INT: '0'..'9'+;\n" + 1014 "ID : 'a'..'z'+;\n" + 1015 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 1016 String found = execParser("T.g", grammar, "TParser", "TLexer", 1017 "a", "2", debug); 1018 assertEquals("2\n", found); 1019 } 1020 1021 @Ignore 1022 // TODO: FAILS. The should probably generate a warning from antlr 1023 // See http://www.antlr.org:8888/browse/ANTLR-162 1024 // testSetWithLabel()1025 public void testSetWithLabel() throws Exception { 1026 1027 String grammar = 1028 "grammar T;\n" + 1029 "options { output = AST; } \n" + 1030 "a : x=(INT|ID) -> $x ;\n" + 1031 "INT: '0'..'9'+;\n" + 1032 "ID : 'a'..'z'+;\n" + 1033 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 1034 String found = execParser("T.g", grammar, "TParser", "TLexer", 1035 "a", "2", debug); 1036 assertEquals("2\n", found); 1037 } 1038 testRewriteAction()1039 @Test public void testRewriteAction() throws Exception { 1040 String grammar = 1041 "grammar T; \n" + 1042 "options { output = AST; }\n" + 1043 "tokens { FLOAT; }\n" + 1044 "r\n" + 1045 " : INT -> {new CommonTree(new CommonToken(FLOAT,$INT.text+\".0\"))} \n" + 1046 " ; \n" + 1047 "INT : '0'..'9'+; \n" + 1048 "WS: (' ' | '\\n' | '\\t')+ {$channel = HIDDEN;}; \n"; 1049 String found = execParser("T.g", grammar, "TParser", "TLexer", 1050 "r", "25", debug); 1051 assertEquals("25.0\n", found); 1052 } 1053 testOptionalSubruleWithoutRealElements()1054 @Test public void testOptionalSubruleWithoutRealElements() throws Exception { 1055 // copy type *and* modifier even though it's optional 1056 // for each invocation of (...)+ in rewrite 1057 String grammar = 1058 "grammar T;\n" + 1059 "options {output=AST;} \n" + 1060 "tokens {PARMS;} \n" + 1061 "\n" + 1062 "modulo \n" + 1063 " : 'modulo' ID ('(' parms+ ')')? -> ^('modulo' ID ^(PARMS parms+)?) \n" + 1064 " ; \n" + 1065 "parms : '#'|ID; \n" + 1066 "ID : ('a'..'z' | 'A'..'Z')+;\n" + 1067 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 1068 String found = execParser("T.g", grammar, "TParser", "TLexer", 1069 "modulo", "modulo abc (x y #)", debug); 1070 assertEquals("(modulo abc (PARMS x y #))\n", found); 1071 } 1072 1073 // C A R D I N A L I T Y I S S U E S 1074 testCardinality()1075 @Test public void testCardinality() throws Exception { 1076 String grammar = 1077 "grammar T;\n" + 1078 "options {output=AST;}\n" + 1079 "tokens {BLOCK;}\n" + 1080 "a : ID ID INT INT INT -> (ID INT)+;\n"+ 1081 "ID : 'a'..'z'+ ;\n" + 1082 "INT : '0'..'9'+; \n" + 1083 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 1084 execParser("T.g", grammar, "TParser", "TLexer", 1085 "a", "a b 3 4 5", debug); 1086 String expecting = 1087 "org.antlr.runtime.tree.RewriteCardinalityException: token ID"; 1088 String found = getFirstLineOfException(); 1089 assertEquals(expecting, found); 1090 } 1091 testCardinality2()1092 @Test public void testCardinality2() throws Exception { 1093 String grammar = 1094 "grammar T;\n" + 1095 "options {output=AST;}\n" + 1096 "a : ID+ -> ID ID ID ;\n" + // only 2 input IDs 1097 "op : '+'|'-' ;\n" + 1098 "ID : 'a'..'z'+ ;\n" + 1099 "INT : '0'..'9'+;\n" + 1100 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 1101 execParser("T.g", grammar, "TParser", "TLexer", 1102 "a", "a b", debug); 1103 String expecting = 1104 "org.antlr.runtime.tree.RewriteCardinalityException: token ID"; 1105 String found = getFirstLineOfException(); 1106 assertEquals(expecting, found); 1107 } 1108 testCardinality3()1109 @Test public void testCardinality3() throws Exception { 1110 String grammar = 1111 "grammar T;\n" + 1112 "options {output=AST;}\n" + 1113 "a : ID? INT -> ID INT ;\n" + 1114 "op : '+'|'-' ;\n" + 1115 "ID : 'a'..'z'+ ;\n" + 1116 "INT : '0'..'9'+;\n" + 1117 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 1118 execParser("T.g", grammar, "TParser", "TLexer", 1119 "a", "3", debug); 1120 String expecting = 1121 "org.antlr.runtime.tree.RewriteEmptyStreamException: token ID"; 1122 String found = getFirstLineOfException(); 1123 assertEquals(expecting, found); 1124 } 1125 testLoopCardinality()1126 @Test public void testLoopCardinality() throws Exception { 1127 String grammar = 1128 "grammar T;\n" + 1129 "options {output=AST;}\n" + 1130 "a : ID? INT -> ID+ INT ;\n" + 1131 "op : '+'|'-' ;\n" + 1132 "ID : 'a'..'z'+ ;\n" + 1133 "INT : '0'..'9'+;\n" + 1134 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 1135 execParser("T.g", grammar, "TParser", "TLexer", 1136 "a", "3", debug); 1137 String expecting = 1138 "org.antlr.runtime.tree.RewriteEarlyExitException"; 1139 String found = getFirstLineOfException(); 1140 assertEquals(expecting, found); 1141 } 1142 testWildcard()1143 @Test public void testWildcard() throws Exception { 1144 String grammar = 1145 "grammar T;\n" + 1146 "options {output=AST;}\n" + 1147 "a : ID c=. -> $c;\n" + 1148 "ID : 'a'..'z'+ ;\n" + 1149 "INT : '0'..'9'+;\n" + 1150 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 1151 String found = execParser("T.g", grammar, "TParser", "TLexer", 1152 "a", "abc 34", debug); 1153 assertEquals("34\n", found); 1154 } 1155 1156 // E R R O R S 1157 testUnknownRule()1158 @Test public void testUnknownRule() throws Exception { 1159 ErrorQueue equeue = new ErrorQueue(); 1160 ErrorManager.setErrorListener(equeue); 1161 1162 String grammar = 1163 "grammar T;\n" + 1164 "options {output=AST;}\n" + 1165 "a : INT -> ugh ;\n" + 1166 "ID : 'a'..'z'+ ;\n" + 1167 "INT : '0'..'9'+;\n" + 1168 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 1169 1170 Grammar g = new Grammar(grammar); 1171 Tool antlr = newTool(); 1172 antlr.setOutputDirectory(null); // write to /dev/null 1173 CodeGenerator generator = new CodeGenerator(antlr, g, "Java"); 1174 g.setCodeGenerator(generator); 1175 generator.genRecognizer(); 1176 1177 int expectedMsgID = ErrorManager.MSG_UNDEFINED_RULE_REF; 1178 Object expectedArg = "ugh"; 1179 Object expectedArg2 = null; 1180 GrammarSemanticsMessage expectedMessage = 1181 new GrammarSemanticsMessage(expectedMsgID, g, null, expectedArg, expectedArg2); 1182 1183 checkError(equeue, expectedMessage); 1184 } 1185 testKnownRuleButNotInLHS()1186 @Test public void testKnownRuleButNotInLHS() throws Exception { 1187 ErrorQueue equeue = new ErrorQueue(); 1188 ErrorManager.setErrorListener(equeue); 1189 1190 String grammar = 1191 "grammar T;\n" + 1192 "options {output=AST;}\n" + 1193 "a : INT -> b ;\n" + 1194 "b : 'b' ;\n" + 1195 "ID : 'a'..'z'+ ;\n" + 1196 "INT : '0'..'9'+;\n" + 1197 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 1198 1199 Grammar g = new Grammar(grammar); 1200 Tool antlr = newTool(); 1201 antlr.setOutputDirectory(null); // write to /dev/null 1202 CodeGenerator generator = new CodeGenerator(antlr, g, "Java"); 1203 g.setCodeGenerator(generator); 1204 generator.genRecognizer(); 1205 1206 int expectedMsgID = ErrorManager.MSG_REWRITE_ELEMENT_NOT_PRESENT_ON_LHS; 1207 Object expectedArg = "b"; 1208 Object expectedArg2 = null; 1209 GrammarSemanticsMessage expectedMessage = 1210 new GrammarSemanticsMessage(expectedMsgID, g, null, expectedArg, expectedArg2); 1211 1212 checkError(equeue, expectedMessage); 1213 } 1214 testUnknownToken()1215 @Test public void testUnknownToken() throws Exception { 1216 ErrorQueue equeue = new ErrorQueue(); 1217 ErrorManager.setErrorListener(equeue); 1218 1219 String grammar = 1220 "grammar T;\n" + 1221 "options {output=AST;}\n" + 1222 "a : INT -> ICK ;\n" + 1223 "ID : 'a'..'z'+ ;\n" + 1224 "INT : '0'..'9'+;\n" + 1225 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 1226 1227 Grammar g = new Grammar(grammar); 1228 Tool antlr = newTool(); 1229 antlr.setOutputDirectory(null); // write to /dev/null 1230 CodeGenerator generator = new CodeGenerator(antlr, g, "Java"); 1231 g.setCodeGenerator(generator); 1232 generator.genRecognizer(); 1233 1234 int expectedMsgID = ErrorManager.MSG_UNDEFINED_TOKEN_REF_IN_REWRITE; 1235 Object expectedArg = "ICK"; 1236 Object expectedArg2 = null; 1237 GrammarSemanticsMessage expectedMessage = 1238 new GrammarSemanticsMessage(expectedMsgID, g, null, expectedArg, expectedArg2); 1239 1240 checkError(equeue, expectedMessage); 1241 } 1242 testUnknownLabel()1243 @Test public void testUnknownLabel() throws Exception { 1244 ErrorQueue equeue = new ErrorQueue(); 1245 ErrorManager.setErrorListener(equeue); 1246 1247 String grammar = 1248 "grammar T;\n" + 1249 "options {output=AST;}\n" + 1250 "a : INT -> $foo ;\n" + 1251 "ID : 'a'..'z'+ ;\n" + 1252 "INT : '0'..'9'+;\n" + 1253 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 1254 1255 Grammar g = new Grammar(grammar); 1256 Tool antlr = newTool(); 1257 antlr.setOutputDirectory(null); // write to /dev/null 1258 CodeGenerator generator = new CodeGenerator(antlr, g, "Java"); 1259 g.setCodeGenerator(generator); 1260 generator.genRecognizer(); 1261 1262 int expectedMsgID = ErrorManager.MSG_UNDEFINED_LABEL_REF_IN_REWRITE; 1263 Object expectedArg = "foo"; 1264 Object expectedArg2 = null; 1265 GrammarSemanticsMessage expectedMessage = 1266 new GrammarSemanticsMessage(expectedMsgID, g, null, expectedArg, expectedArg2); 1267 1268 checkError(equeue, expectedMessage); 1269 } 1270 testUnknownCharLiteralToken()1271 @Test public void testUnknownCharLiteralToken() throws Exception { 1272 ErrorQueue equeue = new ErrorQueue(); 1273 ErrorManager.setErrorListener(equeue); 1274 1275 String grammar = 1276 "grammar T;\n" + 1277 "options {output=AST;}\n" + 1278 "a : INT -> 'a' ;\n" + 1279 "ID : 'a'..'z'+ ;\n" + 1280 "INT : '0'..'9'+;\n" + 1281 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 1282 1283 Grammar g = new Grammar(grammar); 1284 Tool antlr = newTool(); 1285 antlr.setOutputDirectory(null); // write to /dev/null 1286 CodeGenerator generator = new CodeGenerator(antlr, g, "Java"); 1287 g.setCodeGenerator(generator); 1288 generator.genRecognizer(); 1289 1290 int expectedMsgID = ErrorManager.MSG_UNDEFINED_TOKEN_REF_IN_REWRITE; 1291 Object expectedArg = "'a'"; 1292 Object expectedArg2 = null; 1293 GrammarSemanticsMessage expectedMessage = 1294 new GrammarSemanticsMessage(expectedMsgID, g, null, expectedArg, expectedArg2); 1295 1296 checkError(equeue, expectedMessage); 1297 } 1298 testUnknownStringLiteralToken()1299 @Test public void testUnknownStringLiteralToken() throws Exception { 1300 ErrorQueue equeue = new ErrorQueue(); 1301 ErrorManager.setErrorListener(equeue); 1302 1303 String grammar = 1304 "grammar T;\n" + 1305 "options {output=AST;}\n" + 1306 "a : INT -> 'foo' ;\n" + 1307 "ID : 'a'..'z'+ ;\n" + 1308 "INT : '0'..'9'+;\n" + 1309 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 1310 1311 Grammar g = new Grammar(grammar); 1312 Tool antlr = newTool(); 1313 antlr.setOutputDirectory(null); // write to /dev/null 1314 CodeGenerator generator = new CodeGenerator(antlr, g, "Java"); 1315 g.setCodeGenerator(generator); 1316 generator.genRecognizer(); 1317 1318 int expectedMsgID = ErrorManager.MSG_UNDEFINED_TOKEN_REF_IN_REWRITE; 1319 Object expectedArg = "'foo'"; 1320 Object expectedArg2 = null; 1321 GrammarSemanticsMessage expectedMessage = 1322 new GrammarSemanticsMessage(expectedMsgID, g, null, expectedArg, expectedArg2); 1323 1324 checkError(equeue, expectedMessage); 1325 } 1326 testExtraTokenInSimpleDecl()1327 @Test public void testExtraTokenInSimpleDecl() throws Exception { 1328 String grammar = 1329 "grammar foo;\n" + 1330 "options {output=AST;}\n" + 1331 "tokens {EXPR;}\n" + 1332 "decl : type ID '=' INT ';' -> ^(EXPR type ID INT) ;\n" + 1333 "type : 'int' | 'float' ;\n" + 1334 "ID : 'a'..'z'+ ;\n" + 1335 "INT : '0'..'9'+;\n" + 1336 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 1337 String found = execParser("foo.g", grammar, "fooParser", "fooLexer", 1338 "decl", "int 34 x=1;", debug); 1339 assertEquals("line 1:4 extraneous input '34' expecting ID\n", this.stderrDuringParse); 1340 assertEquals("(EXPR int x 1)\n", found); // tree gets correct x and 1 tokens 1341 } 1342 testMissingIDInSimpleDecl()1343 @Test public void testMissingIDInSimpleDecl() throws Exception { 1344 String grammar = 1345 "grammar foo;\n" + 1346 "options {output=AST;}\n" + 1347 "tokens {EXPR;}\n" + 1348 "decl : type ID '=' INT ';' -> ^(EXPR type ID INT) ;\n" + 1349 "type : 'int' | 'float' ;\n" + 1350 "ID : 'a'..'z'+ ;\n" + 1351 "INT : '0'..'9'+;\n" + 1352 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 1353 String found = execParser("foo.g", grammar, "fooParser", "fooLexer", 1354 "decl", "int =1;", debug); 1355 assertEquals("line 1:4 missing ID at '='\n", this.stderrDuringParse); 1356 assertEquals("(EXPR int <missing ID> 1)\n", found); // tree gets invented ID token 1357 } 1358 testMissingSetInSimpleDecl()1359 @Test public void testMissingSetInSimpleDecl() throws Exception { 1360 String grammar = 1361 "grammar foo;\n" + 1362 "options {output=AST;}\n" + 1363 "tokens {EXPR;}\n" + 1364 "decl : type ID '=' INT ';' -> ^(EXPR type ID INT) ;\n" + 1365 "type : 'int' | 'float' ;\n" + 1366 "ID : 'a'..'z'+ ;\n" + 1367 "INT : '0'..'9'+;\n" + 1368 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 1369 String found = execParser("foo.g", grammar, "fooParser", "fooLexer", 1370 "decl", "x=1;", debug); 1371 assertEquals("line 1:0 mismatched input 'x' expecting set null\n", this.stderrDuringParse); 1372 assertEquals("(EXPR <error: x> x 1)\n", found); // tree gets invented ID token 1373 } 1374 testMissingTokenGivesErrorNode()1375 @Test public void testMissingTokenGivesErrorNode() throws Exception { 1376 String grammar = 1377 "grammar foo;\n" + 1378 "options {output=AST;}\n" + 1379 "a : ID INT -> ID INT ;\n" + 1380 "ID : 'a'..'z'+ ;\n" + 1381 "INT : '0'..'9'+;\n" + 1382 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 1383 String found = execParser("foo.g", grammar, "fooParser", "fooLexer", 1384 "a", "abc", debug); 1385 assertEquals("line 1:3 missing INT at '<EOF>'\n", this.stderrDuringParse); 1386 // doesn't do in-line recovery for sets (yet?) 1387 assertEquals("abc <missing INT>\n", found); 1388 } 1389 testExtraTokenGivesErrorNode()1390 @Test public void testExtraTokenGivesErrorNode() throws Exception { 1391 String grammar = 1392 "grammar foo;\n" + 1393 "options {output=AST;}\n" + 1394 "a : b c -> b c;\n" + 1395 "b : ID -> ID ;\n" + 1396 "c : INT -> INT ;\n" + 1397 "ID : 'a'..'z'+ ;\n" + 1398 "INT : '0'..'9'+;\n" + 1399 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 1400 String found = execParser("foo.g", grammar, "fooParser", "fooLexer", 1401 "a", "abc ick 34", debug); 1402 assertEquals("line 1:4 extraneous input 'ick' expecting INT\n", this.stderrDuringParse); 1403 assertEquals("abc 34\n", found); 1404 } 1405 testMissingFirstTokenGivesErrorNode()1406 @Test public void testMissingFirstTokenGivesErrorNode() throws Exception { 1407 String grammar = 1408 "grammar foo;\n" + 1409 "options {output=AST;}\n" + 1410 "a : ID INT -> ID INT ;\n" + 1411 "ID : 'a'..'z'+ ;\n" + 1412 "INT : '0'..'9'+;\n" + 1413 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 1414 String found = execParser("foo.g", grammar, "fooParser", "fooLexer", 1415 "a", "34", debug); 1416 assertEquals("line 1:0 missing ID at '34'\n", this.stderrDuringParse); 1417 assertEquals("<missing ID> 34\n", found); 1418 } 1419 testMissingFirstTokenGivesErrorNode2()1420 @Test public void testMissingFirstTokenGivesErrorNode2() throws Exception { 1421 String grammar = 1422 "grammar foo;\n" + 1423 "options {output=AST;}\n" + 1424 "a : b c -> b c;\n" + 1425 "b : ID -> ID ;\n" + 1426 "c : INT -> INT ;\n" + 1427 "ID : 'a'..'z'+ ;\n" + 1428 "INT : '0'..'9'+;\n" + 1429 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 1430 String found = execParser("foo.g", grammar, "fooParser", "fooLexer", 1431 "a", "34", debug); 1432 // finds an error at the first token, 34, and re-syncs. 1433 // re-synchronizing does not consume a token because 34 follows 1434 // ref to rule b (start of c). It then matches 34 in c. 1435 assertEquals("line 1:0 missing ID at '34'\n", this.stderrDuringParse); 1436 assertEquals("<missing ID> 34\n", found); 1437 } 1438 testNoViableAltGivesErrorNode()1439 @Test public void testNoViableAltGivesErrorNode() throws Exception { 1440 String grammar = 1441 "grammar foo;\n" + 1442 "options {output=AST;}\n" + 1443 "a : b -> b | c -> c;\n" + 1444 "b : ID -> ID ;\n" + 1445 "c : INT -> INT ;\n" + 1446 "ID : 'a'..'z'+ ;\n" + 1447 "S : '*' ;\n" + 1448 "INT : '0'..'9'+;\n" + 1449 "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n"; 1450 String found = execParser("foo.g", grammar, "fooParser", "fooLexer", 1451 "a", "*", debug); 1452 // finds an error at the first token, 34, and re-syncs. 1453 // re-synchronizing does not consume a token because 34 follows 1454 // ref to rule b (start of c). It then matches 34 in c. 1455 assertEquals("line 1:0 no viable alternative at input '*'\n", this.stderrDuringParse); 1456 assertEquals("<unexpected: [@0,0:0='*',<6>,1:0], resync=*>\n", found); 1457 } 1458 1459 } 1460