1 /*
2  * [The "BSD license"]
3  *  Copyright (c) 2010 Terence Parr
4  *  All rights reserved.
5  *
6  *  Redistribution and use in source and binary forms, with or without
7  *  modification, are permitted provided that the following conditions
8  *  are met:
9  *  1. Redistributions of source code must retain the above copyright
10  *      notice, this list of conditions and the following disclaimer.
11  *  2. Redistributions in binary form must reproduce the above copyright
12  *      notice, this list of conditions and the following disclaimer in the
13  *      documentation and/or other materials provided with the distribution.
14  *  3. The name of the author may not be used to endorse or promote products
15  *      derived from this software without specific prior written permission.
16  *
17  *  THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
18  *  IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
19  *  OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
20  *  IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
21  *  INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
22  *  NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23  *  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24  *  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25  *  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
26  *  THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27  */
28 package org.antlr.test;
29 
30 import org.antlr.Tool;
31 import org.antlr.codegen.CodeGenerator;
32 import org.stringtemplate.v4.ST;
33 import org.antlr.tool.Grammar;
34 import org.junit.Test;
35 
36 public class TestLexer extends BaseTest {
37 	protected boolean debug = false;
38 
39 	/** Public default constructor used by TestRig */
TestLexer()40 	public TestLexer() {
41 	}
42 
testSetText()43 	@Test public void testSetText() throws Exception {
44 		// this must return A not I to the parser; calling a nonfragment rule
45 		// from a nonfragment rule does not set the overall token.
46 		String grammar =
47 			"grammar P;\n"+
48 			"a : A {System.out.println(input);} ;\n"+
49 			"A : '\\\\' 't' {setText(\"\t\");} ;\n" +
50 			"WS : (' '|'\\n') {$channel=HIDDEN;} ;";
51 		String found = execParser("P.g", grammar, "PParser", "PLexer",
52 				    "a", "\\t", debug);
53 		assertEquals("\t\n", found);
54 	}
55 
testRefToRuleDoesNotSetTokenNorEmitAnother()56 	@Test public void testRefToRuleDoesNotSetTokenNorEmitAnother() throws Exception {
57 		// this must return A not I to the parser; calling a nonfragment rule
58 		// from a nonfragment rule does not set the overall token.
59 		String grammar =
60 			"grammar P;\n"+
61 			"a : A EOF {System.out.println(input);} ;\n"+
62 			"A : '-' I ;\n" +
63 			"I : '0'..'9'+ ;\n"+
64 			"WS : (' '|'\\n') {$channel=HIDDEN;} ;";
65 		String found = execParser("P.g", grammar, "PParser", "PLexer",
66 				    "a", "-34", debug);
67 		assertEquals("-34\n", found);
68 	}
69 
testRefToRuleDoesNotSetChannel()70 	@Test public void testRefToRuleDoesNotSetChannel() throws Exception {
71 		// this must set channel of A to HIDDEN.  $channel is local to rule
72 		// like $type.
73 		String grammar =
74 			"grammar P;\n"+
75 			"a : A EOF {System.out.println($A.text+\", channel=\"+$A.channel);} ;\n"+
76 			"A : '-' WS I ;\n" +
77 			"I : '0'..'9'+ ;\n"+
78 			"WS : (' '|'\\n') {$channel=HIDDEN;} ;";
79 		String found = execParser("P.g", grammar, "PParser", "PLexer",
80 				    "a", "- 34", debug);
81 		assertEquals("- 34, channel=0\n", found);
82 	}
83 
testWeCanSetType()84 	@Test public void testWeCanSetType() throws Exception {
85 		String grammar =
86 			"grammar P;\n"+
87 			"tokens {X;}\n" +
88 			"a : X EOF {System.out.println(input);} ;\n"+
89 			"A : '-' I {$type = X;} ;\n" +
90 			"I : '0'..'9'+ ;\n"+
91 			"WS : (' '|'\\n') {$channel=HIDDEN;} ;";
92 		String found = execParser("P.g", grammar, "PParser", "PLexer",
93 				    "a", "-34", debug);
94 		assertEquals("-34\n", found);
95 	}
96 
testRefToFragment()97 	@Test public void testRefToFragment() throws Exception {
98 		// this must return A not I to the parser; calling a nonfragment rule
99 		// from a nonfragment rule does not set the overall token.
100 		String grammar =
101 			"grammar P;\n"+
102 			"a : A {System.out.println(input);} ;\n"+
103 			"A : '-' I ;\n" +
104 			"fragment I : '0'..'9'+ ;\n"+
105 			"WS : (' '|'\\n') {$channel=HIDDEN;} ;";
106 		String found = execParser("P.g", grammar, "PParser", "PLexer",
107 				    "a", "-34", debug);
108 		assertEquals("-34\n", found);
109 	}
110 
testMultipleRefToFragment()111 	@Test public void testMultipleRefToFragment() throws Exception {
112 		// this must return A not I to the parser; calling a nonfragment rule
113 		// from a nonfragment rule does not set the overall token.
114 		String grammar =
115 			"grammar P;\n"+
116 			"a : A EOF {System.out.println(input);} ;\n"+
117 			"A : I '.' I ;\n" +
118 			"fragment I : '0'..'9'+ ;\n"+
119 			"WS : (' '|'\\n') {$channel=HIDDEN;} ;";
120 		String found = execParser("P.g", grammar, "PParser", "PLexer",
121 				    "a", "3.14159", debug);
122 		assertEquals("3.14159\n", found);
123 	}
124 
testLabelInSubrule()125 	@Test public void testLabelInSubrule() throws Exception {
126 		// can we see v outside?
127 		String grammar =
128 			"grammar P;\n"+
129 			"a : A EOF ;\n"+
130 			"A : 'hi' WS (v=I)? {$channel=0; System.out.println($v.text);} ;\n" +
131 			"fragment I : '0'..'9'+ ;\n"+
132 			"WS : (' '|'\\n') {$channel=HIDDEN;} ;";
133 		String found = execParser("P.g", grammar, "PParser", "PLexer",
134 				    "a", "hi 342", debug);
135 		assertEquals("342\n", found);
136 	}
137 
testRefToTokenInLexer()138 	@Test public void testRefToTokenInLexer() throws Exception {
139 		String grammar =
140 			"grammar P;\n"+
141 			"a : A EOF ;\n"+
142 			"A : I {System.out.println($I.text);} ;\n" +
143 			"fragment I : '0'..'9'+ ;\n"+
144 			"WS : (' '|'\\n') {$channel=HIDDEN;} ;";
145 		String found = execParser("P.g", grammar, "PParser", "PLexer",
146 				    "a", "342", debug);
147 		assertEquals("342\n", found);
148 	}
149 
testListLabelInLexer()150 	@Test public void testListLabelInLexer() throws Exception {
151 		String grammar =
152 			"grammar P;\n"+
153 			"a : A ;\n"+
154 			"A : i+=I+ {for (Object t : $i) System.out.print(\" \"+((Token)t).getText());} ;\n" +
155 			"fragment I : '0'..'9'+ ;\n"+
156 			"WS : (' '|'\\n') {$channel=HIDDEN;} ;";
157 		String found = execParser("P.g", grammar, "PParser", "PLexer",
158 				    "a", "33 297", debug);
159 		assertEquals(" 33 297\n", found);
160 	}
161 
testDupListRefInLexer()162 	@Test public void testDupListRefInLexer() throws Exception {
163 		String grammar =
164 			"grammar P;\n"+
165 			"a : A ;\n"+
166 			"A : i+=I WS i+=I {$channel=0; for (Object t : $i) System.out.print(\" \"+((Token)t).getText());} ;\n" +
167 			"fragment I : '0'..'9'+ ;\n"+
168 			"WS : (' '|'\\n') {$channel=HIDDEN;} ;";
169 		String found = execParser("P.g", grammar, "PParser", "PLexer",
170 				    "a", "33 297", debug);
171 		assertEquals(" 33 297\n", found);
172 	}
173 
testCharLabelInLexer()174 	@Test public void testCharLabelInLexer() {
175 		String grammar =
176 			"grammar T;\n" +
177 			"a : B ;\n" +
178 			"B : x='a' {System.out.println((char)$x);} ;\n" ;
179 		String found = execParser("T.g", grammar, "TParser", "TLexer",
180 								  "a", "a", debug);
181 		assertEquals("a\n", found);
182 	}
183 
testRepeatedLabelInLexer()184 	@Test public void testRepeatedLabelInLexer() {
185 		String grammar =
186 			"lexer grammar T;\n" +
187 			"B : x='a' x='b' ;\n" ;
188 		boolean found =
189 			rawGenerateAndBuildRecognizer(
190 				"T.g", grammar, null, "T", false);
191 		boolean expecting = true; // should be ok
192 		assertEquals(expecting, found);
193 	}
194 
testRepeatedRuleLabelInLexer()195 	@Test public void testRepeatedRuleLabelInLexer() {
196 		String grammar =
197 			"lexer grammar T;\n" +
198 			"B : x=A x=A ;\n" +
199 			"fragment A : 'a' ;\n" ;
200 		boolean found =
201 			rawGenerateAndBuildRecognizer(
202 				"T.g", grammar, null, "T", false);
203 		boolean expecting = true; // should be ok
204 		assertEquals(expecting, found);
205 	}
206 
testIsolatedEOTEdge()207 	@Test public void testIsolatedEOTEdge() {
208 		String grammar =
209 			"lexer grammar T;\n" +
210 			"QUOTED_CONTENT \n" +
211 			"        : 'q' (~'q')* (('x' 'q') )* 'q' ; \n";
212 		boolean found =
213 			rawGenerateAndBuildRecognizer(
214 				"T.g", grammar, null, "T", false);
215 		boolean expecting = true; // should be ok
216 		assertEquals(expecting, found);
217 	}
218 
testEscapedLiterals()219 	@Test public void testEscapedLiterals() {
220 		/* Grammar:
221 			A : '\"' ;  should match a single double-quote: "
222 			B : '\\\"' ; should match input \"
223 		*/
224 		String grammar =
225 			"lexer grammar T;\n" +
226 			"A : '\\\"' ;\n" +
227 			"B : '\\\\\\\"' ;\n" ; // '\\\"'
228 		boolean found =
229 			rawGenerateAndBuildRecognizer(
230 				"T.g", grammar, null, "T", false);
231 		boolean expecting = true; // should be ok
232 		assertEquals(expecting, found);
233 	}
234 
testNewlineLiterals()235     @Test public void testNewlineLiterals() throws Exception {
236         Grammar g = new Grammar(
237             "lexer grammar T;\n" +
238             "A : '\\n\\n' ;\n"  // ANTLR sees '\n\n'
239         );
240         String expecting = "match(\"\\n\\n\")";
241 
242         Tool antlr = newTool();
243         antlr.setOutputDirectory(null); // write to /dev/null
244         CodeGenerator generator = new CodeGenerator(antlr, g, "Java");
245         g.setCodeGenerator(generator);
246         generator.genRecognizer(); // codegen phase sets some vars we need
247         ST codeST = generator.getRecognizerST();
248         String code = codeST.render();
249         int m = code.indexOf("match(\"");
250         String found = code.substring(m,m+expecting.length());
251 
252         assertEquals(expecting, found);
253     }
254 }
255