1 /*
2  * [The "BSD license"]
3  *  Copyright (c) 2010 Terence Parr
4  *  All rights reserved.
5  *
6  *  Redistribution and use in source and binary forms, with or without
7  *  modification, are permitted provided that the following conditions
8  *  are met:
9  *  1. Redistributions of source code must retain the above copyright
10  *      notice, this list of conditions and the following disclaimer.
11  *  2. Redistributions in binary form must reproduce the above copyright
12  *      notice, this list of conditions and the following disclaimer in the
13  *      documentation and/or other materials provided with the distribution.
14  *  3. The name of the author may not be used to endorse or promote products
15  *      derived from this software without specific prior written permission.
16  *
17  *  THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
18  *  IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
19  *  OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
20  *  IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
21  *  INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
22  *  NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23  *  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24  *  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25  *  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
26  *  THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27  */
28 
29 package org.antlr.test;
30 
31 import org.antlr.runtime.*;
32 import org.antlr.tool.Grammar;
33 import org.antlr.tool.Interpreter;
34 import org.junit.Test;
35 
36 /** This actually tests new (12/4/09) buffered but on-demand fetching stream */
37 public class TestCommonTokenStream extends BaseTest {
testFirstToken()38     @Test public void testFirstToken() throws Exception {
39         Grammar g = new Grammar(
40             "lexer grammar t;\n"+
41             "ID : 'a'..'z'+;\n" +
42             "INT : '0'..'9'+;\n" +
43             "SEMI : ';';\n" +
44             "ASSIGN : '=';\n" +
45             "PLUS : '+';\n" +
46             "MULT : '*';\n" +
47             "WS : ' '+;\n");
48         // Tokens: 012345678901234567
49         // Input:  x = 3 * 0 + 2 * 0;
50         CharStream input = new ANTLRStringStream("x = 3 * 0 + 2 * 0;");
51         Interpreter lexEngine = new Interpreter(g, input);
52         BufferedTokenStream tokens = new BufferedTokenStream(lexEngine);
53 
54         String result = tokens.LT(1).getText();
55         String expecting = "x";
56         assertEquals(expecting, result);
57     }
58 
test2ndToken()59     @Test public void test2ndToken() throws Exception {
60         Grammar g = new Grammar(
61             "lexer grammar t;\n"+
62             "ID : 'a'..'z'+;\n" +
63             "INT : '0'..'9'+;\n" +
64             "SEMI : ';';\n" +
65             "ASSIGN : '=';\n" +
66             "PLUS : '+';\n" +
67             "MULT : '*';\n" +
68             "WS : ' '+;\n");
69         // Tokens: 012345678901234567
70         // Input:  x = 3 * 0 + 2 * 0;
71         CharStream input = new ANTLRStringStream("x = 3 * 0 + 2 * 0;");
72         Interpreter lexEngine = new Interpreter(g, input);
73         BufferedTokenStream tokens = new BufferedTokenStream(lexEngine);
74 
75         String result = tokens.LT(2).getText();
76         String expecting = " ";
77         assertEquals(expecting, result);
78     }
79 
testCompleteBuffer()80     @Test public void testCompleteBuffer() throws Exception {
81         Grammar g = new Grammar(
82             "lexer grammar t;\n"+
83             "ID : 'a'..'z'+;\n" +
84             "INT : '0'..'9'+;\n" +
85             "SEMI : ';';\n" +
86             "ASSIGN : '=';\n" +
87             "PLUS : '+';\n" +
88             "MULT : '*';\n" +
89             "WS : ' '+;\n");
90         // Tokens: 012345678901234567
91         // Input:  x = 3 * 0 + 2 * 0;
92         CharStream input = new ANTLRStringStream("x = 3 * 0 + 2 * 0;");
93         Interpreter lexEngine = new Interpreter(g, input);
94         BufferedTokenStream tokens = new BufferedTokenStream(lexEngine);
95 
96         int i = 1;
97         Token t = tokens.LT(i);
98         while ( t.getType()!=Token.EOF ) {
99             i++;
100             t = tokens.LT(i);
101         }
102         tokens.LT(i++); // push it past end
103         tokens.LT(i++);
104 
105         String result = tokens.toString();
106         String expecting = "x = 3 * 0 + 2 * 0;";
107         assertEquals(expecting, result);
108     }
109 
testCompleteBufferAfterConsuming()110     @Test public void testCompleteBufferAfterConsuming() throws Exception {
111         Grammar g = new Grammar(
112             "lexer grammar t;\n"+
113             "ID : 'a'..'z'+;\n" +
114             "INT : '0'..'9'+;\n" +
115             "SEMI : ';';\n" +
116             "ASSIGN : '=';\n" +
117             "PLUS : '+';\n" +
118             "MULT : '*';\n" +
119             "WS : ' '+;\n");
120         // Tokens: 012345678901234567
121         // Input:  x = 3 * 0 + 2 * 0;
122         CharStream input = new ANTLRStringStream("x = 3 * 0 + 2 * 0;");
123         Interpreter lexEngine = new Interpreter(g, input);
124         BufferedTokenStream tokens = new BufferedTokenStream(lexEngine);
125 
126         Token t = tokens.LT(1);
127         while ( t.getType()!=Token.EOF ) {
128             tokens.consume();
129             t = tokens.LT(1);
130         }
131         tokens.consume();
132         tokens.LT(1); // push it past end
133         tokens.consume();
134         tokens.LT(1);
135 
136         String result = tokens.toString();
137         String expecting = "x = 3 * 0 + 2 * 0;";
138         assertEquals(expecting, result);
139     }
140 
testLookback()141     @Test public void testLookback() throws Exception {
142         Grammar g = new Grammar(
143             "lexer grammar t;\n"+
144             "ID : 'a'..'z'+;\n" +
145             "INT : '0'..'9'+;\n" +
146             "SEMI : ';';\n" +
147             "ASSIGN : '=';\n" +
148             "PLUS : '+';\n" +
149             "MULT : '*';\n" +
150             "WS : ' '+;\n");
151         // Tokens: 012345678901234567
152         // Input:  x = 3 * 0 + 2 * 0;
153         CharStream input = new ANTLRStringStream("x = 3 * 0 + 2 * 0;");
154         Interpreter lexEngine = new Interpreter(g, input);
155         BufferedTokenStream tokens = new BufferedTokenStream(lexEngine);
156 
157         tokens.consume(); // get x into buffer
158         Token t = tokens.LT(-1);
159         assertEquals("x", t.getText());
160 
161         tokens.consume();
162         tokens.consume(); // consume '='
163         t = tokens.LT(-3);
164         assertEquals("x", t.getText());
165         t = tokens.LT(-2);
166         assertEquals(" ", t.getText());
167         t = tokens.LT(-1);
168         assertEquals("=", t.getText());
169     }
170 
testOffChannel()171     @Test public void testOffChannel() throws Exception {
172         TokenSource lexer = // simulate input " x =34  ;\n"
173             new TokenSource() {
174                 int i = 0;
175                 Token[] tokens = {
176                     new CommonToken(1," "),
177                     new CommonToken(1,"x"),
178                     new CommonToken(1," "),
179                     new CommonToken(1,"="),
180                     new CommonToken(1,"34"),
181                     new CommonToken(1," "),
182                     new CommonToken(1," "),
183                     new CommonToken(1,";"),
184                     new CommonToken(1,"\n"),
185                     new CommonToken(Token.EOF,"")
186                 };
187                 {
188                     tokens[0].setChannel(Lexer.HIDDEN);
189                     tokens[2].setChannel(Lexer.HIDDEN);
190                     tokens[5].setChannel(Lexer.HIDDEN);
191                     tokens[6].setChannel(Lexer.HIDDEN);
192                     tokens[8].setChannel(Lexer.HIDDEN);
193                 }
194                 public Token nextToken() {
195                     return tokens[i++];
196                 }
197                 public String getSourceName() { return "test"; }
198             };
199 
200         CommonTokenStream tokens = new CommonTokenStream(lexer);
201 
202         assertEquals("x", tokens.LT(1).getText()); // must skip first off channel token
203         tokens.consume();
204         assertEquals("=", tokens.LT(1).getText());
205         assertEquals("x", tokens.LT(-1).getText());
206 
207         tokens.consume();
208         assertEquals("34", tokens.LT(1).getText());
209         assertEquals("=", tokens.LT(-1).getText());
210 
211         tokens.consume();
212         assertEquals(";", tokens.LT(1).getText());
213         assertEquals("34", tokens.LT(-1).getText());
214 
215         tokens.consume();
216         assertEquals(Token.EOF, tokens.LA(1));
217         assertEquals(";", tokens.LT(-1).getText());
218 
219         assertEquals("34", tokens.LT(-2).getText());
220         assertEquals("=", tokens.LT(-3).getText());
221         assertEquals("x", tokens.LT(-4).getText());
222     }
223 }