1 /*
2  *  Licensed to the Apache Software Foundation (ASF) under one or more
3  *  contributor license agreements.  See the NOTICE file distributed with
4  *  this work for additional information regarding copyright ownership.
5  *  The ASF licenses this file to You under the Apache License, Version 2.0
6  *  (the "License"); you may not use this file except in compliance with
7  *  the License.  You may obtain a copy of the License at
8  *
9  *     http://www.apache.org/licenses/LICENSE-2.0
10  *
11  *  Unless required by applicable law or agreed to in writing, software
12  *  distributed under the License is distributed on an "AS IS" BASIS,
13  *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14  *  See the License for the specific language governing permissions and
15  *  limitations under the License.
16  */
17 
18 package org.apache.harmony.tests.java.io;
19 
20 import java.io.ByteArrayInputStream;
21 import java.io.CharArrayReader;
22 import java.io.IOException;
23 import java.io.PipedInputStream;
24 import java.io.PipedOutputStream;
25 import java.io.Reader;
26 import java.io.StreamTokenizer;
27 import java.io.StringBufferInputStream;
28 
29 import tests.support.Support_StringReader;
30 
31 public class StreamTokenizerTest extends junit.framework.TestCase {
32     Support_StringReader r;
33 
34     StreamTokenizer st;
35 
36     String testString;
37 
38     /**
39      * java.io.StreamTokenizer#StreamTokenizer(java.io.InputStream)
40      */
41     @SuppressWarnings("deprecation")
test_ConstructorLjava_io_InputStream()42     public void test_ConstructorLjava_io_InputStream() throws IOException {
43         st = new StreamTokenizer(new StringBufferInputStream(
44                 "/comments\n d 8 'h'"));
45 
46         assertEquals("the next token returned should be the letter d",
47                 StreamTokenizer.TT_WORD, st.nextToken());
48         assertEquals("the next token returned should be the letter d",
49                 "d", st.sval);
50 
51         assertEquals("the next token returned should be the digit 8",
52                 StreamTokenizer.TT_NUMBER, st.nextToken());
53         assertEquals("the next token returned should be the digit 8",
54                 8.0, st.nval);
55 
56         assertEquals("the next token returned should be the quote character",
57                 39, st.nextToken());
58         assertEquals("the next token returned should be the quote character",
59                 "h", st.sval);
60     }
61 
62     /**
63      * java.io.StreamTokenizer#StreamTokenizer(java.io.Reader)
64      */
test_ConstructorLjava_io_Reader()65     public void test_ConstructorLjava_io_Reader() throws IOException {
66         setTest("/testing\n d 8 'h' ");
67         assertEquals("the next token returned should be the letter d skipping the comments",
68                 StreamTokenizer.TT_WORD, st.nextToken());
69         assertEquals("the next token returned should be the letter d",
70                 "d", st.sval);
71 
72         assertEquals("the next token returned should be the digit 8",
73                 StreamTokenizer.TT_NUMBER, st.nextToken());
74         assertEquals("the next token returned should be the digit 8",
75                 8.0, st.nval);
76 
77         assertEquals("the next token returned should be the quote character",
78                 39, st.nextToken());
79         assertEquals("the next token returned should be the quote character",
80                 "h", st.sval);
81     }
82 
83     /**
84      * java.io.StreamTokenizer#commentChar(int)
85      */
test_commentCharI()86     public void test_commentCharI() throws IOException {
87         setTest("*comment \n / 8 'h' ");
88         st.ordinaryChar('/');
89         st.commentChar('*');
90         assertEquals("nextToken() did not return the character / skiping the comments starting with *",
91                 47, st.nextToken());
92         assertTrue("the next token returned should be the digit 8", st
93                 .nextToken() == StreamTokenizer.TT_NUMBER
94                 && st.nval == 8.0);
95         assertTrue("the next token returned should be the quote character",
96                 st.nextToken() == 39 && st.sval.equals("h"));
97     }
98 
99     /**
100      * java.io.StreamTokenizer#eolIsSignificant(boolean)
101      */
test_eolIsSignificantZ()102     public void test_eolIsSignificantZ() throws IOException {
103         setTest("d 8\n");
104         // by default end of line characters are not significant
105         assertTrue("nextToken did not return d",
106                 st.nextToken() == StreamTokenizer.TT_WORD
107                         && st.sval.equals("d"));
108         assertTrue("nextToken did not return 8",
109                 st.nextToken() == StreamTokenizer.TT_NUMBER
110                         && st.nval == 8.0);
111         assertTrue("nextToken should be the end of file",
112                 st.nextToken() == StreamTokenizer.TT_EOF);
113         setTest("d\n");
114         st.eolIsSignificant(true);
115         // end of line characters are significant
116         assertTrue("nextToken did not return d",
117                 st.nextToken() == StreamTokenizer.TT_WORD
118                         && st.sval.equals("d"));
119         assertTrue("nextToken is the end of line",
120                 st.nextToken() == StreamTokenizer.TT_EOL);
121     }
122 
123     /**
124      * java.io.StreamTokenizer#lineno()
125      */
test_lineno()126     public void test_lineno() throws IOException {
127         setTest("d\n 8\n");
128         assertEquals("the lineno should be 1", 1, st.lineno());
129         st.nextToken();
130         st.nextToken();
131         assertEquals("the lineno should be 2", 2, st.lineno());
132         st.nextToken();
133         assertEquals("the next line no should be 3", 3, st.lineno());
134     }
135 
136     /**
137      * java.io.StreamTokenizer#lowerCaseMode(boolean)
138      */
test_lowerCaseModeZ()139     public void test_lowerCaseModeZ() throws Exception {
140         // SM.
141         setTest("HELLOWORLD");
142         st.lowerCaseMode(true);
143 
144         st.nextToken();
145         assertEquals("sval not converted to lowercase.", "helloworld", st.sval
146         );
147     }
148 
149     /**
150      * java.io.StreamTokenizer#nextToken()
151      */
152     @SuppressWarnings("deprecation")
test_nextToken()153     public void test_nextToken() throws IOException {
154         // SM.
155         setTest("\r\n/* fje fje 43.4 f \r\n f g */  456.459 \r\n"
156                 + "Hello  / 	\r\n \r\n \n \r \257 Hi \'Hello World\'");
157         st.ordinaryChar('/');
158         st.slashStarComments(true);
159         st.nextToken();
160         assertTrue("Wrong Token type1: " + (char) st.ttype,
161                 st.ttype == StreamTokenizer.TT_NUMBER);
162         st.nextToken();
163         assertTrue("Wrong Token type2: " + st.ttype,
164                 st.ttype == StreamTokenizer.TT_WORD);
165         st.nextToken();
166         assertTrue("Wrong Token type3: " + st.ttype, st.ttype == '/');
167         st.nextToken();
168         assertTrue("Wrong Token type4: " + st.ttype,
169                 st.ttype == StreamTokenizer.TT_WORD);
170         st.nextToken();
171         assertTrue("Wrong Token type5: " + st.ttype,
172                 st.ttype == StreamTokenizer.TT_WORD);
173         st.nextToken();
174         assertTrue("Wrong Token type6: " + st.ttype, st.ttype == '\'');
175         assertTrue("Wrong Token type7: " + st.ttype, st.sval
176                 .equals("Hello World"));
177         st.nextToken();
178         assertTrue("Wrong Token type8: " + st.ttype, st.ttype == -1);
179 
180         final PipedInputStream pin = new PipedInputStream();
181         PipedOutputStream pout = new PipedOutputStream(pin);
182         pout.write("hello\n\r\r".getBytes("UTF-8"));
183         StreamTokenizer s = new StreamTokenizer(pin);
184         s.eolIsSignificant(true);
185         assertTrue("Wrong token 1,1",
186                 s.nextToken() == StreamTokenizer.TT_WORD
187                         && s.sval.equals("hello"));
188         assertTrue("Wrong token 1,2", s.nextToken() == '\n');
189         assertTrue("Wrong token 1,3", s.nextToken() == '\n');
190         assertTrue("Wrong token 1,4", s.nextToken() == '\n');
191         pout.close();
192         assertTrue("Wrong token 1,5",
193                 s.nextToken() == StreamTokenizer.TT_EOF);
194         StreamTokenizer tokenizer = new StreamTokenizer(
195                 new Support_StringReader("\n \r\n#"));
196         tokenizer.ordinaryChar('\n'); // make \n ordinary
197         tokenizer.eolIsSignificant(true);
198         assertTrue("Wrong token 2,1", tokenizer.nextToken() == '\n');
199         assertTrue("Wrong token 2,2", tokenizer.nextToken() == '\n');
200         assertEquals("Wrong token 2,3", '#', tokenizer.nextToken());
201     }
202 
203     /**
204      * java.io.StreamTokenizer#ordinaryChar(int)
205      */
test_ordinaryCharI()206     public void test_ordinaryCharI() throws IOException {
207         // SM.
208         setTest("Ffjein 893");
209         st.ordinaryChar('F');
210         st.nextToken();
211         assertTrue("OrdinaryChar failed." + (char) st.ttype,
212                 st.ttype == 'F');
213     }
214 
215     /**
216      * java.io.StreamTokenizer#ordinaryChars(int, int)
217      */
test_ordinaryCharsII()218     public void test_ordinaryCharsII() throws IOException {
219         // SM.
220         setTest("azbc iof z 893");
221         st.ordinaryChars('a', 'z');
222         assertEquals("OrdinaryChars failed.", 'a', st.nextToken());
223         assertEquals("OrdinaryChars failed.", 'z', st.nextToken());
224     }
225 
226     /**
227      * java.io.StreamTokenizer#parseNumbers()
228      */
test_parseNumbers()229     public void test_parseNumbers() throws IOException {
230         // SM
231         setTest("9.9 678");
232         assertTrue("Base behavior failed.",
233                 st.nextToken() == StreamTokenizer.TT_NUMBER);
234         st.ordinaryChars('0', '9');
235         assertEquals("setOrdinary failed.", '6', st.nextToken());
236         st.parseNumbers();
237         assertTrue("parseNumbers failed.",
238                 st.nextToken() == StreamTokenizer.TT_NUMBER);
239     }
240 
241     /**
242      * java.io.StreamTokenizer#pushBack()
243      */
test_pushBack()244     public void test_pushBack() throws IOException {
245         // SM.
246         setTest("Hello 897");
247         st.nextToken();
248         st.pushBack();
249         assertTrue("PushBack failed.",
250                 st.nextToken() == StreamTokenizer.TT_WORD);
251     }
252 
253     /**
254      * java.io.StreamTokenizer#quoteChar(int)
255      */
test_quoteCharI()256     public void test_quoteCharI() throws IOException {
257         // SM
258         setTest("<Hello World<    HelloWorldH");
259         st.quoteChar('<');
260         assertEquals("QuoteChar failed.", '<', st.nextToken());
261         assertEquals("QuoteChar failed.", "Hello World", st.sval);
262         st.quoteChar('H');
263         st.nextToken();
264         assertEquals("QuoteChar failed for word.", "elloWorld", st.sval
265         );
266     }
267 
268     /**
269      * java.io.StreamTokenizer#resetSyntax()
270      */
test_resetSyntax()271     public void test_resetSyntax() throws IOException {
272         // SM
273         setTest("H 9\' ello World");
274         st.resetSyntax();
275         assertTrue("resetSyntax failed1." + (char) st.ttype,
276                 st.nextToken() == 'H');
277         assertTrue("resetSyntax failed1." + (char) st.ttype,
278                 st.nextToken() == ' ');
279         assertTrue("resetSyntax failed2." + (char) st.ttype,
280                 st.nextToken() == '9');
281         assertTrue("resetSyntax failed3." + (char) st.ttype,
282                 st.nextToken() == '\'');
283     }
284 
285     /**
286      * java.io.StreamTokenizer#slashSlashComments(boolean)
287      */
test_slashSlashCommentsZ()288     public void test_slashSlashCommentsZ() throws IOException {
289         // SM.
290         setTest("// foo \r\n /fiji \r\n -456");
291         st.ordinaryChar('/');
292         st.slashSlashComments(true);
293         assertEquals("Test failed.", '/', st.nextToken());
294         assertTrue("Test failed.",
295                 st.nextToken() == StreamTokenizer.TT_WORD);
296     }
297 
298     /**
299      * java.io.StreamTokenizer#slashSlashComments(boolean)
300      */
test_slashSlashComments_withSSOpen()301     public void test_slashSlashComments_withSSOpen() throws IOException {
302         Reader reader = new CharArrayReader("t // t t t".toCharArray());
303 
304         StreamTokenizer st = new StreamTokenizer(reader);
305         st.slashSlashComments(true);
306 
307         assertEquals(StreamTokenizer.TT_WORD, st.nextToken());
308         assertEquals(StreamTokenizer.TT_EOF, st.nextToken());
309     }
310 
311     /**
312      * java.io.StreamTokenizer#slashSlashComments(boolean)
313      */
test_slashSlashComments_withSSOpen_NoComment()314     public void test_slashSlashComments_withSSOpen_NoComment() throws IOException {
315         Reader reader = new CharArrayReader("// t".toCharArray());
316 
317         StreamTokenizer st = new StreamTokenizer(reader);
318         st.slashSlashComments(true);
319         st.ordinaryChar('/');
320 
321         assertEquals(StreamTokenizer.TT_EOF, st.nextToken());
322     }
323 
324     /**
325      * java.io.StreamTokenizer#slashSlashComments(boolean)
326      */
test_slashSlashComments_withSSClosed()327     public void test_slashSlashComments_withSSClosed() throws IOException {
328         Reader reader = new CharArrayReader("// t".toCharArray());
329 
330         StreamTokenizer st = new StreamTokenizer(reader);
331         st.slashSlashComments(false);
332         st.ordinaryChar('/');
333 
334         assertEquals('/', st.nextToken());
335         assertEquals('/', st.nextToken());
336         assertEquals(StreamTokenizer.TT_WORD, st.nextToken());
337     }
338 
339     /**
340      * java.io.StreamTokenizer#slashStarComments(boolean)
341      */
test_slashStarCommentsZ()342     public void test_slashStarCommentsZ() throws IOException {
343         setTest("/* foo \r\n /fiji \r\n*/ -456");
344         st.ordinaryChar('/');
345         st.slashStarComments(true);
346         assertTrue("Test failed.",
347                 st.nextToken() == StreamTokenizer.TT_NUMBER);
348     }
349 
350     /**
351      * java.io.StreamTokenizer#slashStarComments(boolean)
352      */
test_slashStarComments_withSTOpen()353     public void test_slashStarComments_withSTOpen() throws IOException {
354         Reader reader = new CharArrayReader("t /* t */ t".toCharArray());
355 
356         StreamTokenizer st = new StreamTokenizer(reader);
357         st.slashStarComments(true);
358 
359         assertEquals(StreamTokenizer.TT_WORD, st.nextToken());
360         assertEquals(StreamTokenizer.TT_WORD, st.nextToken());
361         assertEquals(StreamTokenizer.TT_EOF, st.nextToken());
362     }
363 
364     /**
365      * java.io.StreamTokenizer#slashStarComments(boolean)
366      */
test_slashStarComments_withSTClosed()367     public void test_slashStarComments_withSTClosed() throws IOException {
368         Reader reader = new CharArrayReader("t /* t */ t".toCharArray());
369 
370         StreamTokenizer st = new StreamTokenizer(reader);
371         st.slashStarComments(false);
372 
373         assertEquals(StreamTokenizer.TT_WORD, st.nextToken());
374         assertEquals(StreamTokenizer.TT_EOF, st.nextToken());
375     }
376 
377     /**
378      * java.io.StreamTokenizer#toString()
379      */
test_toString()380     public void test_toString() throws IOException {
381         setTest("ABC Hello World");
382         st.nextToken();
383         assertTrue("toString failed." + st.toString(),
384                 st.toString().equals(
385                         "Token[ABC], line 1"));
386 
387         // Regression test for HARMONY-4070
388         byte[] data = new byte[] { (byte) '-' };
389         StreamTokenizer tokenizer = new StreamTokenizer(
390                 new ByteArrayInputStream(data));
391         tokenizer.nextToken();
392         String result = tokenizer.toString();
393         assertEquals("Token['-'], line 1", result);
394     }
395 
396     /**
397      * java.io.StreamTokenizer#whitespaceChars(int, int)
398      */
test_whitespaceCharsII()399     public void test_whitespaceCharsII() throws IOException {
400         setTest("azbc iof z 893");
401         st.whitespaceChars('a', 'z');
402         assertTrue("OrdinaryChar failed.",
403                 st.nextToken() == StreamTokenizer.TT_NUMBER);
404     }
405 
406     /**
407      * java.io.StreamTokenizer#wordChars(int, int)
408      */
test_wordCharsII()409     public void test_wordCharsII() throws IOException {
410         setTest("A893 -9B87");
411         st.wordChars('0', '9');
412         assertTrue("WordChar failed1.",
413                 st.nextToken() == StreamTokenizer.TT_WORD);
414         assertEquals("WordChar failed2.", "A893", st.sval);
415         assertTrue("WordChar failed3.",
416                 st.nextToken() == StreamTokenizer.TT_NUMBER);
417         st.nextToken();
418         assertEquals("WordChar failed4.", "B87", st.sval);
419 
420         setTest("    Hello World");
421         st.wordChars(' ', ' ');
422         st.nextToken();
423         assertEquals("WordChars failed for whitespace.", "Hello World", st.sval
424         );
425 
426         setTest("    Hello World\r\n  \'Hello World\' Hello\' World");
427         st.wordChars(' ', ' ');
428         st.wordChars('\'', '\'');
429         st.nextToken();
430         assertTrue("WordChars failed for whitespace: " + st.sval, st.sval
431                 .equals("Hello World"));
432         st.nextToken();
433         assertTrue("WordChars failed for quote1: " + st.sval, st.sval
434                 .equals("\'Hello World\' Hello\' World"));
435     }
436 
setTest(String s)437     private void setTest(String s) {
438         testString = s;
439         r = new Support_StringReader(testString);
440         st = new StreamTokenizer(r);
441     }
442 
setUp()443     protected void setUp() {
444     }
445 
tearDown()446     protected void tearDown() {
447     }
448 }
449