1/*
2 *  Licensed to the Apache Software Foundation (ASF) under one or more
3 *  contributor license agreements.  See the NOTICE file distributed with
4 *  this work for additional information regarding copyright ownership.
5 *  The ASF licenses this file to You under the Apache License, Version 2.0
6 *  (the "License"); you may not use this file except in compliance with
7 *  the License.  You may obtain a copy of the License at
8 *
9 *     http://www.apache.org/licenses/LICENSE-2.0
10 *
11 *  Unless required by applicable law or agreed to in writing, software
12 *  distributed under the License is distributed on an "AS IS" BASIS,
13 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 *  See the License for the specific language governing permissions and
15 *  limitations under the License.
16 */
17
18package org.apache.harmony.tests.java.io;
19
20import java.io.ByteArrayInputStream;
21import java.io.CharArrayReader;
22import java.io.IOException;
23import java.io.PipedInputStream;
24import java.io.PipedOutputStream;
25import java.io.Reader;
26import java.io.StreamTokenizer;
27import java.io.StringBufferInputStream;
28
29import tests.support.Support_StringReader;
30
31public class StreamTokenizerTest extends junit.framework.TestCase {
32    Support_StringReader r;
33
34    StreamTokenizer st;
35
36    String testString;
37
38    /**
39     * java.io.StreamTokenizer#StreamTokenizer(java.io.InputStream)
40     */
41    @SuppressWarnings("deprecation")
42    public void test_ConstructorLjava_io_InputStream() throws IOException {
43        st = new StreamTokenizer(new StringBufferInputStream(
44                "/comments\n d 8 'h'"));
45
46        assertEquals("the next token returned should be the letter d",
47                StreamTokenizer.TT_WORD, st.nextToken());
48        assertEquals("the next token returned should be the letter d",
49                "d", st.sval);
50
51        assertEquals("the next token returned should be the digit 8",
52                StreamTokenizer.TT_NUMBER, st.nextToken());
53        assertEquals("the next token returned should be the digit 8",
54                8.0, st.nval);
55
56        assertEquals("the next token returned should be the quote character",
57                39, st.nextToken());
58        assertEquals("the next token returned should be the quote character",
59                "h", st.sval);
60    }
61
62    /**
63     * java.io.StreamTokenizer#StreamTokenizer(java.io.Reader)
64     */
65    public void test_ConstructorLjava_io_Reader() throws IOException {
66        setTest("/testing\n d 8 'h' ");
67        assertEquals("the next token returned should be the letter d skipping the comments",
68                StreamTokenizer.TT_WORD, st.nextToken());
69        assertEquals("the next token returned should be the letter d",
70                "d", st.sval);
71
72        assertEquals("the next token returned should be the digit 8",
73                StreamTokenizer.TT_NUMBER, st.nextToken());
74        assertEquals("the next token returned should be the digit 8",
75                8.0, st.nval);
76
77        assertEquals("the next token returned should be the quote character",
78                39, st.nextToken());
79        assertEquals("the next token returned should be the quote character",
80                "h", st.sval);
81    }
82
83    /**
84     * java.io.StreamTokenizer#commentChar(int)
85     */
86    public void test_commentCharI() throws IOException {
87        setTest("*comment \n / 8 'h' ");
88        st.ordinaryChar('/');
89        st.commentChar('*');
90        assertEquals("nextToken() did not return the character / skiping the comments starting with *",
91                47, st.nextToken());
92        assertTrue("the next token returned should be the digit 8", st
93                .nextToken() == StreamTokenizer.TT_NUMBER
94                && st.nval == 8.0);
95        assertTrue("the next token returned should be the quote character",
96                st.nextToken() == 39 && st.sval.equals("h"));
97    }
98
99    /**
100     * java.io.StreamTokenizer#eolIsSignificant(boolean)
101     */
102    public void test_eolIsSignificantZ() throws IOException {
103        setTest("d 8\n");
104        // by default end of line characters are not significant
105        assertTrue("nextToken did not return d",
106                st.nextToken() == StreamTokenizer.TT_WORD
107                        && st.sval.equals("d"));
108        assertTrue("nextToken did not return 8",
109                st.nextToken() == StreamTokenizer.TT_NUMBER
110                        && st.nval == 8.0);
111        assertTrue("nextToken should be the end of file",
112                st.nextToken() == StreamTokenizer.TT_EOF);
113        setTest("d\n");
114        st.eolIsSignificant(true);
115        // end of line characters are significant
116        assertTrue("nextToken did not return d",
117                st.nextToken() == StreamTokenizer.TT_WORD
118                        && st.sval.equals("d"));
119        assertTrue("nextToken is the end of line",
120                st.nextToken() == StreamTokenizer.TT_EOL);
121    }
122
123    /**
124     * java.io.StreamTokenizer#lineno()
125     */
126    public void test_lineno() throws IOException {
127        setTest("d\n 8\n");
128        assertEquals("the lineno should be 1", 1, st.lineno());
129        st.nextToken();
130        st.nextToken();
131        assertEquals("the lineno should be 2", 2, st.lineno());
132        st.nextToken();
133        assertEquals("the next line no should be 3", 3, st.lineno());
134    }
135
136    /**
137     * java.io.StreamTokenizer#lowerCaseMode(boolean)
138     */
139    public void test_lowerCaseModeZ() throws Exception {
140        // SM.
141        setTest("HELLOWORLD");
142        st.lowerCaseMode(true);
143
144        st.nextToken();
145        assertEquals("sval not converted to lowercase.", "helloworld", st.sval
146        );
147    }
148
149    /**
150     * java.io.StreamTokenizer#nextToken()
151     */
152    @SuppressWarnings("deprecation")
153    public void test_nextToken() throws IOException {
154        // SM.
155        setTest("\r\n/* fje fje 43.4 f \r\n f g */  456.459 \r\n"
156                + "Hello  / 	\r\n \r\n \n \r \257 Hi \'Hello World\'");
157        st.ordinaryChar('/');
158        st.slashStarComments(true);
159        st.nextToken();
160        assertTrue("Wrong Token type1: " + (char) st.ttype,
161                st.ttype == StreamTokenizer.TT_NUMBER);
162        st.nextToken();
163        assertTrue("Wrong Token type2: " + st.ttype,
164                st.ttype == StreamTokenizer.TT_WORD);
165        st.nextToken();
166        assertTrue("Wrong Token type3: " + st.ttype, st.ttype == '/');
167        st.nextToken();
168        assertTrue("Wrong Token type4: " + st.ttype,
169                st.ttype == StreamTokenizer.TT_WORD);
170        st.nextToken();
171        assertTrue("Wrong Token type5: " + st.ttype,
172                st.ttype == StreamTokenizer.TT_WORD);
173        st.nextToken();
174        assertTrue("Wrong Token type6: " + st.ttype, st.ttype == '\'');
175        assertTrue("Wrong Token type7: " + st.ttype, st.sval
176                .equals("Hello World"));
177        st.nextToken();
178        assertTrue("Wrong Token type8: " + st.ttype, st.ttype == -1);
179
180        final PipedInputStream pin = new PipedInputStream();
181        PipedOutputStream pout = new PipedOutputStream(pin);
182        pout.write("hello\n\r\r".getBytes("UTF-8"));
183        StreamTokenizer s = new StreamTokenizer(pin);
184        s.eolIsSignificant(true);
185        assertTrue("Wrong token 1,1",
186                s.nextToken() == StreamTokenizer.TT_WORD
187                        && s.sval.equals("hello"));
188        assertTrue("Wrong token 1,2", s.nextToken() == '\n');
189        assertTrue("Wrong token 1,3", s.nextToken() == '\n');
190        assertTrue("Wrong token 1,4", s.nextToken() == '\n');
191        pout.close();
192        assertTrue("Wrong token 1,5",
193                s.nextToken() == StreamTokenizer.TT_EOF);
194        StreamTokenizer tokenizer = new StreamTokenizer(
195                new Support_StringReader("\n \r\n#"));
196        tokenizer.ordinaryChar('\n'); // make \n ordinary
197        tokenizer.eolIsSignificant(true);
198        assertTrue("Wrong token 2,1", tokenizer.nextToken() == '\n');
199        assertTrue("Wrong token 2,2", tokenizer.nextToken() == '\n');
200        assertEquals("Wrong token 2,3", '#', tokenizer.nextToken());
201    }
202
203    /**
204     * java.io.StreamTokenizer#ordinaryChar(int)
205     */
206    public void test_ordinaryCharI() throws IOException {
207        // SM.
208        setTest("Ffjein 893");
209        st.ordinaryChar('F');
210        st.nextToken();
211        assertTrue("OrdinaryChar failed." + (char) st.ttype,
212                st.ttype == 'F');
213    }
214
215    /**
216     * java.io.StreamTokenizer#ordinaryChars(int, int)
217     */
218    public void test_ordinaryCharsII() throws IOException {
219        // SM.
220        setTest("azbc iof z 893");
221        st.ordinaryChars('a', 'z');
222        assertEquals("OrdinaryChars failed.", 'a', st.nextToken());
223        assertEquals("OrdinaryChars failed.", 'z', st.nextToken());
224    }
225
226    /**
227     * java.io.StreamTokenizer#parseNumbers()
228     */
229    public void test_parseNumbers() throws IOException {
230        // SM
231        setTest("9.9 678");
232        assertTrue("Base behavior failed.",
233                st.nextToken() == StreamTokenizer.TT_NUMBER);
234        st.ordinaryChars('0', '9');
235        assertEquals("setOrdinary failed.", '6', st.nextToken());
236        st.parseNumbers();
237        assertTrue("parseNumbers failed.",
238                st.nextToken() == StreamTokenizer.TT_NUMBER);
239    }
240
241    /**
242     * java.io.StreamTokenizer#pushBack()
243     */
244    public void test_pushBack() throws IOException {
245        // SM.
246        setTest("Hello 897");
247        st.nextToken();
248        st.pushBack();
249        assertTrue("PushBack failed.",
250                st.nextToken() == StreamTokenizer.TT_WORD);
251    }
252
253    /**
254     * java.io.StreamTokenizer#quoteChar(int)
255     */
256    public void test_quoteCharI() throws IOException {
257        // SM
258        setTest("<Hello World<    HelloWorldH");
259        st.quoteChar('<');
260        assertEquals("QuoteChar failed.", '<', st.nextToken());
261        assertEquals("QuoteChar failed.", "Hello World", st.sval);
262        st.quoteChar('H');
263        st.nextToken();
264        assertEquals("QuoteChar failed for word.", "elloWorld", st.sval
265        );
266    }
267
268    /**
269     * java.io.StreamTokenizer#resetSyntax()
270     */
271    public void test_resetSyntax() throws IOException {
272        // SM
273        setTest("H 9\' ello World");
274        st.resetSyntax();
275        assertTrue("resetSyntax failed1." + (char) st.ttype,
276                st.nextToken() == 'H');
277        assertTrue("resetSyntax failed1." + (char) st.ttype,
278                st.nextToken() == ' ');
279        assertTrue("resetSyntax failed2." + (char) st.ttype,
280                st.nextToken() == '9');
281        assertTrue("resetSyntax failed3." + (char) st.ttype,
282                st.nextToken() == '\'');
283    }
284
285    /**
286     * java.io.StreamTokenizer#slashSlashComments(boolean)
287     */
288    public void test_slashSlashCommentsZ() throws IOException {
289        // SM.
290        setTest("// foo \r\n /fiji \r\n -456");
291        st.ordinaryChar('/');
292        st.slashSlashComments(true);
293        assertEquals("Test failed.", '/', st.nextToken());
294        assertTrue("Test failed.",
295                st.nextToken() == StreamTokenizer.TT_WORD);
296    }
297
298    /**
299     * java.io.StreamTokenizer#slashSlashComments(boolean)
300     */
301    public void test_slashSlashComments_withSSOpen() throws IOException {
302        Reader reader = new CharArrayReader("t // t t t".toCharArray());
303
304        StreamTokenizer st = new StreamTokenizer(reader);
305        st.slashSlashComments(true);
306
307        assertEquals(StreamTokenizer.TT_WORD, st.nextToken());
308        assertEquals(StreamTokenizer.TT_EOF, st.nextToken());
309    }
310
311    /**
312     * java.io.StreamTokenizer#slashSlashComments(boolean)
313     */
314    public void test_slashSlashComments_withSSOpen_NoComment() throws IOException {
315        Reader reader = new CharArrayReader("// t".toCharArray());
316
317        StreamTokenizer st = new StreamTokenizer(reader);
318        st.slashSlashComments(true);
319        st.ordinaryChar('/');
320
321        assertEquals(StreamTokenizer.TT_EOF, st.nextToken());
322    }
323
324    /**
325     * java.io.StreamTokenizer#slashSlashComments(boolean)
326     */
327    public void test_slashSlashComments_withSSClosed() throws IOException {
328        Reader reader = new CharArrayReader("// t".toCharArray());
329
330        StreamTokenizer st = new StreamTokenizer(reader);
331        st.slashSlashComments(false);
332        st.ordinaryChar('/');
333
334        assertEquals('/', st.nextToken());
335        assertEquals('/', st.nextToken());
336        assertEquals(StreamTokenizer.TT_WORD, st.nextToken());
337    }
338
339    /**
340     * java.io.StreamTokenizer#slashStarComments(boolean)
341     */
342    public void test_slashStarCommentsZ() throws IOException {
343        setTest("/* foo \r\n /fiji \r\n*/ -456");
344        st.ordinaryChar('/');
345        st.slashStarComments(true);
346        assertTrue("Test failed.",
347                st.nextToken() == StreamTokenizer.TT_NUMBER);
348    }
349
350    /**
351     * java.io.StreamTokenizer#slashStarComments(boolean)
352     */
353    public void test_slashStarComments_withSTOpen() throws IOException {
354        Reader reader = new CharArrayReader("t /* t */ t".toCharArray());
355
356        StreamTokenizer st = new StreamTokenizer(reader);
357        st.slashStarComments(true);
358
359        assertEquals(StreamTokenizer.TT_WORD, st.nextToken());
360        assertEquals(StreamTokenizer.TT_WORD, st.nextToken());
361        assertEquals(StreamTokenizer.TT_EOF, st.nextToken());
362    }
363
364    /**
365     * java.io.StreamTokenizer#slashStarComments(boolean)
366     */
367    public void test_slashStarComments_withSTClosed() throws IOException {
368        Reader reader = new CharArrayReader("t /* t */ t".toCharArray());
369
370        StreamTokenizer st = new StreamTokenizer(reader);
371        st.slashStarComments(false);
372
373        assertEquals(StreamTokenizer.TT_WORD, st.nextToken());
374        assertEquals(StreamTokenizer.TT_EOF, st.nextToken());
375    }
376
377    /**
378     * java.io.StreamTokenizer#toString()
379     */
380    public void test_toString() throws IOException {
381        setTest("ABC Hello World");
382        st.nextToken();
383        assertTrue("toString failed." + st.toString(),
384                st.toString().equals(
385                        "Token[ABC], line 1"));
386
387        // Regression test for HARMONY-4070
388        byte[] data = new byte[] { (byte) '-' };
389        StreamTokenizer tokenizer = new StreamTokenizer(
390                new ByteArrayInputStream(data));
391        tokenizer.nextToken();
392        String result = tokenizer.toString();
393        assertEquals("Token['-'], line 1", result);
394    }
395
396    /**
397     * java.io.StreamTokenizer#whitespaceChars(int, int)
398     */
399    public void test_whitespaceCharsII() throws IOException {
400        setTest("azbc iof z 893");
401        st.whitespaceChars('a', 'z');
402        assertTrue("OrdinaryChar failed.",
403                st.nextToken() == StreamTokenizer.TT_NUMBER);
404    }
405
406    /**
407     * java.io.StreamTokenizer#wordChars(int, int)
408     */
409    public void test_wordCharsII() throws IOException {
410        setTest("A893 -9B87");
411        st.wordChars('0', '9');
412        assertTrue("WordChar failed1.",
413                st.nextToken() == StreamTokenizer.TT_WORD);
414        assertEquals("WordChar failed2.", "A893", st.sval);
415        assertTrue("WordChar failed3.",
416                st.nextToken() == StreamTokenizer.TT_NUMBER);
417        st.nextToken();
418        assertEquals("WordChar failed4.", "B87", st.sval);
419
420        setTest("    Hello World");
421        st.wordChars(' ', ' ');
422        st.nextToken();
423        assertEquals("WordChars failed for whitespace.", "Hello World", st.sval
424        );
425
426        setTest("    Hello World\r\n  \'Hello World\' Hello\' World");
427        st.wordChars(' ', ' ');
428        st.wordChars('\'', '\'');
429        st.nextToken();
430        assertTrue("WordChars failed for whitespace: " + st.sval, st.sval
431                .equals("Hello World"));
432        st.nextToken();
433        assertTrue("WordChars failed for quote1: " + st.sval, st.sval
434                .equals("\'Hello World\' Hello\' World"));
435    }
436
437    private void setTest(String s) {
438        testString = s;
439        r = new Support_StringReader(testString);
440        st = new StreamTokenizer(r);
441    }
442
443    protected void setUp() {
444    }
445
446    protected void tearDown() {
447    }
448}
449