1/* 2 * Licensed to the Apache Software Foundation (ASF) under one or more 3 * contributor license agreements. See the NOTICE file distributed with 4 * this work for additional information regarding copyright ownership. 5 * The ASF licenses this file to You under the Apache License, Version 2.0 6 * (the "License"); you may not use this file except in compliance with 7 * the License. You may obtain a copy of the License at 8 * 9 * http://www.apache.org/licenses/LICENSE-2.0 10 * 11 * Unless required by applicable law or agreed to in writing, software 12 * distributed under the License is distributed on an "AS IS" BASIS, 13 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 * See the License for the specific language governing permissions and 15 * limitations under the License. 16 */ 17 18package org.apache.harmony.luni.tests.java.io; 19 20import java.io.ByteArrayInputStream; 21import java.io.CharArrayReader; 22import java.io.IOException; 23import java.io.PipedInputStream; 24import java.io.PipedOutputStream; 25import java.io.Reader; 26import java.io.StreamTokenizer; 27import java.io.StringBufferInputStream; 28 29import tests.support.Support_StringReader; 30 31public class StreamTokenizerTest extends junit.framework.TestCase { 32 Support_StringReader r; 33 34 StreamTokenizer st; 35 36 String testString; 37 38 /** 39 * @tests java.io.StreamTokenizer#StreamTokenizer(java.io.InputStream) 40 */ 41 @SuppressWarnings("deprecation") 42 public void test_ConstructorLjava_io_InputStream() throws IOException { 43 st = new StreamTokenizer(new StringBufferInputStream( 44 "/comments\n d 8 'h'")); 45 46 assertEquals("the next token returned should be the letter d", 47 StreamTokenizer.TT_WORD, st.nextToken()); 48 assertEquals("the next token returned should be the letter d", 49 "d", st.sval); 50 51 assertEquals("the next token returned should be the digit 8", 52 StreamTokenizer.TT_NUMBER, st.nextToken()); 53 assertEquals("the next token returned should be the digit 8", 54 8.0, st.nval); 55 56 assertEquals("the next token returned should be the quote character", 57 39, st.nextToken()); 58 assertEquals("the next token returned should be the quote character", 59 "h", st.sval); 60 } 61 62 /** 63 * @tests java.io.StreamTokenizer#StreamTokenizer(java.io.Reader) 64 */ 65 public void test_ConstructorLjava_io_Reader() throws IOException { 66 setTest("/testing\n d 8 'h' "); 67 assertEquals("the next token returned should be the letter d skipping the comments", 68 StreamTokenizer.TT_WORD, st.nextToken()); 69 assertEquals("the next token returned should be the letter d", 70 "d", st.sval); 71 72 assertEquals("the next token returned should be the digit 8", 73 StreamTokenizer.TT_NUMBER, st.nextToken()); 74 assertEquals("the next token returned should be the digit 8", 75 8.0, st.nval); 76 77 assertEquals("the next token returned should be the quote character", 78 39, st.nextToken()); 79 assertEquals("the next token returned should be the quote character", 80 "h", st.sval); 81 } 82 83 /** 84 * @tests java.io.StreamTokenizer#commentChar(int) 85 */ 86 public void test_commentCharI() throws IOException { 87 setTest("*comment \n / 8 'h' "); 88 st.ordinaryChar('/'); 89 st.commentChar('*'); 90 assertEquals("nextToken() did not return the character / skiping the comments starting with *", 91 47, st.nextToken()); 92 assertTrue("the next token returned should be the digit 8", st 93 .nextToken() == StreamTokenizer.TT_NUMBER 94 && st.nval == 8.0); 95 assertTrue("the next token returned should be the quote character", 96 st.nextToken() == 39 && st.sval.equals("h")); 97 } 98 99 /** 100 * @tests java.io.StreamTokenizer#eolIsSignificant(boolean) 101 */ 102 public void test_eolIsSignificantZ() throws IOException { 103 setTest("d 8\n"); 104 // by default end of line characters are not significant 105 assertTrue("nextToken did not return d", 106 st.nextToken() == StreamTokenizer.TT_WORD 107 && st.sval.equals("d")); 108 assertTrue("nextToken did not return 8", 109 st.nextToken() == StreamTokenizer.TT_NUMBER 110 && st.nval == 8.0); 111 assertTrue("nextToken should be the end of file", 112 st.nextToken() == StreamTokenizer.TT_EOF); 113 setTest("d\n"); 114 st.eolIsSignificant(true); 115 // end of line characters are significant 116 assertTrue("nextToken did not return d", 117 st.nextToken() == StreamTokenizer.TT_WORD 118 && st.sval.equals("d")); 119 assertTrue("nextToken is the end of line", 120 st.nextToken() == StreamTokenizer.TT_EOL); 121 } 122 123 /** 124 * @tests java.io.StreamTokenizer#lineno() 125 */ 126 public void test_lineno() throws IOException { 127 setTest("d\n 8\n"); 128 assertEquals("the lineno should be 1", 1, st.lineno()); 129 st.nextToken(); 130 st.nextToken(); 131 assertEquals("the lineno should be 2", 2, st.lineno()); 132 st.nextToken(); 133 assertEquals("the next line no should be 3", 3, st.lineno()); 134 } 135 136 /** 137 * @tests java.io.StreamTokenizer#lowerCaseMode(boolean) 138 */ 139 public void test_lowerCaseModeZ() throws Exception { 140 // SM. 141 setTest("HELLOWORLD"); 142 st.lowerCaseMode(true); 143 144 st.nextToken(); 145 assertEquals("sval not converted to lowercase.", "helloworld", st.sval 146 ); 147 } 148 149 /** 150 * @tests java.io.StreamTokenizer#nextToken() 151 */ 152 @SuppressWarnings("deprecation") 153 public void test_nextToken() throws IOException { 154 // SM. 155 setTest("\r\n/* fje fje 43.4 f \r\n f g */ 456.459 \r\n" 156 + "Hello / \r\n \r\n \n \r \257 Hi \'Hello World\'"); 157 st.ordinaryChar('/'); 158 st.slashStarComments(true); 159 st.nextToken(); 160 assertTrue("Wrong Token type1: " + (char) st.ttype, 161 st.ttype == StreamTokenizer.TT_NUMBER); 162 st.nextToken(); 163 assertTrue("Wrong Token type2: " + st.ttype, 164 st.ttype == StreamTokenizer.TT_WORD); 165 st.nextToken(); 166 assertTrue("Wrong Token type3: " + st.ttype, st.ttype == '/'); 167 st.nextToken(); 168 assertTrue("Wrong Token type4: " + st.ttype, 169 st.ttype == StreamTokenizer.TT_WORD); 170 st.nextToken(); 171 assertTrue("Wrong Token type5: " + st.ttype, 172 st.ttype == StreamTokenizer.TT_WORD); 173 st.nextToken(); 174 assertTrue("Wrong Token type6: " + st.ttype, st.ttype == '\''); 175 assertTrue("Wrong Token type7: " + st.ttype, st.sval 176 .equals("Hello World")); 177 st.nextToken(); 178 assertTrue("Wrong Token type8: " + st.ttype, st.ttype == -1); 179 180 final PipedInputStream pin = new PipedInputStream(); 181 PipedOutputStream pout = new PipedOutputStream(pin); 182 pout.write("hello\n\r\r".getBytes("UTF-8")); 183 StreamTokenizer s = new StreamTokenizer(pin); 184 s.eolIsSignificant(true); 185 assertTrue("Wrong token 1,1", 186 s.nextToken() == StreamTokenizer.TT_WORD 187 && s.sval.equals("hello")); 188 assertTrue("Wrong token 1,2", s.nextToken() == '\n'); 189 assertTrue("Wrong token 1,3", s.nextToken() == '\n'); 190 assertTrue("Wrong token 1,4", s.nextToken() == '\n'); 191 pout.close(); 192 assertTrue("Wrong token 1,5", 193 s.nextToken() == StreamTokenizer.TT_EOF); 194 StreamTokenizer tokenizer = new StreamTokenizer( 195 new Support_StringReader("\n \r\n#")); 196 tokenizer.ordinaryChar('\n'); // make \n ordinary 197 tokenizer.eolIsSignificant(true); 198 assertTrue("Wrong token 2,1", tokenizer.nextToken() == '\n'); 199 assertTrue("Wrong token 2,2", tokenizer.nextToken() == '\n'); 200 assertEquals("Wrong token 2,3", '#', tokenizer.nextToken()); 201 } 202 203 /** 204 * @tests java.io.StreamTokenizer#ordinaryChar(int) 205 */ 206 public void test_ordinaryCharI() throws IOException { 207 // SM. 208 setTest("Ffjein 893"); 209 st.ordinaryChar('F'); 210 st.nextToken(); 211 assertTrue("OrdinaryChar failed." + (char) st.ttype, 212 st.ttype == 'F'); 213 } 214 215 /** 216 * @tests java.io.StreamTokenizer#ordinaryChars(int, int) 217 */ 218 public void test_ordinaryCharsII() throws IOException { 219 // SM. 220 setTest("azbc iof z 893"); 221 st.ordinaryChars('a', 'z'); 222 assertEquals("OrdinaryChars failed.", 'a', st.nextToken()); 223 assertEquals("OrdinaryChars failed.", 'z', st.nextToken()); 224 } 225 226 /** 227 * @tests java.io.StreamTokenizer#parseNumbers() 228 */ 229 public void test_parseNumbers() throws IOException { 230 // SM 231 setTest("9.9 678"); 232 assertTrue("Base behavior failed.", 233 st.nextToken() == StreamTokenizer.TT_NUMBER); 234 st.ordinaryChars('0', '9'); 235 assertEquals("setOrdinary failed.", '6', st.nextToken()); 236 st.parseNumbers(); 237 assertTrue("parseNumbers failed.", 238 st.nextToken() == StreamTokenizer.TT_NUMBER); 239 } 240 241 /** 242 * @tests java.io.StreamTokenizer#pushBack() 243 */ 244 public void test_pushBack() throws IOException { 245 // SM. 246 setTest("Hello 897"); 247 st.nextToken(); 248 st.pushBack(); 249 assertTrue("PushBack failed.", 250 st.nextToken() == StreamTokenizer.TT_WORD); 251 } 252 253 /** 254 * @tests java.io.StreamTokenizer#quoteChar(int) 255 */ 256 public void test_quoteCharI() throws IOException { 257 // SM 258 setTest("<Hello World< HelloWorldH"); 259 st.quoteChar('<'); 260 assertEquals("QuoteChar failed.", '<', st.nextToken()); 261 assertEquals("QuoteChar failed.", "Hello World", st.sval); 262 st.quoteChar('H'); 263 st.nextToken(); 264 assertEquals("QuoteChar failed for word.", "elloWorld", st.sval 265 ); 266 } 267 268 /** 269 * @tests java.io.StreamTokenizer#resetSyntax() 270 */ 271 public void test_resetSyntax() throws IOException { 272 // SM 273 setTest("H 9\' ello World"); 274 st.resetSyntax(); 275 assertTrue("resetSyntax failed1." + (char) st.ttype, 276 st.nextToken() == 'H'); 277 assertTrue("resetSyntax failed1." + (char) st.ttype, 278 st.nextToken() == ' '); 279 assertTrue("resetSyntax failed2." + (char) st.ttype, 280 st.nextToken() == '9'); 281 assertTrue("resetSyntax failed3." + (char) st.ttype, 282 st.nextToken() == '\''); 283 } 284 285 /** 286 * @tests java.io.StreamTokenizer#slashSlashComments(boolean) 287 */ 288 public void test_slashSlashCommentsZ() throws IOException { 289 // SM. 290 setTest("// foo \r\n /fiji \r\n -456"); 291 st.ordinaryChar('/'); 292 st.slashSlashComments(true); 293 assertEquals("Test failed.", '/', st.nextToken()); 294 assertTrue("Test failed.", 295 st.nextToken() == StreamTokenizer.TT_WORD); 296 } 297 298 /** 299 * @tests java.io.StreamTokenizer#slashSlashComments(boolean) 300 */ 301 public void test_slashSlashComments_withSSOpen() throws IOException { 302 Reader reader = new CharArrayReader( "t // t t t".toCharArray()); 303 304 StreamTokenizer st = new StreamTokenizer(reader); 305 st.slashSlashComments(true); 306 307 assertEquals(StreamTokenizer.TT_WORD,st.nextToken()); 308 assertEquals(StreamTokenizer.TT_EOF,st.nextToken()); 309 } 310 311 /** 312 * @tests java.io.StreamTokenizer#slashSlashComments(boolean) 313 */ 314 public void test_slashSlashComments_withSSOpen_NoComment() throws IOException { 315 Reader reader = new CharArrayReader( "// t".toCharArray()); 316 317 StreamTokenizer st = new StreamTokenizer(reader); 318 st.slashSlashComments(true); 319 st.ordinaryChar('/'); 320 321 assertEquals(StreamTokenizer.TT_EOF,st.nextToken()); 322 } 323 324 /** 325 * @tests java.io.StreamTokenizer#slashSlashComments(boolean) 326 */ 327 public void test_slashSlashComments_withSSClosed() throws IOException { 328 Reader reader = new CharArrayReader( "// t".toCharArray()); 329 330 StreamTokenizer st = new StreamTokenizer(reader); 331 st.slashSlashComments(false); 332 st.ordinaryChar('/'); 333 334 assertEquals('/',st.nextToken()); 335 assertEquals('/',st.nextToken()); 336 assertEquals(StreamTokenizer.TT_WORD,st.nextToken()); 337 } 338 339 /** 340 * @tests java.io.StreamTokenizer#slashStarComments(boolean) 341 */ 342 public void test_slashStarCommentsZ() throws IOException { 343 setTest("/* foo \r\n /fiji \r\n*/ -456"); 344 st.ordinaryChar('/'); 345 st.slashStarComments(true); 346 assertTrue("Test failed.", 347 st.nextToken() == StreamTokenizer.TT_NUMBER); 348 } 349 350 /** 351 * @tests java.io.StreamTokenizer#slashStarComments(boolean) 352 */ 353 public void test_slashStarComments_withSTOpen() throws IOException { 354 Reader reader = new CharArrayReader( "t /* t */ t".toCharArray()); 355 356 StreamTokenizer st = new StreamTokenizer(reader); 357 st.slashStarComments(true); 358 359 assertEquals(StreamTokenizer.TT_WORD,st.nextToken()); 360 assertEquals(StreamTokenizer.TT_WORD,st.nextToken()); 361 assertEquals(StreamTokenizer.TT_EOF,st.nextToken()); 362 } 363 364 /** 365 * @tests java.io.StreamTokenizer#slashStarComments(boolean) 366 */ 367 public void test_slashStarComments_withSTClosed() throws IOException { 368 Reader reader = new CharArrayReader( "t /* t */ t".toCharArray()); 369 370 StreamTokenizer st = new StreamTokenizer(reader); 371 st.slashStarComments(false); 372 373 assertEquals(StreamTokenizer.TT_WORD,st.nextToken()); 374 assertEquals(StreamTokenizer.TT_EOF,st.nextToken()); 375 } 376 377 /** 378 * @tests java.io.StreamTokenizer#toString() 379 */ 380 public void test_toString() throws IOException { 381 setTest("ABC Hello World"); 382 st.nextToken(); 383 assertTrue("toString failed." + st.toString(), 384 st.toString().equals( 385 "Token[ABC], line 1")); 386 387 // Regression test for HARMONY-4070 388 byte[] data = new byte[] { (byte) '-' }; 389 StreamTokenizer tokenizer = new StreamTokenizer( 390 new ByteArrayInputStream(data)); 391 tokenizer.nextToken(); 392 String result = tokenizer.toString(); 393 assertEquals("Token['-'], line 1", result); 394 } 395 396 /** 397 * @tests java.io.StreamTokenizer#whitespaceChars(int, int) 398 */ 399 public void test_whitespaceCharsII() throws IOException { 400 setTest("azbc iof z 893"); 401 st.whitespaceChars('a', 'z'); 402 assertTrue("OrdinaryChar failed.", 403 st.nextToken() == StreamTokenizer.TT_NUMBER); 404 } 405 406 /** 407 * @tests java.io.StreamTokenizer#wordChars(int, int) 408 */ 409 public void test_wordCharsII() throws IOException { 410 setTest("A893 -9B87"); 411 st.wordChars('0', '9'); 412 assertTrue("WordChar failed1.", 413 st.nextToken() == StreamTokenizer.TT_WORD); 414 assertEquals("WordChar failed2.", "A893", st.sval); 415 assertTrue("WordChar failed3.", 416 st.nextToken() == StreamTokenizer.TT_NUMBER); 417 st.nextToken(); 418 assertEquals("WordChar failed4.", "B87", st.sval); 419 420 setTest(" Hello World"); 421 st.wordChars(' ', ' '); 422 st.nextToken(); 423 assertEquals("WordChars failed for whitespace.", "Hello World", st.sval 424 ); 425 426 setTest(" Hello World\r\n \'Hello World\' Hello\' World"); 427 st.wordChars(' ', ' '); 428 st.wordChars('\'', '\''); 429 st.nextToken(); 430 assertTrue("WordChars failed for whitespace: " + st.sval, st.sval 431 .equals("Hello World")); 432 st.nextToken(); 433 assertTrue("WordChars failed for quote1: " + st.sval, st.sval 434 .equals("\'Hello World\' Hello\' World")); 435 } 436 437 private void setTest(String s) { 438 testString = s; 439 r = new Support_StringReader(testString); 440 st = new StreamTokenizer(r); 441 } 442 443 protected void setUp() { 444 } 445 446 protected void tearDown() { 447 } 448} 449