StreamTokenizerTest.java revision 2ad60cfc28e14ee8f0bb038720836a4696c478ad
1/* 2 * Licensed to the Apache Software Foundation (ASF) under one or more 3 * contributor license agreements. See the NOTICE file distributed with 4 * this work for additional information regarding copyright ownership. 5 * The ASF licenses this file to You under the Apache License, Version 2.0 6 * (the "License"); you may not use this file except in compliance with 7 * the License. You may obtain a copy of the License at 8 * 9 * http://www.apache.org/licenses/LICENSE-2.0 10 * 11 * Unless required by applicable law or agreed to in writing, software 12 * distributed under the License is distributed on an "AS IS" BASIS, 13 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 * See the License for the specific language governing permissions and 15 * limitations under the License. 16 */ 17 18package tests.api.java.io; 19 20import junit.framework.Assert; 21 22import java.io.ByteArrayInputStream; 23import java.io.CharArrayReader; 24import java.io.IOException; 25import java.io.PipedInputStream; 26import java.io.PipedOutputStream; 27import java.io.Reader; 28import java.io.StreamTokenizer; 29import java.io.StringBufferInputStream; 30import java.io.StringReader; 31 32import tests.support.Support_StringReader; 33// TODO: most of the assertTrue calls in this test case should be 34// replaced with assertEquals (possibly two assertEquals) see 35// test_ConstructorLjava_io_InputStream for example. 36// This gives much more helpful error messages. 37 38public class StreamTokenizerTest extends junit.framework.TestCase { 39 Support_StringReader r; 40 41 StreamTokenizer st; 42 43 String testString; 44 45 /** 46 * @tests java.io.StreamTokenizer#StreamTokenizer(java.io.InputStream) 47 */ 48 public void test_ConstructorLjava_io_InputStream() throws IOException { 49 st = new StreamTokenizer(new StringBufferInputStream( 50 "/comments\n d 8 'h'")); 51 52 assertEquals("the next token returned should be the letter d", 53 StreamTokenizer.TT_WORD, st.nextToken()); 54 assertEquals("the next token returned should be the letter d", 55 "d", st.sval); 56 57 assertEquals("the next token returned should be the digit 8", 58 StreamTokenizer.TT_NUMBER, st.nextToken()); 59 assertEquals("the next token returned should be the digit 8", 60 8.0, st.nval); 61 62 assertEquals("the next token returned should be the quote character", 63 39, st.nextToken()); 64 assertEquals("the next token returned should be the quote character", 65 "h", st.sval); 66 } 67 68 /** 69 * @tests java.io.StreamTokenizer#StreamTokenizer(java.io.Reader) 70 */ 71 public void test_ConstructorLjava_io_Reader() throws IOException { 72 setTest("/testing\n d 8 'h' "); 73 assertEquals("the next token returned should be the letter d skipping the comments", 74 StreamTokenizer.TT_WORD, st.nextToken()); 75 assertEquals("the next token returned should be the letter d", 76 "d", st.sval); 77 78 assertEquals("the next token returned should be the digit 8", 79 StreamTokenizer.TT_NUMBER, st.nextToken()); 80 assertEquals("the next token returned should be the digit 8", 81 8.0, st.nval); 82 83 assertEquals("the next token returned should be the quote character", 84 39, st.nextToken()); 85 assertEquals("the next token returned should be the quote character", 86 "h", st.sval); 87 } 88 89 /** 90 * @tests java.io.StreamTokenizer#commentChar(int) 91 */ 92 public void test_commentCharI() throws IOException { 93 setTest("*comment \n / 8 'h' "); 94 st.ordinaryChar('/'); 95 st.commentChar('*'); 96 assertEquals("nextToken() did not return the character / skiping the comments starting with *", 97 47, st.nextToken()); 98 assertTrue("the next token returned should be the digit 8", st 99 .nextToken() == StreamTokenizer.TT_NUMBER 100 && st.nval == 8.0); 101 assertTrue("the next token returned should be the quote character", 102 st.nextToken() == 39 && st.sval.equals("h")); 103 } 104 105 /** 106 * @tests java.io.StreamTokenizer#eolIsSignificant(boolean) 107 */ 108 public void test_eolIsSignificantZ() throws IOException { 109 setTest("d 8\n"); 110 // by default end of line characters are not significant 111 assertTrue("nextToken did not return d", 112 st.nextToken() == StreamTokenizer.TT_WORD 113 && st.sval.equals("d")); 114 assertTrue("nextToken did not return 8", 115 st.nextToken() == StreamTokenizer.TT_NUMBER 116 && st.nval == 8.0); 117 assertTrue("nextToken should be the end of file", 118 st.nextToken() == StreamTokenizer.TT_EOF); 119 setTest("d\n"); 120 st.eolIsSignificant(true); 121 // end of line characters are significant 122 assertTrue("nextToken did not return d", 123 st.nextToken() == StreamTokenizer.TT_WORD 124 && st.sval.equals("d")); 125 assertTrue("nextToken is the end of line", 126 st.nextToken() == StreamTokenizer.TT_EOL); 127 } 128 129 /** 130 * @tests java.io.StreamTokenizer#lineno() 131 */ 132 public void test_lineno() throws IOException { 133 setTest("d\n 8\n"); 134 assertEquals("the lineno should be 1", 1, st.lineno()); 135 st.nextToken(); 136 st.nextToken(); 137 assertEquals("the lineno should be 2", 2, st.lineno()); 138 st.nextToken(); 139 assertEquals("the next line no should be 3", 3, st.lineno()); 140 } 141 142 /** 143 * @tests java.io.StreamTokenizer#lowerCaseMode(boolean) 144 */ 145 public void test_lowerCaseModeZ() throws Exception { 146 // SM. 147 setTest("HELLOWORLD"); 148 st.lowerCaseMode(true); 149 150 st.nextToken(); 151 assertEquals("sval not converted to lowercase.", "helloworld", st.sval 152 ); 153 } 154 155 /** 156 * @tests java.io.StreamTokenizer#nextToken() 157 */ 158 public void test_nextToken() throws IOException { 159 // SM. 160 setTest("\r\n/* fje fje 43.4 f \r\n f g */ 456.459 \r\n" 161 + "Hello / \r\n \r\n \n \r \257 Hi \'Hello World\'"); 162 st.ordinaryChar('/'); 163 st.slashStarComments(true); 164 st.nextToken(); 165 assertTrue("Wrong Token type1: " + (char) st.ttype, 166 st.ttype == StreamTokenizer.TT_NUMBER); 167 st.nextToken(); 168 assertTrue("Wrong Token type2: " + st.ttype, 169 st.ttype == StreamTokenizer.TT_WORD); 170 st.nextToken(); 171 assertTrue("Wrong Token type3: " + st.ttype, st.ttype == '/'); 172 st.nextToken(); 173 assertTrue("Wrong Token type4: " + st.ttype, 174 st.ttype == StreamTokenizer.TT_WORD); 175 st.nextToken(); 176 assertTrue("Wrong Token type5: " + st.ttype, 177 st.ttype == StreamTokenizer.TT_WORD); 178 st.nextToken(); 179 assertTrue("Wrong Token type6: " + st.ttype, st.ttype == '\''); 180 assertTrue("Wrong Token type7: " + st.ttype, st.sval 181 .equals("Hello World")); 182 st.nextToken(); 183 assertTrue("Wrong Token type8: " + st.ttype, st.ttype == -1); 184 185 final PipedInputStream pin = new PipedInputStream(); 186 PipedOutputStream pout = new PipedOutputStream(pin); 187 pout.write("hello\n\r\r".getBytes()); 188 StreamTokenizer s = new StreamTokenizer(pin); 189 s.eolIsSignificant(true); 190 assertTrue("Wrong token 1,1", 191 s.nextToken() == StreamTokenizer.TT_WORD 192 && s.sval.equals("hello")); 193 assertTrue("Wrong token 1,2", s.nextToken() == '\n'); 194 assertTrue("Wrong token 1,3", s.nextToken() == '\n'); 195 assertTrue("Wrong token 1,4", s.nextToken() == '\n'); 196 pout.close(); 197 assertTrue("Wrong token 1,5", 198 s.nextToken() == StreamTokenizer.TT_EOF); 199 StreamTokenizer tokenizer = new StreamTokenizer( 200 new Support_StringReader("\n \r\n#")); 201 tokenizer.ordinaryChar('\n'); // make \n ordinary 202 tokenizer.eolIsSignificant(true); 203 assertTrue("Wrong token 2,1", tokenizer.nextToken() == '\n'); 204 assertTrue("Wrong token 2,2", tokenizer.nextToken() == '\n'); 205 assertEquals("Wrong token 2,3", '#', tokenizer.nextToken()); 206 } 207 208 /** 209 * @tests java.io.StreamTokenizer#ordinaryChar(int) 210 */ 211 public void test_ordinaryCharI() throws IOException { 212 // SM. 213 setTest("Ffjein 893"); 214 st.ordinaryChar('F'); 215 st.nextToken(); 216 assertTrue("OrdinaryChar failed." + (char) st.ttype, 217 st.ttype == 'F'); 218 } 219 220 /** 221 * @tests java.io.StreamTokenizer#ordinaryChars(int, int) 222 */ 223 public void test_ordinaryCharsII() throws IOException { 224 // SM. 225 setTest("azbc iof z 893"); 226 st.ordinaryChars('a', 'z'); 227 assertEquals("OrdinaryChars failed.", 'a', st.nextToken()); 228 assertEquals("OrdinaryChars failed.", 'z', st.nextToken()); 229 } 230 231 /** 232 * @tests java.io.StreamTokenizer#parseNumbers() 233 */ 234 public void test_parseNumbers() throws IOException { 235 // SM 236 setTest("9.9 678"); 237 assertTrue("Base behavior failed.", 238 st.nextToken() == StreamTokenizer.TT_NUMBER); 239 st.ordinaryChars('0', '9'); 240 assertEquals("setOrdinary failed.", '6', st.nextToken()); 241 st.parseNumbers(); 242 assertTrue("parseNumbers failed.", 243 st.nextToken() == StreamTokenizer.TT_NUMBER); 244 } 245 246 /** 247 * @tests java.io.StreamTokenizer#pushBack() 248 */ 249 public void test_pushBack() throws IOException { 250 // SM. 251 setTest("Hello 897"); 252 st.nextToken(); 253 st.pushBack(); 254 assertTrue("PushBack failed.", 255 st.nextToken() == StreamTokenizer.TT_WORD); 256 } 257 258 /** 259 * @tests java.io.StreamTokenizer#quoteChar(int) 260 */ 261 public void test_quoteCharI() throws IOException { 262 // SM 263 setTest("<Hello World< HelloWorldH"); 264 st.quoteChar('<'); 265 assertEquals("QuoteChar failed.", '<', st.nextToken()); 266 assertEquals("QuoteChar failed.", "Hello World", st.sval); 267 st.quoteChar('H'); 268 st.nextToken(); 269 assertEquals("QuoteChar failed for word.", "elloWorld", st.sval 270 ); 271 } 272 273 /** 274 * @tests java.io.StreamTokenizer#resetSyntax() 275 */ 276 public void test_resetSyntax() throws IOException { 277 // SM 278 setTest("H 9\' ello World"); 279 st.resetSyntax(); 280 assertTrue("resetSyntax failed1." + (char) st.ttype, 281 st.nextToken() == 'H'); 282 assertTrue("resetSyntax failed1." + (char) st.ttype, 283 st.nextToken() == ' '); 284 assertTrue("resetSyntax failed2." + (char) st.ttype, 285 st.nextToken() == '9'); 286 assertTrue("resetSyntax failed3." + (char) st.ttype, 287 st.nextToken() == '\''); 288 } 289 290 /** 291 * @tests java.io.StreamTokenizer#slashSlashComments(boolean) 292 */ 293 public void test_slashSlashCommentsZ() throws IOException { 294 // SM. 295 setTest("// foo \r\n /fiji \r\n -456"); 296 st.ordinaryChar('/'); 297 st.slashSlashComments(true); 298 assertEquals("Test failed.", '/', st.nextToken()); 299 assertTrue("Test failed.", 300 st.nextToken() == StreamTokenizer.TT_WORD); 301 } 302 303 /** 304 * @tests java.io.StreamTokenizer#slashSlashComments(boolean) 305 */ 306 public void test_slashSlashComments_withSSOpen() throws IOException { 307 Reader reader = new CharArrayReader( "t // t t t".toCharArray()); 308 309 StreamTokenizer st = new StreamTokenizer(reader); 310 st.slashSlashComments(true); 311 312 assertEquals(StreamTokenizer.TT_WORD,st.nextToken()); 313 assertEquals(StreamTokenizer.TT_EOF,st.nextToken()); 314 } 315 316 /** 317 * @tests java.io.StreamTokenizer#slashSlashComments(boolean) 318 */ 319 public void test_slashSlashComments_withSSOpen_NoComment() throws IOException { 320 Reader reader = new CharArrayReader( "// t".toCharArray()); 321 322 StreamTokenizer st = new StreamTokenizer(reader); 323 st.slashSlashComments(true); 324 st.ordinaryChar('/'); 325 326 assertEquals(StreamTokenizer.TT_EOF,st.nextToken()); 327 } 328 329 /** 330 * @tests java.io.StreamTokenizer#slashSlashComments(boolean) 331 */ 332 public void test_slashSlashComments_withSSClosed() throws IOException { 333 Reader reader = new CharArrayReader( "// t".toCharArray()); 334 335 StreamTokenizer st = new StreamTokenizer(reader); 336 st.slashSlashComments(false); 337 st.ordinaryChar('/'); 338 339 assertEquals('/',st.nextToken()); 340 assertEquals('/',st.nextToken()); 341 assertEquals(StreamTokenizer.TT_WORD,st.nextToken()); 342 } 343 344 /** 345 * @tests java.io.StreamTokenizer#slashStarComments(boolean) 346 */ 347 public void test_slashStarCommentsZ() throws IOException { 348 setTest("/* foo \r\n /fiji \r\n*/ -456"); 349 st.ordinaryChar('/'); 350 st.slashStarComments(true); 351 assertTrue("Test failed.", 352 st.nextToken() == StreamTokenizer.TT_NUMBER); 353 } 354 355 /** 356 * @tests java.io.StreamTokenizer#slashStarComments(boolean) 357 */ 358 public void test_slashStarComments_withSTOpen() throws IOException { 359 Reader reader = new CharArrayReader( "t /* t */ t".toCharArray()); 360 361 StreamTokenizer st = new StreamTokenizer(reader); 362 st.slashStarComments(true); 363 364 assertEquals(StreamTokenizer.TT_WORD,st.nextToken()); 365 assertEquals(StreamTokenizer.TT_WORD,st.nextToken()); 366 assertEquals(StreamTokenizer.TT_EOF,st.nextToken()); 367 } 368 369 /** 370 * @tests java.io.StreamTokenizer#slashStarComments(boolean) 371 */ 372 public void test_slashStarComments_withSTClosed() throws IOException { 373 Reader reader = new CharArrayReader( "t /* t */ t".toCharArray()); 374 375 StreamTokenizer st = new StreamTokenizer(reader); 376 st.slashStarComments(false); 377 378 assertEquals(StreamTokenizer.TT_WORD,st.nextToken()); 379 assertEquals(StreamTokenizer.TT_EOF,st.nextToken()); 380 } 381 382 /** 383 * @tests java.io.StreamTokenizer#toString() 384 */ 385 public void test_toString() throws IOException { 386 setTest("ABC Hello World"); 387 st.nextToken(); 388 assertTrue("toString failed." + st.toString(), 389 st.toString().equals( 390 "Token[ABC], line 1")); 391 } 392 393 /** 394 * @tests java.io.StreamTokenizer#whitespaceChars(int, int) 395 */ 396 public void test_whitespaceCharsII() throws IOException { 397 setTest("azbc iof z 893"); 398 st.whitespaceChars('a', 'z'); 399 assertTrue("OrdinaryChar failed.", 400 st.nextToken() == StreamTokenizer.TT_NUMBER); 401 } 402 403 /** 404 * @tests java.io.StreamTokenizer#wordChars(int, int) 405 */ 406 public void test_wordCharsII() throws IOException { 407 setTest("A893 -9B87"); 408 st.wordChars('0', '9'); 409 assertTrue("WordChar failed1.", 410 st.nextToken() == StreamTokenizer.TT_WORD); 411 assertEquals("WordChar failed2.", "A893", st.sval); 412 assertTrue("WordChar failed3.", 413 st.nextToken() == StreamTokenizer.TT_NUMBER); 414 st.nextToken(); 415 assertEquals("WordChar failed4.", "B87", st.sval); 416 417 setTest(" Hello World"); 418 st.wordChars(' ', ' '); 419 st.nextToken(); 420 assertEquals("WordChars failed for whitespace.", "Hello World", st.sval 421 ); 422 423 setTest(" Hello World\r\n \'Hello World\' Hello\' World"); 424 st.wordChars(' ', ' '); 425 st.wordChars('\'', '\''); 426 st.nextToken(); 427 assertTrue("WordChars failed for whitespace: " + st.sval, st.sval 428 .equals("Hello World")); 429 st.nextToken(); 430 assertTrue("WordChars failed for quote1: " + st.sval, st.sval 431 .equals("\'Hello World\' Hello\' World")); 432 } 433 434 private void setTest(String s) { 435 testString = s; 436 r = new Support_StringReader(testString); 437 st = new StreamTokenizer(r); 438 } 439 440 protected void setUp() { 441 } 442 443 protected void tearDown() { 444 } 445 446 public void test_basicStringTokenizerMethods() 447 { 448 String str = "Testing 12345 \n alpha \r\n omega"; 449 String strb = "-3.8 'BLIND mice' \r sEe /* how */ they run"; 450 StringReader aa = new StringReader(str); 451 StringReader ba = new StringReader(strb); 452 StreamTokenizer a = new StreamTokenizer(aa); 453 StreamTokenizer b = new StreamTokenizer(ba); 454 455 try { 456 Assert.assertTrue(a.lineno() == 1); 457 Assert.assertTrue(a.nextToken() == StreamTokenizer.TT_WORD); 458 Assert.assertTrue(a.toString().equals("Token[Testing], line 1")); 459 Assert.assertTrue(a.nextToken() == StreamTokenizer.TT_NUMBER); 460 Assert.assertTrue(a.toString().equals("Token[n=12345.0], line 1")); 461 Assert.assertTrue(a.nextToken() == StreamTokenizer.TT_WORD); 462 Assert.assertTrue(a.toString().equals("Token[alpha], line 2")); 463 Assert.assertTrue(a.nextToken() == StreamTokenizer.TT_WORD); 464 Assert.assertTrue(a.toString().equals("Token[omega], line 3")); 465 Assert.assertTrue(a.nextToken() == StreamTokenizer.TT_EOF); 466 Assert.assertTrue(a.toString().equals("Token[EOF], line 3")); 467 468 b.commentChar('u'); 469 b.eolIsSignificant(true); 470 b.lowerCaseMode(true); 471 b.ordinaryChar('y'); 472 b.slashStarComments(true); 473 474 Assert.assertTrue(b.nextToken() == StreamTokenizer.TT_NUMBER); 475 Assert.assertTrue(b.nval == -3.8); 476 Assert.assertTrue(b.toString().equals("Token[n=-3.8], line 1")); 477 Assert.assertTrue(b.nextToken() == 39); // ' 478 Assert.assertTrue(b.toString().equals("Token[BLIND mice], line 1")); 479 Assert.assertTrue(b.nextToken() == 10); // \n 480 Assert.assertTrue(b.toString().equals("Token[EOL], line 2")); 481 Assert.assertTrue(b.nextToken() == StreamTokenizer.TT_WORD); 482 Assert.assertTrue(b.toString().equals("Token[see], line 2")); 483 Assert.assertTrue(b.nextToken() == StreamTokenizer.TT_WORD); 484 Assert.assertTrue(b.toString().equals("Token[the], line 2")); 485 Assert.assertTrue(b.nextToken() == 121); // y 486 Assert.assertTrue(b.toString().equals("Token['y'], line 2")); 487 Assert.assertTrue(b.nextToken() == StreamTokenizer.TT_WORD); 488 Assert.assertTrue(b.toString().equals("Token[r], line 2")); 489 Assert.assertTrue(b.nextToken() == StreamTokenizer.TT_EOF); 490 Assert.assertTrue(b.toString().equals("Token[EOF], line 2")); 491 } 492 catch (Exception ex){ 493 System.out.println("Exception found in StreamTokenizer"); 494 ex.printStackTrace(); 495 throw new RuntimeException("error in test, see stdout"); 496 } 497 } 498 499 public void test_harmonyRegressionTest() { 500 byte[] data = new byte[] {(byte) '-'}; 501 StreamTokenizer tokenizer = new StreamTokenizer(new ByteArrayInputStream(data)); 502 try { 503 tokenizer.nextToken(); 504 } catch(Exception e) { 505 Assert.fail(e.getMessage()); 506 } 507 String result = tokenizer.toString(); 508 Assert.assertEquals("Token['-'], line 1", result); 509 } 510 511 public void test_harmonyRegressionTest2() { 512 byte[] data = new byte[] {(byte) '"', 513 (byte) 'H', 514 (byte) 'e', 515 (byte) 'l', 516 (byte) 'l', 517 (byte) 'o', 518 (byte) '"'}; 519 StreamTokenizer tokenizer = new StreamTokenizer(new ByteArrayInputStream(data)); 520 try { 521 tokenizer.nextToken(); 522 } catch (Exception e) { 523 e.printStackTrace(); 524 } 525 String result = tokenizer.toString(); 526 Assert.assertEquals("Token[Hello], line 1", result); 527 } 528} 529