1/* 2 * Copyright (C) 2010 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17package org.json; 18 19import junit.framework.AssertionFailedError; 20import junit.framework.TestCase; 21 22/** 23 * This black box test was written without inspecting the non-free org.json sourcecode. 24 */ 25public class JSONTokenerTest extends TestCase { 26 27 public void testNulls() throws JSONException { 28 // JSONTokener accepts null, only to fail later on almost all APIs! 29 new JSONTokener(null).back(); 30 31 try { 32 new JSONTokener(null).more(); 33 fail(); 34 } catch (NullPointerException e) { 35 } 36 37 try { 38 new JSONTokener(null).next(); 39 fail(); 40 } catch (NullPointerException e) { 41 } 42 43 try { 44 new JSONTokener(null).next(3); 45 fail(); 46 } catch (NullPointerException e) { 47 } 48 49 try { 50 new JSONTokener(null).next('A'); 51 fail(); 52 } catch (NullPointerException e) { 53 } 54 55 try { 56 new JSONTokener(null).nextClean(); 57 fail(); 58 } catch (NullPointerException e) { 59 } 60 61 try { 62 new JSONTokener(null).nextString('"'); 63 fail(); 64 } catch (NullPointerException e) { 65 } 66 67 try { 68 new JSONTokener(null).nextTo('A'); 69 fail(); 70 } catch (NullPointerException e) { 71 } 72 73 try { 74 new JSONTokener(null).nextTo("ABC"); 75 fail(); 76 } catch (NullPointerException e) { 77 } 78 79 try { 80 new JSONTokener(null).nextValue(); 81 fail(); 82 } catch (NullPointerException e) { 83 } 84 85 try { 86 new JSONTokener(null).skipPast("ABC"); 87 fail(); 88 } catch (NullPointerException e) { 89 } 90 91 try { 92 new JSONTokener(null).skipTo('A'); 93 fail(); 94 } catch (NullPointerException e) { 95 } 96 97 assertEquals("foo! at character 0 of null", 98 new JSONTokener(null).syntaxError("foo!").getMessage()); 99 100 assertEquals(" at character 0 of null", new JSONTokener(null).toString()); 101 } 102 103 public void testEmptyString() throws JSONException { 104 JSONTokener backTokener = new JSONTokener(""); 105 backTokener.back(); 106 assertEquals(" at character 0 of ", backTokener.toString()); 107 assertFalse(new JSONTokener("").more()); 108 assertEquals('\0', new JSONTokener("").next()); 109 try { 110 new JSONTokener("").next(3); 111 fail(); 112 } catch (JSONException expected) { 113 } 114 try { 115 new JSONTokener("").next('A'); 116 fail(); 117 } catch (JSONException e) { 118 } 119 assertEquals('\0', new JSONTokener("").nextClean()); 120 try { 121 new JSONTokener("").nextString('"'); 122 fail(); 123 } catch (JSONException e) { 124 } 125 assertEquals("", new JSONTokener("").nextTo('A')); 126 assertEquals("", new JSONTokener("").nextTo("ABC")); 127 try { 128 new JSONTokener("").nextValue(); 129 fail(); 130 } catch (JSONException e) { 131 } 132 new JSONTokener("").skipPast("ABC"); 133 assertEquals('\0', new JSONTokener("").skipTo('A')); 134 assertEquals("foo! at character 0 of ", 135 new JSONTokener("").syntaxError("foo!").getMessage()); 136 assertEquals(" at character 0 of ", new JSONTokener("").toString()); 137 } 138 139 public void testCharacterNavigation() throws JSONException { 140 JSONTokener abcdeTokener = new JSONTokener("ABCDE"); 141 assertEquals('A', abcdeTokener.next()); 142 assertEquals('B', abcdeTokener.next('B')); 143 assertEquals("CD", abcdeTokener.next(2)); 144 try { 145 abcdeTokener.next(2); 146 fail(); 147 } catch (JSONException e) { 148 } 149 assertEquals('E', abcdeTokener.nextClean()); 150 assertEquals('\0', abcdeTokener.next()); 151 assertFalse(abcdeTokener.more()); 152 abcdeTokener.back(); 153 assertTrue(abcdeTokener.more()); 154 assertEquals('E', abcdeTokener.next()); 155 } 156 157 public void testBackNextAndMore() throws JSONException { 158 JSONTokener abcTokener = new JSONTokener("ABC"); 159 assertTrue(abcTokener.more()); 160 abcTokener.next(); 161 abcTokener.next(); 162 assertTrue(abcTokener.more()); 163 abcTokener.next(); 164 assertFalse(abcTokener.more()); 165 abcTokener.back(); 166 assertTrue(abcTokener.more()); 167 abcTokener.next(); 168 assertFalse(abcTokener.more()); 169 abcTokener.back(); 170 abcTokener.back(); 171 abcTokener.back(); 172 abcTokener.back(); // you can back up before the beginning of a String! 173 assertEquals('A', abcTokener.next()); 174 } 175 176 public void testNextMatching() throws JSONException { 177 JSONTokener abcdTokener = new JSONTokener("ABCD"); 178 assertEquals('A', abcdTokener.next('A')); 179 try { 180 abcdTokener.next('C'); // although it failed, this op consumes a character of input 181 fail(); 182 } catch (JSONException e) { 183 } 184 assertEquals('C', abcdTokener.next('C')); 185 assertEquals('D', abcdTokener.next('D')); 186 try { 187 abcdTokener.next('E'); 188 fail(); 189 } catch (JSONException e) { 190 } 191 } 192 193 public void testNextN() throws JSONException { 194 JSONTokener abcdeTokener = new JSONTokener("ABCDEF"); 195 assertEquals("", abcdeTokener.next(0)); 196 try { 197 abcdeTokener.next(7); 198 fail(); 199 } catch (JSONException e) { 200 } 201 assertEquals("ABC", abcdeTokener.next(3)); 202 try { 203 abcdeTokener.next(4); 204 fail(); 205 } catch (JSONException e) { 206 } 207 } 208 209 public void testNextNWithAllRemaining() throws JSONException { 210 JSONTokener tokener = new JSONTokener("ABCDEF"); 211 tokener.next(3); 212 try { 213 tokener.next(3); 214 } catch (JSONException e) { 215 AssertionFailedError error = new AssertionFailedError("off-by-one error?"); 216 error.initCause(e); 217 throw error; 218 } 219 } 220 221 public void testNext0() throws JSONException { 222 JSONTokener tokener = new JSONTokener("ABCDEF"); 223 tokener.next(5); 224 tokener.next(); 225 try { 226 tokener.next(0); 227 } catch (JSONException e) { 228 Error error = new AssertionFailedError("Returning an empty string should be valid"); 229 error.initCause(e); 230 throw error; 231 } 232 } 233 234 public void testNextCleanComments() throws JSONException { 235 JSONTokener tokener = new JSONTokener( 236 " A /*XX*/B/*XX//XX\n//XX\nXX*/C//X//X//X\nD/*X*///X\n"); 237 assertEquals('A', tokener.nextClean()); 238 assertEquals('B', tokener.nextClean()); 239 assertEquals('C', tokener.nextClean()); 240 assertEquals('D', tokener.nextClean()); 241 assertEquals('\0', tokener.nextClean()); 242 } 243 244 public void testNextCleanNestedCStyleComments() throws JSONException { 245 JSONTokener tokener = new JSONTokener("A /* B /* C */ D */ E"); 246 assertEquals('A', tokener.nextClean()); 247 assertEquals('D', tokener.nextClean()); 248 assertEquals('*', tokener.nextClean()); 249 assertEquals('/', tokener.nextClean()); 250 assertEquals('E', tokener.nextClean()); 251 } 252 253 /** 254 * Some applications rely on parsing '#' to lead an end-of-line comment. 255 * http://b/2571423 256 */ 257 public void testNextCleanHashComments() throws JSONException { 258 JSONTokener tokener = new JSONTokener("A # B */ /* C */ \nD #"); 259 assertEquals('A', tokener.nextClean()); 260 assertEquals('D', tokener.nextClean()); 261 assertEquals('\0', tokener.nextClean()); 262 } 263 264 public void testNextCleanCommentsTrailingSingleSlash() throws JSONException { 265 JSONTokener tokener = new JSONTokener(" / S /"); 266 assertEquals('/', tokener.nextClean()); 267 assertEquals('S', tokener.nextClean()); 268 assertEquals('/', tokener.nextClean()); 269 assertEquals("nextClean doesn't consume a trailing slash", 270 '\0', tokener.nextClean()); 271 } 272 273 public void testNextCleanTrailingOpenComment() throws JSONException { 274 try { 275 new JSONTokener(" /* ").nextClean(); 276 fail(); 277 } catch (JSONException e) { 278 } 279 assertEquals('\0', new JSONTokener(" // ").nextClean()); 280 } 281 282 public void testNextCleanNewlineDelimiters() throws JSONException { 283 assertEquals('B', new JSONTokener(" // \r\n B ").nextClean()); 284 assertEquals('B', new JSONTokener(" // \n B ").nextClean()); 285 assertEquals('B', new JSONTokener(" // \r B ").nextClean()); 286 } 287 288 public void testNextCleanSkippedWhitespace() throws JSONException { 289 assertEquals("character tabulation", 'A', new JSONTokener("\tA").nextClean()); 290 assertEquals("line feed", 'A', new JSONTokener("\nA").nextClean()); 291 assertEquals("carriage return", 'A', new JSONTokener("\rA").nextClean()); 292 assertEquals("space", 'A', new JSONTokener(" A").nextClean()); 293 } 294 295 /** 296 * Tests which characters tokener treats as ignorable whitespace. See Kevin Bourrillion's 297 * <a href="https://spreadsheets.google.com/pub?key=pd8dAQyHbdewRsnE5x5GzKQ">list 298 * of whitespace characters</a>. 299 */ 300 public void testNextCleanRetainedWhitespace() throws JSONException { 301 assertNotClean("null", '\u0000'); 302 assertNotClean("next line", '\u0085'); 303 assertNotClean("non-breaking space", '\u00a0'); 304 assertNotClean("ogham space mark", '\u1680'); 305 assertNotClean("mongolian vowel separator", '\u180e'); 306 assertNotClean("en quad", '\u2000'); 307 assertNotClean("em quad", '\u2001'); 308 assertNotClean("en space", '\u2002'); 309 assertNotClean("em space", '\u2003'); 310 assertNotClean("three-per-em space", '\u2004'); 311 assertNotClean("four-per-em space", '\u2005'); 312 assertNotClean("six-per-em space", '\u2006'); 313 assertNotClean("figure space", '\u2007'); 314 assertNotClean("punctuation space", '\u2008'); 315 assertNotClean("thin space", '\u2009'); 316 assertNotClean("hair space", '\u200a'); 317 assertNotClean("zero-width space", '\u200b'); 318 assertNotClean("left-to-right mark", '\u200e'); 319 assertNotClean("right-to-left mark", '\u200f'); 320 assertNotClean("line separator", '\u2028'); 321 assertNotClean("paragraph separator", '\u2029'); 322 assertNotClean("narrow non-breaking space", '\u202f'); 323 assertNotClean("medium mathematical space", '\u205f'); 324 assertNotClean("ideographic space", '\u3000'); 325 assertNotClean("line tabulation", '\u000b'); 326 assertNotClean("form feed", '\u000c'); 327 assertNotClean("information separator 4", '\u001c'); 328 assertNotClean("information separator 3", '\u001d'); 329 assertNotClean("information separator 2", '\u001e'); 330 assertNotClean("information separator 1", '\u001f'); 331 } 332 333 private void assertNotClean(String name, char c) throws JSONException { 334 assertEquals("The character " + name + " is not whitespace according to the JSON spec.", 335 c, new JSONTokener(new String(new char[] { c, 'A' })).nextClean()); 336 } 337 338 public void testNextString() throws JSONException { 339 assertEquals("", new JSONTokener("'").nextString('\'')); 340 assertEquals("", new JSONTokener("\"").nextString('\"')); 341 assertEquals("ABC", new JSONTokener("ABC'DEF").nextString('\'')); 342 assertEquals("ABC", new JSONTokener("ABC'''DEF").nextString('\'')); 343 344 // nextString permits slash-escaping of arbitrary characters! 345 assertEquals("ABC", new JSONTokener("A\\B\\C'DEF").nextString('\'')); 346 347 JSONTokener tokener = new JSONTokener(" 'abc' 'def' \"ghi\""); 348 tokener.next(); 349 assertEquals('\'', tokener.next()); 350 assertEquals("abc", tokener.nextString('\'')); 351 tokener.next(); 352 assertEquals('\'', tokener.next()); 353 assertEquals("def", tokener.nextString('\'')); 354 tokener.next(); 355 assertEquals('"', tokener.next()); 356 assertEquals("ghi", tokener.nextString('\"')); 357 assertFalse(tokener.more()); 358 } 359 360 public void testNextStringNoDelimiter() throws JSONException { 361 try { 362 new JSONTokener("").nextString('\''); 363 fail(); 364 } catch (JSONException e) { 365 } 366 367 JSONTokener tokener = new JSONTokener(" 'abc"); 368 tokener.next(); 369 tokener.next(); 370 try { 371 tokener.next('\''); 372 fail(); 373 } catch (JSONException e) { 374 } 375 } 376 377 public void testNextStringEscapedQuote() throws JSONException { 378 try { 379 new JSONTokener("abc\\").nextString('"'); 380 fail(); 381 } catch (JSONException e) { 382 } 383 384 // we're mixing Java escaping like \" and JavaScript escaping like \\\" 385 // which makes these tests extra tricky to read! 386 assertEquals("abc\"def", new JSONTokener("abc\\\"def\"ghi").nextString('"')); 387 assertEquals("abc\\def", new JSONTokener("abc\\\\def\"ghi").nextString('"')); 388 assertEquals("abc/def", new JSONTokener("abc\\/def\"ghi").nextString('"')); 389 assertEquals("abc\bdef", new JSONTokener("abc\\bdef\"ghi").nextString('"')); 390 assertEquals("abc\fdef", new JSONTokener("abc\\fdef\"ghi").nextString('"')); 391 assertEquals("abc\ndef", new JSONTokener("abc\\ndef\"ghi").nextString('"')); 392 assertEquals("abc\rdef", new JSONTokener("abc\\rdef\"ghi").nextString('"')); 393 assertEquals("abc\tdef", new JSONTokener("abc\\tdef\"ghi").nextString('"')); 394 } 395 396 public void testNextStringUnicodeEscaped() throws JSONException { 397 // we're mixing Java escaping like \\ and JavaScript escaping like \\u 398 assertEquals("abc def", new JSONTokener("abc\\u0020def\"ghi").nextString('"')); 399 assertEquals("abcU0020def", new JSONTokener("abc\\U0020def\"ghi").nextString('"')); 400 401 // JSON requires 4 hex characters after a unicode escape 402 try { 403 new JSONTokener("abc\\u002\"").nextString('"'); 404 fail(); 405 } catch (JSONException e) { 406 } 407 try { 408 new JSONTokener("abc\\u").nextString('"'); 409 fail(); 410 } catch (JSONException e) { 411 } 412 try { 413 new JSONTokener("abc\\u \"").nextString('"'); 414 fail(); 415 } catch (JSONException e) { 416 } 417 assertEquals("abc\"def", new JSONTokener("abc\\u0022def\"ghi").nextString('"')); 418 try { 419 new JSONTokener("abc\\u000G\"").nextString('"'); 420 fail(); 421 } catch (JSONException e) { 422 } 423 } 424 425 public void testNextStringNonQuote() throws JSONException { 426 assertEquals("AB", new JSONTokener("ABC").nextString('C')); 427 assertEquals("ABCD", new JSONTokener("AB\\CDC").nextString('C')); 428 assertEquals("AB\nC", new JSONTokener("AB\\nCn").nextString('n')); 429 } 430 431 public void testNextTo() throws JSONException { 432 assertEquals("ABC", new JSONTokener("ABCDEFG").nextTo("DHI")); 433 assertEquals("ABCDEF", new JSONTokener("ABCDEF").nextTo("")); 434 435 JSONTokener tokener = new JSONTokener("ABC\rDEF\nGHI\r\nJKL"); 436 assertEquals("ABC", tokener.nextTo("M")); 437 assertEquals('\r', tokener.next()); 438 assertEquals("DEF", tokener.nextTo("M")); 439 assertEquals('\n', tokener.next()); 440 assertEquals("GHI", tokener.nextTo("M")); 441 assertEquals('\r', tokener.next()); 442 assertEquals('\n', tokener.next()); 443 assertEquals("JKL", tokener.nextTo("M")); 444 445 tokener = new JSONTokener("ABCDEFGHI"); 446 assertEquals("ABC", tokener.nextTo("DEF")); 447 assertEquals("", tokener.nextTo("DEF")); 448 assertEquals('D', tokener.next()); 449 assertEquals("", tokener.nextTo("DEF")); 450 assertEquals('E', tokener.next()); 451 assertEquals("", tokener.nextTo("DEF")); 452 assertEquals('F', tokener.next()); 453 assertEquals("GHI", tokener.nextTo("DEF")); 454 assertEquals("", tokener.nextTo("DEF")); 455 456 tokener = new JSONTokener(" \t \fABC \t DEF"); 457 assertEquals("ABC", tokener.nextTo("DEF")); 458 assertEquals('D', tokener.next()); 459 460 tokener = new JSONTokener(" \t \fABC \n DEF"); 461 assertEquals("ABC", tokener.nextTo("\n")); 462 assertEquals("", tokener.nextTo("\n")); 463 464 tokener = new JSONTokener(""); 465 try { 466 tokener.nextTo(null); 467 fail(); 468 } catch (NullPointerException e) { 469 } 470 } 471 472 public void testNextToTrimming() { 473 assertEquals("ABC", new JSONTokener("\t ABC \tDEF").nextTo("DE")); 474 assertEquals("ABC", new JSONTokener("\t ABC \tDEF").nextTo('D')); 475 } 476 477 public void testNextToTrailing() { 478 assertEquals("ABC DEF", new JSONTokener("\t ABC DEF \t").nextTo("G")); 479 assertEquals("ABC DEF", new JSONTokener("\t ABC DEF \t").nextTo('G')); 480 } 481 482 public void testNextToDoesntStopOnNull() { 483 String message = "nextTo() shouldn't stop after \\0 characters"; 484 JSONTokener tokener = new JSONTokener(" \0\t \fABC \n DEF"); 485 assertEquals(message, "ABC", tokener.nextTo("D")); 486 assertEquals(message, '\n', tokener.next()); 487 assertEquals(message, "", tokener.nextTo("D")); 488 } 489 490 public void testNextToConsumesNull() { 491 String message = "nextTo shouldn't consume \\0."; 492 JSONTokener tokener = new JSONTokener("ABC\0DEF"); 493 assertEquals(message, "ABC", tokener.nextTo("\0")); 494 assertEquals(message, '\0', tokener.next()); 495 assertEquals(message, "DEF", tokener.nextTo("\0")); 496 } 497 498 public void testSkipPast() { 499 JSONTokener tokener = new JSONTokener("ABCDEF"); 500 tokener.skipPast("ABC"); 501 assertEquals('D', tokener.next()); 502 tokener.skipPast("EF"); 503 assertEquals('\0', tokener.next()); 504 505 tokener = new JSONTokener("ABCDEF"); 506 tokener.skipPast("ABCDEF"); 507 assertEquals('\0', tokener.next()); 508 509 tokener = new JSONTokener("ABCDEF"); 510 tokener.skipPast("G"); 511 assertEquals('\0', tokener.next()); 512 513 tokener = new JSONTokener("ABC\0ABC"); 514 tokener.skipPast("ABC"); 515 assertEquals('\0', tokener.next()); 516 assertEquals('A', tokener.next()); 517 518 tokener = new JSONTokener("\0ABC"); 519 tokener.skipPast("ABC"); 520 assertEquals('\0', tokener.next()); 521 522 tokener = new JSONTokener("ABC\nDEF"); 523 tokener.skipPast("DEF"); 524 assertEquals('\0', tokener.next()); 525 526 tokener = new JSONTokener("ABC"); 527 tokener.skipPast("ABCDEF"); 528 assertEquals('\0', tokener.next()); 529 530 tokener = new JSONTokener("ABCDABCDABCD"); 531 tokener.skipPast("ABC"); 532 assertEquals('D', tokener.next()); 533 tokener.skipPast("ABC"); 534 assertEquals('D', tokener.next()); 535 tokener.skipPast("ABC"); 536 assertEquals('D', tokener.next()); 537 538 tokener = new JSONTokener(""); 539 try { 540 tokener.skipPast(null); 541 fail(); 542 } catch (NullPointerException e) { 543 } 544 } 545 546 public void testSkipTo() { 547 JSONTokener tokener = new JSONTokener("ABCDEF"); 548 tokener.skipTo('A'); 549 assertEquals('A', tokener.next()); 550 tokener.skipTo('D'); 551 assertEquals('D', tokener.next()); 552 tokener.skipTo('G'); 553 assertEquals('E', tokener.next()); 554 tokener.skipTo('A'); 555 assertEquals('F', tokener.next()); 556 557 tokener = new JSONTokener("ABC\nDEF"); 558 tokener.skipTo('F'); 559 assertEquals('F', tokener.next()); 560 561 tokener = new JSONTokener("ABCfDEF"); 562 tokener.skipTo('F'); 563 assertEquals('F', tokener.next()); 564 565 tokener = new JSONTokener("ABC/* DEF */"); 566 tokener.skipTo('D'); 567 assertEquals('D', tokener.next()); 568 } 569 570 public void testSkipToStopsOnNull() { 571 JSONTokener tokener = new JSONTokener("ABC\0DEF"); 572 tokener.skipTo('F'); 573 assertEquals("skipTo shouldn't stop when it sees '\\0'", 'F', tokener.next()); 574 } 575 576 public void testBomIgnoredAsFirstCharacterOfDocument() throws JSONException { 577 JSONTokener tokener = new JSONTokener("\ufeff[]"); 578 JSONArray array = (JSONArray) tokener.nextValue(); 579 assertEquals(0, array.length()); 580 } 581 582 public void testBomTreatedAsCharacterInRestOfDocument() throws JSONException { 583 JSONTokener tokener = new JSONTokener("[\ufeff]"); 584 JSONArray array = (JSONArray) tokener.nextValue(); 585 assertEquals(1, array.length()); 586 } 587 588 public void testDehexchar() { 589 assertEquals( 0, JSONTokener.dehexchar('0')); 590 assertEquals( 1, JSONTokener.dehexchar('1')); 591 assertEquals( 2, JSONTokener.dehexchar('2')); 592 assertEquals( 3, JSONTokener.dehexchar('3')); 593 assertEquals( 4, JSONTokener.dehexchar('4')); 594 assertEquals( 5, JSONTokener.dehexchar('5')); 595 assertEquals( 6, JSONTokener.dehexchar('6')); 596 assertEquals( 7, JSONTokener.dehexchar('7')); 597 assertEquals( 8, JSONTokener.dehexchar('8')); 598 assertEquals( 9, JSONTokener.dehexchar('9')); 599 assertEquals(10, JSONTokener.dehexchar('A')); 600 assertEquals(11, JSONTokener.dehexchar('B')); 601 assertEquals(12, JSONTokener.dehexchar('C')); 602 assertEquals(13, JSONTokener.dehexchar('D')); 603 assertEquals(14, JSONTokener.dehexchar('E')); 604 assertEquals(15, JSONTokener.dehexchar('F')); 605 assertEquals(10, JSONTokener.dehexchar('a')); 606 assertEquals(11, JSONTokener.dehexchar('b')); 607 assertEquals(12, JSONTokener.dehexchar('c')); 608 assertEquals(13, JSONTokener.dehexchar('d')); 609 assertEquals(14, JSONTokener.dehexchar('e')); 610 assertEquals(15, JSONTokener.dehexchar('f')); 611 612 for (int c = 0; c <= 0xFFFF; c++) { 613 if ((c >= '0' && c <= '9') || (c >= 'A' && c <= 'F') || (c >= 'a' && c <= 'f')) { 614 continue; 615 } 616 assertEquals("dehexchar " + c, -1, JSONTokener.dehexchar((char) c)); 617 } 618 } 619} 620