1// Copyright (C) 2016 and later: Unicode, Inc. and others. 2// License & terms of use: http://www.unicode.org/copyright.html 3/******************************************************************** 4 * COPYRIGHT: 5 * Copyright (c) 1997-2016, International Business Machines Corporation and 6 * others. All Rights Reserved. 7 ********************************************************************/ 8 9#include "unicode/utypes.h" 10 11#if !UCONFIG_NO_NORMALIZATION 12 13#include "unicode/uchar.h" 14#include "unicode/errorcode.h" 15#include "unicode/normlzr.h" 16#include "unicode/uniset.h" 17#include "unicode/usetiter.h" 18#include "unicode/schriter.h" 19#include "unicode/utf16.h" 20#include "cmemory.h" 21#include "cstring.h" 22#include "normalizer2impl.h" 23#include "tstnorm.h" 24 25#define ARRAY_LENGTH(array) UPRV_LENGTHOF(array) 26 27#define CASE(id,test) case id: \ 28 name = #test; \ 29 if (exec) { \ 30 logln(#test "---"); \ 31 logln((UnicodeString)""); \ 32 test(); \ 33 } \ 34 break 35 36static UErrorCode status = U_ZERO_ERROR; 37 38void BasicNormalizerTest::runIndexedTest(int32_t index, UBool exec, 39 const char* &name, char* /*par*/) { 40 switch (index) { 41 CASE(0,TestDecomp); 42 CASE(1,TestCompatDecomp); 43 CASE(2,TestCanonCompose); 44 CASE(3,TestCompatCompose); 45 CASE(4,TestPrevious); 46 CASE(5,TestHangulDecomp); 47 CASE(6,TestHangulCompose); 48 CASE(7,TestTibetan); 49 CASE(8,TestCompositionExclusion); 50 CASE(9,TestZeroIndex); 51 CASE(10,TestVerisign); 52 CASE(11,TestPreviousNext); 53 CASE(12,TestNormalizerAPI); 54 CASE(13,TestConcatenate); 55 CASE(14,FindFoldFCDExceptions); 56 CASE(15,TestCompare); 57 CASE(16,TestSkippable); 58#if !UCONFIG_NO_FILE_IO && !UCONFIG_NO_LEGACY_CONVERSION 59 CASE(17,TestCustomComp); 60 CASE(18,TestCustomFCC); 61#endif 62 CASE(19,TestFilteredNormalizer2Coverage); 63 default: name = ""; break; 64 } 65} 66 67/** 68 * Convert Java-style strings with \u Unicode escapes into UnicodeString objects 69 */ 70static UnicodeString str(const char *input) 71{ 72 UnicodeString str(input, ""); // Invariant conversion 73 return str.unescape(); 74} 75 76 77BasicNormalizerTest::BasicNormalizerTest() 78{ 79 // canonTest 80 // Input Decomposed Composed 81 82 canonTests[0][0] = str("cat"); canonTests[0][1] = str("cat"); canonTests[0][2] = str("cat"); 83 84 canonTests[1][0] = str("\\u00e0ardvark"); canonTests[1][1] = str("a\\u0300ardvark"); canonTests[1][2] = str("\\u00e0ardvark"); 85 86 canonTests[2][0] = str("\\u1e0a"); canonTests[2][1] = str("D\\u0307"); canonTests[2][2] = str("\\u1e0a"); // D-dot_above 87 88 canonTests[3][0] = str("D\\u0307"); canonTests[3][1] = str("D\\u0307"); canonTests[3][2] = str("\\u1e0a"); // D dot_above 89 90 canonTests[4][0] = str("\\u1e0c\\u0307"); canonTests[4][1] = str("D\\u0323\\u0307"); canonTests[4][2] = str("\\u1e0c\\u0307"); // D-dot_below dot_above 91 92 canonTests[5][0] = str("\\u1e0a\\u0323"); canonTests[5][1] = str("D\\u0323\\u0307"); canonTests[5][2] = str("\\u1e0c\\u0307"); // D-dot_above dot_below 93 94 canonTests[6][0] = str("D\\u0307\\u0323"); canonTests[6][1] = str("D\\u0323\\u0307"); canonTests[6][2] = str("\\u1e0c\\u0307"); // D dot_below dot_above 95 96 canonTests[7][0] = str("\\u1e10\\u0307\\u0323"); canonTests[7][1] = str("D\\u0327\\u0323\\u0307"); canonTests[7][2] = str("\\u1e10\\u0323\\u0307"); // D dot_below cedilla dot_above 97 98 canonTests[8][0] = str("D\\u0307\\u0328\\u0323"); canonTests[8][1] = str("D\\u0328\\u0323\\u0307"); canonTests[8][2] = str("\\u1e0c\\u0328\\u0307"); // D dot_above ogonek dot_below 99 100 canonTests[9][0] = str("\\u1E14"); canonTests[9][1] = str("E\\u0304\\u0300"); canonTests[9][2] = str("\\u1E14"); // E-macron-grave 101 102 canonTests[10][0] = str("\\u0112\\u0300"); canonTests[10][1] = str("E\\u0304\\u0300"); canonTests[10][2] = str("\\u1E14"); // E-macron + grave 103 104 canonTests[11][0] = str("\\u00c8\\u0304"); canonTests[11][1] = str("E\\u0300\\u0304"); canonTests[11][2] = str("\\u00c8\\u0304"); // E-grave + macron 105 106 canonTests[12][0] = str("\\u212b"); canonTests[12][1] = str("A\\u030a"); canonTests[12][2] = str("\\u00c5"); // angstrom_sign 107 108 canonTests[13][0] = str("\\u00c5"); canonTests[13][1] = str("A\\u030a"); canonTests[13][2] = str("\\u00c5"); // A-ring 109 110 canonTests[14][0] = str("\\u00C4ffin"); canonTests[14][1] = str("A\\u0308ffin"); canonTests[14][2] = str("\\u00C4ffin"); 111 112 canonTests[15][0] = str("\\u00C4\\uFB03n"); canonTests[15][1] = str("A\\u0308\\uFB03n"); canonTests[15][2] = str("\\u00C4\\uFB03n"); 113 114 canonTests[16][0] = str("Henry IV"); canonTests[16][1] = str("Henry IV"); canonTests[16][2] = str("Henry IV"); 115 116 canonTests[17][0] = str("Henry \\u2163"); canonTests[17][1] = str("Henry \\u2163"); canonTests[17][2] = str("Henry \\u2163"); 117 118 canonTests[18][0] = str("\\u30AC"); canonTests[18][1] = str("\\u30AB\\u3099"); canonTests[18][2] = str("\\u30AC"); // ga (Katakana) 119 120 canonTests[19][0] = str("\\u30AB\\u3099"); canonTests[19][1] = str("\\u30AB\\u3099"); canonTests[19][2] = str("\\u30AC"); // ka + ten 121 122 canonTests[20][0] = str("\\uFF76\\uFF9E"); canonTests[20][1] = str("\\uFF76\\uFF9E"); canonTests[20][2] = str("\\uFF76\\uFF9E"); // hw_ka + hw_ten 123 124 canonTests[21][0] = str("\\u30AB\\uFF9E"); canonTests[21][1] = str("\\u30AB\\uFF9E"); canonTests[21][2] = str("\\u30AB\\uFF9E"); // ka + hw_ten 125 126 canonTests[22][0] = str("\\uFF76\\u3099"); canonTests[22][1] = str("\\uFF76\\u3099"); canonTests[22][2] = str("\\uFF76\\u3099"); // hw_ka + ten 127 128 canonTests[23][0] = str("A\\u0300\\u0316"); canonTests[23][1] = str("A\\u0316\\u0300"); canonTests[23][2] = str("\\u00C0\\u0316"); 129 130 /* compatTest */ 131 // Input Decomposed Composed 132 compatTests[0][0] = str("cat"); compatTests[0][1] = str("cat"); compatTests[0][2] = str("cat") ; 133 134 compatTests[1][0] = str("\\uFB4f"); compatTests[1][1] = str("\\u05D0\\u05DC"); compatTests[1][2] = str("\\u05D0\\u05DC"); // Alef-Lamed vs. Alef, Lamed 135 136 compatTests[2][0] = str("\\u00C4ffin"); compatTests[2][1] = str("A\\u0308ffin"); compatTests[2][2] = str("\\u00C4ffin") ; 137 138 compatTests[3][0] = str("\\u00C4\\uFB03n"); compatTests[3][1] = str("A\\u0308ffin"); compatTests[3][2] = str("\\u00C4ffin") ; // ffi ligature -> f + f + i 139 140 compatTests[4][0] = str("Henry IV"); compatTests[4][1] = str("Henry IV"); compatTests[4][2] = str("Henry IV") ; 141 142 compatTests[5][0] = str("Henry \\u2163"); compatTests[5][1] = str("Henry IV"); compatTests[5][2] = str("Henry IV") ; 143 144 compatTests[6][0] = str("\\u30AC"); compatTests[6][1] = str("\\u30AB\\u3099"); compatTests[6][2] = str("\\u30AC") ; // ga (Katakana) 145 146 compatTests[7][0] = str("\\u30AB\\u3099"); compatTests[7][1] = str("\\u30AB\\u3099"); compatTests[7][2] = str("\\u30AC") ; // ka + ten 147 148 compatTests[8][0] = str("\\uFF76\\u3099"); compatTests[8][1] = str("\\u30AB\\u3099"); compatTests[8][2] = str("\\u30AC") ; // hw_ka + ten 149 150 /* These two are broken in Unicode 2.1.2 but fixed in 2.1.5 and later */ 151 compatTests[9][0] = str("\\uFF76\\uFF9E"); compatTests[9][1] = str("\\u30AB\\u3099"); compatTests[9][2] = str("\\u30AC") ; // hw_ka + hw_ten 152 153 compatTests[10][0] = str("\\u30AB\\uFF9E"); compatTests[10][1] = str("\\u30AB\\u3099"); compatTests[10][2] = str("\\u30AC") ; // ka + hw_ten 154 155 /* Hangul Canonical */ 156 // Input Decomposed Composed 157 hangulCanon[0][0] = str("\\ud4db"); hangulCanon[0][1] = str("\\u1111\\u1171\\u11b6"); hangulCanon[0][2] = str("\\ud4db") ; 158 159 hangulCanon[1][0] = str("\\u1111\\u1171\\u11b6"), hangulCanon[1][1] = str("\\u1111\\u1171\\u11b6"), hangulCanon[1][2] = str("\\ud4db"); 160} 161 162BasicNormalizerTest::~BasicNormalizerTest() 163{ 164} 165 166void BasicNormalizerTest::TestPrevious() 167{ 168 Normalizer* norm = new Normalizer("", UNORM_NFD); 169 170 logln("testing decomp..."); 171 uint32_t i; 172 for (i = 0; i < ARRAY_LENGTH(canonTests); i++) { 173 backAndForth(norm, canonTests[i][0]); 174 } 175 176 logln("testing compose..."); 177 norm->setMode(UNORM_NFC); 178 for (i = 0; i < ARRAY_LENGTH(canonTests); i++) { 179 backAndForth(norm, canonTests[i][0]); 180 } 181 182 delete norm; 183} 184 185void BasicNormalizerTest::TestDecomp() 186{ 187 Normalizer* norm = new Normalizer("", UNORM_NFD); 188 iterateTest(norm, canonTests, ARRAY_LENGTH(canonTests), 1); 189 staticTest(UNORM_NFD, 0, canonTests, ARRAY_LENGTH(canonTests), 1); 190 delete norm; 191} 192 193void BasicNormalizerTest::TestCompatDecomp() 194{ 195 Normalizer* norm = new Normalizer("", UNORM_NFKD); 196 iterateTest(norm, compatTests, ARRAY_LENGTH(compatTests), 1); 197 198 staticTest(UNORM_NFKD, 0, 199 compatTests, ARRAY_LENGTH(compatTests), 1); 200 delete norm; 201} 202 203void BasicNormalizerTest::TestCanonCompose() 204{ 205 Normalizer* norm = new Normalizer("", UNORM_NFC); 206 iterateTest(norm, canonTests, ARRAY_LENGTH(canonTests), 2); 207 208 staticTest(UNORM_NFC, 0, canonTests, 209 ARRAY_LENGTH(canonTests), 2); 210 delete norm; 211} 212 213void BasicNormalizerTest::TestCompatCompose() 214{ 215 Normalizer* norm = new Normalizer("", UNORM_NFKC); 216 iterateTest(norm, compatTests, ARRAY_LENGTH(compatTests), 2); 217 218 staticTest(UNORM_NFKC, 0, 219 compatTests, ARRAY_LENGTH(compatTests), 2); 220 delete norm; 221} 222 223 224//------------------------------------------------------------------------------- 225 226void BasicNormalizerTest::TestHangulCompose() 227{ 228 // Make sure that the static composition methods work 229 logln("Canonical composition..."); 230 staticTest(UNORM_NFC, 0, hangulCanon, ARRAY_LENGTH(hangulCanon), 2); 231 logln("Compatibility composition..."); 232 233 // Now try iterative composition.... 234 logln("Static composition..."); 235 Normalizer* norm = new Normalizer("", UNORM_NFC); 236 iterateTest(norm, hangulCanon, ARRAY_LENGTH(hangulCanon), 2); 237 norm->setMode(UNORM_NFKC); 238 239 // And finally, make sure you can do it in reverse too 240 logln("Reverse iteration..."); 241 norm->setMode(UNORM_NFC); 242 for (uint32_t i = 0; i < ARRAY_LENGTH(hangulCanon); i++) { 243 backAndForth(norm, hangulCanon[i][0]); 244 } 245 delete norm; 246} 247 248void BasicNormalizerTest::TestHangulDecomp() 249{ 250 // Make sure that the static decomposition methods work 251 logln("Canonical decomposition..."); 252 staticTest(UNORM_NFD, 0, hangulCanon, ARRAY_LENGTH(hangulCanon), 1); 253 logln("Compatibility decomposition..."); 254 255 // Now the iterative decomposition methods... 256 logln("Iterative decomposition..."); 257 Normalizer* norm = new Normalizer("", UNORM_NFD); 258 iterateTest(norm, hangulCanon, ARRAY_LENGTH(hangulCanon), 1); 259 norm->setMode(UNORM_NFKD); 260 261 // And finally, make sure you can do it in reverse too 262 logln("Reverse iteration..."); 263 norm->setMode(UNORM_NFD); 264 for (uint32_t i = 0; i < ARRAY_LENGTH(hangulCanon); i++) { 265 backAndForth(norm, hangulCanon[i][0]); 266 } 267 delete norm; 268} 269 270/** 271 * The Tibetan vowel sign AA, 0f71, was messed up prior to Unicode version 2.1.9. 272 */ 273void BasicNormalizerTest::TestTibetan(void) { 274 UnicodeString decomp[1][3]; 275 decomp[0][0] = str("\\u0f77"); 276 decomp[0][1] = str("\\u0f77"); 277 decomp[0][2] = str("\\u0fb2\\u0f71\\u0f80"); 278 279 UnicodeString compose[1][3]; 280 compose[0][0] = str("\\u0fb2\\u0f71\\u0f80"); 281 compose[0][1] = str("\\u0fb2\\u0f71\\u0f80"); 282 compose[0][2] = str("\\u0fb2\\u0f71\\u0f80"); 283 284 staticTest(UNORM_NFD, 0, decomp, ARRAY_LENGTH(decomp), 1); 285 staticTest(UNORM_NFKD, 0, decomp, ARRAY_LENGTH(decomp), 2); 286 staticTest(UNORM_NFC, 0, compose, ARRAY_LENGTH(compose), 1); 287 staticTest(UNORM_NFKC, 0, compose, ARRAY_LENGTH(compose), 2); 288} 289 290/** 291 * Make sure characters in the CompositionExclusion.txt list do not get 292 * composed to. 293 */ 294void BasicNormalizerTest::TestCompositionExclusion(void) { 295 // This list is generated from CompositionExclusion.txt. 296 // Update whenever the normalizer tables are updated. Note 297 // that we test all characters listed, even those that can be 298 // derived from the Unicode DB and are therefore commented 299 // out. 300 // ### TODO read composition exclusion from source/data/unidata file 301 // and test against that 302 UnicodeString EXCLUDED = str( 303 "\\u0340\\u0341\\u0343\\u0344\\u0374\\u037E\\u0387\\u0958" 304 "\\u0959\\u095A\\u095B\\u095C\\u095D\\u095E\\u095F\\u09DC" 305 "\\u09DD\\u09DF\\u0A33\\u0A36\\u0A59\\u0A5A\\u0A5B\\u0A5E" 306 "\\u0B5C\\u0B5D\\u0F43\\u0F4D\\u0F52\\u0F57\\u0F5C\\u0F69" 307 "\\u0F73\\u0F75\\u0F76\\u0F78\\u0F81\\u0F93\\u0F9D\\u0FA2" 308 "\\u0FA7\\u0FAC\\u0FB9\\u1F71\\u1F73\\u1F75\\u1F77\\u1F79" 309 "\\u1F7B\\u1F7D\\u1FBB\\u1FBE\\u1FC9\\u1FCB\\u1FD3\\u1FDB" 310 "\\u1FE3\\u1FEB\\u1FEE\\u1FEF\\u1FF9\\u1FFB\\u1FFD\\u2000" 311 "\\u2001\\u2126\\u212A\\u212B\\u2329\\u232A\\uF900\\uFA10" 312 "\\uFA12\\uFA15\\uFA20\\uFA22\\uFA25\\uFA26\\uFA2A\\uFB1F" 313 "\\uFB2A\\uFB2B\\uFB2C\\uFB2D\\uFB2E\\uFB2F\\uFB30\\uFB31" 314 "\\uFB32\\uFB33\\uFB34\\uFB35\\uFB36\\uFB38\\uFB39\\uFB3A" 315 "\\uFB3B\\uFB3C\\uFB3E\\uFB40\\uFB41\\uFB43\\uFB44\\uFB46" 316 "\\uFB47\\uFB48\\uFB49\\uFB4A\\uFB4B\\uFB4C\\uFB4D\\uFB4E" 317 ); 318 for (int32_t i=0; i<EXCLUDED.length(); ++i) { 319 UnicodeString a(EXCLUDED.charAt(i)); 320 UnicodeString b; 321 UnicodeString c; 322 Normalizer::normalize(a, UNORM_NFKD, 0, b, status); 323 Normalizer::normalize(b, UNORM_NFC, 0, c, status); 324 if (c == a) { 325 errln("FAIL: " + hex(a) + " x DECOMP_COMPAT => " + 326 hex(b) + " x COMPOSE => " + 327 hex(c)); 328 } else if (verbose) { 329 logln("Ok: " + hex(a) + " x DECOMP_COMPAT => " + 330 hex(b) + " x COMPOSE => " + 331 hex(c)); 332 } 333 } 334} 335 336/** 337 * Test for a problem that showed up just before ICU 1.6 release 338 * having to do with combining characters with an index of zero. 339 * Such characters do not participate in any canonical 340 * decompositions. However, having an index of zero means that 341 * they all share one typeMask[] entry, that is, they all have to 342 * map to the same canonical class, which is not the case, in 343 * reality. 344 */ 345void BasicNormalizerTest::TestZeroIndex(void) { 346 const char* DATA[] = { 347 // Expect col1 x COMPOSE_COMPAT => col2 348 // Expect col2 x DECOMP => col3 349 "A\\u0316\\u0300", "\\u00C0\\u0316", "A\\u0316\\u0300", 350 "A\\u0300\\u0316", "\\u00C0\\u0316", "A\\u0316\\u0300", 351 "A\\u0327\\u0300", "\\u00C0\\u0327", "A\\u0327\\u0300", 352 "c\\u0321\\u0327", "c\\u0321\\u0327", "c\\u0321\\u0327", 353 "c\\u0327\\u0321", "\\u00E7\\u0321", "c\\u0327\\u0321", 354 }; 355 int32_t DATA_length = UPRV_LENGTHOF(DATA); 356 357 for (int32_t i=0; i<DATA_length; i+=3) { 358 UErrorCode status = U_ZERO_ERROR; 359 UnicodeString a(DATA[i], ""); 360 a = a.unescape(); 361 UnicodeString b; 362 Normalizer::normalize(a, UNORM_NFKC, 0, b, status); 363 if (U_FAILURE(status)) { 364 dataerrln("Error calling normalize UNORM_NFKC: %s", u_errorName(status)); 365 } else { 366 UnicodeString exp(DATA[i+1], ""); 367 exp = exp.unescape(); 368 if (b == exp) { 369 logln((UnicodeString)"Ok: " + hex(a) + " x COMPOSE_COMPAT => " + hex(b)); 370 } else { 371 errln((UnicodeString)"FAIL: " + hex(a) + " x COMPOSE_COMPAT => " + hex(b) + 372 ", expect " + hex(exp)); 373 } 374 } 375 Normalizer::normalize(b, UNORM_NFD, 0, a, status); 376 if (U_FAILURE(status)) { 377 dataerrln("Error calling normalize UNORM_NFD: %s", u_errorName(status)); 378 } else { 379 UnicodeString exp = UnicodeString(DATA[i+2], "").unescape(); 380 if (a == exp) { 381 logln((UnicodeString)"Ok: " + hex(b) + " x DECOMP => " + hex(a)); 382 } else { 383 errln((UnicodeString)"FAIL: " + hex(b) + " x DECOMP => " + hex(a) + 384 ", expect " + hex(exp)); 385 } 386 } 387 } 388} 389 390/** 391 * Run a few specific cases that are failing for Verisign. 392 */ 393void BasicNormalizerTest::TestVerisign(void) { 394 /* 395 > Their input: 396 > 05B8 05B9 05B1 0591 05C3 05B0 05AC 059F 397 > Their output (supposedly from ICU): 398 > 05B8 05B1 05B9 0591 05C3 05B0 05AC 059F 399 > My output from charlint: 400 > 05B1 05B8 05B9 0591 05C3 05B0 05AC 059F 401 402 05B8 05B9 05B1 0591 05C3 05B0 05AC 059F => 05B1 05B8 05B9 0591 05C3 05B0 403 05AC 059F 404 405 U+05B8 18 E HEBREW POINT QAMATS 406 U+05B9 19 F HEBREW POINT HOLAM 407 U+05B1 11 HEBREW POINT HATAF SEGOL 408 U+0591 220 HEBREW ACCENT ETNAHTA 409 U+05C3 0 HEBREW PUNCTUATION SOF PASUQ 410 U+05B0 10 HEBREW POINT SHEVA 411 U+05AC 230 HEBREW ACCENT ILUY 412 U+059F 230 HEBREW ACCENT QARNEY PARA 413 414 U+05B1 11 HEBREW POINT HATAF SEGOL 415 U+05B8 18 HEBREW POINT QAMATS 416 U+05B9 19 HEBREW POINT HOLAM 417 U+0591 220 HEBREW ACCENT ETNAHTA 418 U+05C3 0 HEBREW PUNCTUATION SOF PASUQ 419 U+05B0 10 HEBREW POINT SHEVA 420 U+05AC 230 HEBREW ACCENT ILUY 421 U+059F 230 HEBREW ACCENT QARNEY PARA 422 423 Wrong result: 424 U+05B8 18 HEBREW POINT QAMATS 425 U+05B1 11 HEBREW POINT HATAF SEGOL 426 U+05B9 19 HEBREW POINT HOLAM 427 U+0591 220 HEBREW ACCENT ETNAHTA 428 U+05C3 0 HEBREW PUNCTUATION SOF PASUQ 429 U+05B0 10 HEBREW POINT SHEVA 430 U+05AC 230 HEBREW ACCENT ILUY 431 U+059F 230 HEBREW ACCENT QARNEY PARA 432 433 434 > Their input: 435 >0592 05B7 05BC 05A5 05B0 05C0 05C4 05AD 436 >Their output (supposedly from ICU): 437 >0592 05B0 05B7 05BC 05A5 05C0 05AD 05C4 438 >My output from charlint: 439 >05B0 05B7 05BC 05A5 0592 05C0 05AD 05C4 440 441 0592 05B7 05BC 05A5 05B0 05C0 05C4 05AD => 05B0 05B7 05BC 05A5 0592 05C0 442 05AD 05C4 443 444 U+0592 230 HEBREW ACCENT SEGOL 445 U+05B7 17 HEBREW POINT PATAH 446 U+05BC 21 HEBREW POINT DAGESH OR MAPIQ 447 U+05A5 220 HEBREW ACCENT MERKHA 448 U+05B0 10 HEBREW POINT SHEVA 449 U+05C0 0 HEBREW PUNCTUATION PASEQ 450 U+05C4 230 HEBREW MARK UPPER DOT 451 U+05AD 222 HEBREW ACCENT DEHI 452 453 U+05B0 10 HEBREW POINT SHEVA 454 U+05B7 17 HEBREW POINT PATAH 455 U+05BC 21 HEBREW POINT DAGESH OR MAPIQ 456 U+05A5 220 HEBREW ACCENT MERKHA 457 U+0592 230 HEBREW ACCENT SEGOL 458 U+05C0 0 HEBREW PUNCTUATION PASEQ 459 U+05AD 222 HEBREW ACCENT DEHI 460 U+05C4 230 HEBREW MARK UPPER DOT 461 462 Wrong result: 463 U+0592 230 HEBREW ACCENT SEGOL 464 U+05B0 10 HEBREW POINT SHEVA 465 U+05B7 17 HEBREW POINT PATAH 466 U+05BC 21 HEBREW POINT DAGESH OR MAPIQ 467 U+05A5 220 HEBREW ACCENT MERKHA 468 U+05C0 0 HEBREW PUNCTUATION PASEQ 469 U+05AD 222 HEBREW ACCENT DEHI 470 U+05C4 230 HEBREW MARK UPPER DOT 471 */ 472 UnicodeString data[2][3]; 473 data[0][0] = str("\\u05B8\\u05B9\\u05B1\\u0591\\u05C3\\u05B0\\u05AC\\u059F"); 474 data[0][1] = str("\\u05B1\\u05B8\\u05B9\\u0591\\u05C3\\u05B0\\u05AC\\u059F"); 475 data[0][2] = str(""); 476 data[1][0] = str("\\u0592\\u05B7\\u05BC\\u05A5\\u05B0\\u05C0\\u05C4\\u05AD"); 477 data[1][1] = str("\\u05B0\\u05B7\\u05BC\\u05A5\\u0592\\u05C0\\u05AD\\u05C4"); 478 data[1][2] = str(""); 479 480 staticTest(UNORM_NFD, 0, data, ARRAY_LENGTH(data), 1); 481 staticTest(UNORM_NFC, 0, data, ARRAY_LENGTH(data), 1); 482} 483 484//------------------------------------------------------------------------ 485// Internal utilities 486// 487 488UnicodeString BasicNormalizerTest::hex(UChar ch) { 489 UnicodeString result; 490 return appendHex(ch, 4, result); 491} 492 493UnicodeString BasicNormalizerTest::hex(const UnicodeString& s) { 494 UnicodeString result; 495 for (int i = 0; i < s.length(); ++i) { 496 if (i != 0) result += (UChar)0x2c/*,*/; 497 appendHex(s[i], 4, result); 498 } 499 return result; 500} 501 502 503inline static void insert(UnicodeString& dest, int pos, UChar32 ch) 504{ 505 dest.replace(pos, 0, ch); 506} 507 508void BasicNormalizerTest::backAndForth(Normalizer* iter, const UnicodeString& input) 509{ 510 UChar32 ch; 511 iter->setText(input, status); 512 513 // Run through the iterator forwards and stick it into a StringBuffer 514 UnicodeString forward; 515 for (ch = iter->first(); ch != iter->DONE; ch = iter->next()) { 516 forward += ch; 517 } 518 519 // Now do it backwards 520 UnicodeString reverse; 521 for (ch = iter->last(); ch != iter->DONE; ch = iter->previous()) { 522 insert(reverse, 0, ch); 523 } 524 525 if (forward != reverse) { 526 errln("Forward/reverse mismatch for input " + hex(input) 527 + ", forward: " + hex(forward) + ", backward: " + hex(reverse)); 528 } 529} 530 531void BasicNormalizerTest::staticTest(UNormalizationMode mode, int options, 532 UnicodeString tests[][3], int length, 533 int outCol) 534{ 535 for (int i = 0; i < length; i++) 536 { 537 UnicodeString& input = tests[i][0]; 538 UnicodeString& expect = tests[i][outCol]; 539 540 logln("Normalizing '" + input + "' (" + hex(input) + ")" ); 541 542 UnicodeString output; 543 Normalizer::normalize(input, mode, options, output, status); 544 545 if (output != expect) { 546 dataerrln(UnicodeString("ERROR: case ") + i + " normalized " + hex(input) + "\n" 547 + " expected " + hex(expect) + "\n" 548 + " static got " + hex(output) ); 549 } 550 } 551} 552 553void BasicNormalizerTest::iterateTest(Normalizer* iter, 554 UnicodeString tests[][3], int length, 555 int outCol) 556{ 557 for (int i = 0; i < length; i++) 558 { 559 UnicodeString& input = tests[i][0]; 560 UnicodeString& expect = tests[i][outCol]; 561 562 logln("Normalizing '" + input + "' (" + hex(input) + ")" ); 563 564 iter->setText(input, status); 565 assertEqual(input, expect, iter, UnicodeString("ERROR: case ") + i + " "); 566 } 567} 568 569void BasicNormalizerTest::assertEqual(const UnicodeString& input, 570 const UnicodeString& expected, 571 Normalizer* iter, 572 const UnicodeString& errPrefix) 573{ 574 UnicodeString result; 575 576 for (UChar32 ch = iter->first(); ch != iter->DONE; ch = iter->next()) { 577 result += ch; 578 } 579 if (result != expected) { 580 dataerrln(errPrefix + "normalized " + hex(input) + "\n" 581 + " expected " + hex(expected) + "\n" 582 + " iterate got " + hex(result) ); 583 } 584} 585 586// helper class for TestPreviousNext() 587// simple UTF-32 character iterator 588class UChar32Iterator { 589public: 590 UChar32Iterator(const UChar32 *text, int32_t len, int32_t index) : 591 s(text), length(len), i(index) {} 592 593 UChar32 current() { 594 if(i<length) { 595 return s[i]; 596 } else { 597 return 0xffff; 598 } 599 } 600 601 UChar32 next() { 602 if(i<length) { 603 return s[i++]; 604 } else { 605 return 0xffff; 606 } 607 } 608 609 UChar32 previous() { 610 if(i>0) { 611 return s[--i]; 612 } else { 613 return 0xffff; 614 } 615 } 616 617 int32_t getIndex() { 618 return i; 619 } 620private: 621 const UChar32 *s; 622 int32_t length, i; 623}; 624 625void 626BasicNormalizerTest::TestPreviousNext(const UChar *src, int32_t srcLength, 627 const UChar32 *expect, int32_t expectLength, 628 const int32_t *expectIndex, // its length=expectLength+1 629 int32_t srcMiddle, int32_t expectMiddle, 630 const char *moves, 631 UNormalizationMode mode, 632 const char *name) { 633 // iterators 634 Normalizer iter(src, srcLength, mode); 635 636 // test getStaticClassID and getDynamicClassID 637 if(iter.getDynamicClassID() != Normalizer::getStaticClassID()) { 638 errln("getStaticClassID != getDynamicClassID for Normalizer."); 639 } 640 641 UChar32Iterator iter32(expect, expectLength, expectMiddle); 642 643 UChar32 c1, c2; 644 char m; 645 646 // initially set the indexes into the middle of the strings 647 iter.setIndexOnly(srcMiddle); 648 649 // move around and compare the iteration code points with 650 // the expected ones 651 const char *move=moves; 652 while((m=*move++)!=0) { 653 if(m=='-') { 654 c1=iter.previous(); 655 c2=iter32.previous(); 656 } else if(m=='0') { 657 c1=iter.current(); 658 c2=iter32.current(); 659 } else /* m=='+' */ { 660 c1=iter.next(); 661 c2=iter32.next(); 662 } 663 664 // compare results 665 if(c1!=c2) { 666 // copy the moves until the current (m) move, and terminate 667 char history[64]; 668 uprv_strcpy(history, moves); 669 history[move-moves]=0; 670 dataerrln("error: mismatch in Normalizer iteration (%s) at %s: " 671 "got c1=U+%04lx != expected c2=U+%04lx", 672 name, history, c1, c2); 673 break; 674 } 675 676 // compare indexes 677 if(iter.getIndex()!=expectIndex[iter32.getIndex()]) { 678 // copy the moves until the current (m) move, and terminate 679 char history[64]; 680 uprv_strcpy(history, moves); 681 history[move-moves]=0; 682 errln("error: index mismatch in Normalizer iteration (%s) at %s: " 683 "Normalizer index %ld expected %ld\n", 684 name, history, iter.getIndex(), expectIndex[iter32.getIndex()]); 685 break; 686 } 687 } 688} 689 690void 691BasicNormalizerTest::TestPreviousNext() { 692 // src and expect strings 693 static const UChar src[]={ 694 U16_LEAD(0x2f999), U16_TRAIL(0x2f999), 695 U16_LEAD(0x1d15f), U16_TRAIL(0x1d15f), 696 0xc4, 697 0x1ed0 698 }; 699 static const UChar32 expect[]={ 700 0x831d, 701 0x1d158, 0x1d165, 702 0x41, 0x308, 703 0x4f, 0x302, 0x301 704 }; 705 706 // expected src indexes corresponding to expect indexes 707 static const int32_t expectIndex[]={ 708 0, 709 2, 2, 710 4, 4, 711 5, 5, 5, 712 6 // behind last character 713 }; 714 715 // src and expect strings for regression test for j2911 716 static const UChar src_j2911[]={ 717 U16_LEAD(0x2f999), U16_TRAIL(0x2f999), 718 0xdd00, 0xd900, // unpaired surrogates - regression test for j2911 719 0xc4, 720 0x4f, 0x302, 0x301 721 }; 722 static const UChar32 expect_j2911[]={ 723 0x831d, 724 0xdd00, 0xd900, // unpaired surrogates - regression test for j2911 725 0xc4, 726 0x1ed0 727 }; 728 729 // expected src indexes corresponding to expect indexes 730 static const int32_t expectIndex_j2911[]={ 731 0, 732 2, 3, 733 4, 734 5, 735 8 // behind last character 736 }; 737 738 // initial indexes into the src and expect strings 739 // for both sets of test data 740 enum { 741 SRC_MIDDLE=4, 742 EXPECT_MIDDLE=3, 743 SRC_MIDDLE_2=2, 744 EXPECT_MIDDLE_2=1 745 }; 746 747 // movement vector 748 // - for previous(), 0 for current(), + for next() 749 // for both sets of test data 750 static const char *const moves="0+0+0--0-0-+++0--+++++++0--------"; 751 752 TestPreviousNext(src, UPRV_LENGTHOF(src), 753 expect, UPRV_LENGTHOF(expect), 754 expectIndex, 755 SRC_MIDDLE, EXPECT_MIDDLE, 756 moves, UNORM_NFD, "basic"); 757 758 TestPreviousNext(src_j2911, UPRV_LENGTHOF(src_j2911), 759 expect_j2911, UPRV_LENGTHOF(expect_j2911), 760 expectIndex_j2911, 761 SRC_MIDDLE, EXPECT_MIDDLE, 762 moves, UNORM_NFKC, "j2911"); 763 764 // try again from different "middle" indexes 765 TestPreviousNext(src, UPRV_LENGTHOF(src), 766 expect, UPRV_LENGTHOF(expect), 767 expectIndex, 768 SRC_MIDDLE_2, EXPECT_MIDDLE_2, 769 moves, UNORM_NFD, "basic_2"); 770 771 TestPreviousNext(src_j2911, UPRV_LENGTHOF(src_j2911), 772 expect_j2911, UPRV_LENGTHOF(expect_j2911), 773 expectIndex_j2911, 774 SRC_MIDDLE_2, EXPECT_MIDDLE_2, 775 moves, UNORM_NFKC, "j2911_2"); 776} 777 778void BasicNormalizerTest::TestConcatenate() { 779 static const char *const 780 cases[][4]={ 781 /* mode, left, right, result */ 782 { 783 "C", 784 "re", 785 "\\u0301sum\\u00e9", 786 "r\\u00e9sum\\u00e9" 787 }, 788 { 789 "C", 790 "a\\u1100", 791 "\\u1161bcdefghijk", 792 "a\\uac00bcdefghijk" 793 }, 794 /* ### TODO: add more interesting cases */ 795 { 796 "D", 797 "\\u03B1\\u0345", 798 "\\u0C4D\\U000110BA\\U0001D169", 799 "\\u03B1\\U0001D169\\U000110BA\\u0C4D\\u0345" 800 } 801 }; 802 803 UnicodeString left, right, expect, result, r; 804 UErrorCode errorCode; 805 UNormalizationMode mode; 806 int32_t i; 807 808 /* test concatenation */ 809 for(i=0; i<UPRV_LENGTHOF(cases); ++i) { 810 switch(*cases[i][0]) { 811 case 'C': mode=UNORM_NFC; break; 812 case 'D': mode=UNORM_NFD; break; 813 case 'c': mode=UNORM_NFKC; break; 814 case 'd': mode=UNORM_NFKD; break; 815 default: mode=UNORM_NONE; break; 816 } 817 818 left=UnicodeString(cases[i][1], "").unescape(); 819 right=UnicodeString(cases[i][2], "").unescape(); 820 expect=UnicodeString(cases[i][3], "").unescape(); 821 822 //result=r=UnicodeString(); 823 errorCode=U_ZERO_ERROR; 824 825 r=Normalizer::concatenate(left, right, result, mode, 0, errorCode); 826 if(U_FAILURE(errorCode) || /*result!=r ||*/ result!=expect) { 827 dataerrln("error in Normalizer::concatenate(), cases[] fails with "+ 828 UnicodeString(u_errorName(errorCode))+", result==expect: expected: "+ 829 hex(expect)+" =========> got: " + hex(result)); 830 } 831 } 832 833 /* test error cases */ 834 835 /* left.getBuffer()==result.getBuffer() */ 836 result=r=expect=UnicodeString("zz", ""); 837 errorCode=U_UNEXPECTED_TOKEN; 838 r=Normalizer::concatenate(left, right, result, mode, 0, errorCode); 839 if(errorCode!=U_UNEXPECTED_TOKEN || result!=r || !result.isBogus()) { 840 errln("error in Normalizer::concatenate(), violates UErrorCode protocol"); 841 } 842 843 left.setToBogus(); 844 errorCode=U_ZERO_ERROR; 845 r=Normalizer::concatenate(left, right, result, mode, 0, errorCode); 846 if(errorCode!=U_ILLEGAL_ARGUMENT_ERROR || result!=r || !result.isBogus()) { 847 errln("error in Normalizer::concatenate(), does not detect left.isBogus()"); 848 } 849} 850 851// reference implementation of Normalizer::compare 852static int32_t 853ref_norm_compare(const UnicodeString &s1, const UnicodeString &s2, uint32_t options, UErrorCode &errorCode) { 854 UnicodeString r1, r2, t1, t2; 855 int32_t normOptions=(int32_t)(options>>UNORM_COMPARE_NORM_OPTIONS_SHIFT); 856 857 if(options&U_COMPARE_IGNORE_CASE) { 858 Normalizer::decompose(s1, FALSE, normOptions, r1, errorCode); 859 Normalizer::decompose(s2, FALSE, normOptions, r2, errorCode); 860 861 r1.foldCase(options); 862 r2.foldCase(options); 863 } else { 864 r1=s1; 865 r2=s2; 866 } 867 868 Normalizer::decompose(r1, FALSE, normOptions, t1, errorCode); 869 Normalizer::decompose(r2, FALSE, normOptions, t2, errorCode); 870 871 if(options&U_COMPARE_CODE_POINT_ORDER) { 872 return t1.compareCodePointOrder(t2); 873 } else { 874 return t1.compare(t2); 875 } 876} 877 878// test wrapper for Normalizer::compare, sets UNORM_INPUT_IS_FCD appropriately 879static int32_t 880_norm_compare(const UnicodeString &s1, const UnicodeString &s2, uint32_t options, UErrorCode &errorCode) { 881 int32_t normOptions=(int32_t)(options>>UNORM_COMPARE_NORM_OPTIONS_SHIFT); 882 883 if( UNORM_YES==Normalizer::quickCheck(s1, UNORM_FCD, normOptions, errorCode) && 884 UNORM_YES==Normalizer::quickCheck(s2, UNORM_FCD, normOptions, errorCode)) { 885 options|=UNORM_INPUT_IS_FCD; 886 } 887 888 return Normalizer::compare(s1, s2, options, errorCode); 889} 890 891// reference implementation of UnicodeString::caseCompare 892static int32_t 893ref_case_compare(const UnicodeString &s1, const UnicodeString &s2, uint32_t options) { 894 UnicodeString t1, t2; 895 896 t1=s1; 897 t2=s2; 898 899 t1.foldCase(options); 900 t2.foldCase(options); 901 902 if(options&U_COMPARE_CODE_POINT_ORDER) { 903 return t1.compareCodePointOrder(t2); 904 } else { 905 return t1.compare(t2); 906 } 907} 908 909// reduce an integer to -1/0/1 910static inline int32_t 911_sign(int32_t value) { 912 if(value==0) { 913 return 0; 914 } else { 915 return (value>>31)|1; 916 } 917} 918 919static const char * 920_signString(int32_t value) { 921 if(value<0) { 922 return "<0"; 923 } else if(value==0) { 924 return "=0"; 925 } else /* value>0 */ { 926 return ">0"; 927 } 928} 929 930void 931BasicNormalizerTest::TestCompare() { 932 // test Normalizer::compare and unorm_compare (thinly wrapped by the former) 933 // by comparing it with its semantic equivalent 934 // since we trust the pieces, this is sufficient 935 936 // test each string with itself and each other 937 // each time with all options 938 static const char *const 939 strings[]={ 940 // some cases from NormalizationTest.txt 941 // 0..3 942 "D\\u031B\\u0307\\u0323", 943 "\\u1E0C\\u031B\\u0307", 944 "D\\u031B\\u0323\\u0307", 945 "d\\u031B\\u0323\\u0307", 946 947 // 4..6 948 "\\u00E4", 949 "a\\u0308", 950 "A\\u0308", 951 952 // Angstrom sign = A ring 953 // 7..10 954 "\\u212B", 955 "\\u00C5", 956 "A\\u030A", 957 "a\\u030A", 958 959 // 11.14 960 "a\\u059A\\u0316\\u302A\\u032Fb", 961 "a\\u302A\\u0316\\u032F\\u059Ab", 962 "a\\u302A\\u0316\\u032F\\u059Ab", 963 "A\\u059A\\u0316\\u302A\\u032Fb", 964 965 // from ICU case folding tests 966 // 15..20 967 "A\\u00df\\u00b5\\ufb03\\U0001040c\\u0131", 968 "ass\\u03bcffi\\U00010434i", 969 "\\u0061\\u0042\\u0131\\u03a3\\u00df\\ufb03\\ud93f\\udfff", 970 "\\u0041\\u0062\\u0069\\u03c3\\u0073\\u0053\\u0046\\u0066\\u0049\\ud93f\\udfff", 971 "\\u0041\\u0062\\u0131\\u03c3\\u0053\\u0073\\u0066\\u0046\\u0069\\ud93f\\udfff", 972 "\\u0041\\u0062\\u0069\\u03c3\\u0073\\u0053\\u0046\\u0066\\u0049\\ud93f\\udffd", 973 974 // U+d800 U+10001 see implementation comment in unorm_cmpEquivFold 975 // vs. U+10000 at bottom - code point order 976 // 21..22 977 "\\ud800\\ud800\\udc01", 978 "\\ud800\\udc00", 979 980 // other code point order tests from ustrtest.cpp 981 // 23..31 982 "\\u20ac\\ud801", 983 "\\u20ac\\ud800\\udc00", 984 "\\ud800", 985 "\\ud800\\uff61", 986 "\\udfff", 987 "\\uff61\\udfff", 988 "\\uff61\\ud800\\udc02", 989 "\\ud800\\udc02", 990 "\\ud84d\\udc56", 991 992 // long strings, see cnormtst.c/TestNormCoverage() 993 // equivalent if case-insensitive 994 // 32..33 995 "\\uAD8B\\uAD8B\\uAD8B\\uAD8B" 996 "\\U0001d15e\\U0001d157\\U0001d165\\U0001d15e\\U0001d15e\\U0001d15e\\U0001d15e" 997 "\\U0001d15e\\U0001d157\\U0001d165\\U0001d15e\\U0001d15e\\U0001d15e\\U0001d15e" 998 "\\U0001d15e\\U0001d157\\U0001d165\\U0001d15e\\U0001d15e\\U0001d15e\\U0001d15e" 999 "\\U0001d157\\U0001d165\\U0001d15e\\U0001d15e\\U0001d15e\\U0001d15e\\U0001d15e" 1000 "\\U0001d157\\U0001d165\\U0001d15e\\U0001d15e\\U0001d15e\\U0001d15e\\U0001d15e" 1001 "aaaaaaaaaaaaaaaaaazzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" 1002 "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" 1003 "ccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc" 1004 "ddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd" 1005 "\\uAD8B\\uAD8B\\uAD8B\\uAD8B" 1006 "d\\u031B\\u0307\\u0323", 1007 1008 "\\u1100\\u116f\\u11aa\\uAD8B\\uAD8B\\u1100\\u116f\\u11aa" 1009 "\\U0001d157\\U0001d165\\U0001d15e\\U0001d15e\\U0001d15e\\U0001d15e\\U0001d15e" 1010 "\\U0001d157\\U0001d165\\U0001d15e\\U0001d15e\\U0001d15e\\U0001d15e\\U0001d15e" 1011 "\\U0001d157\\U0001d165\\U0001d15e\\U0001d15e\\U0001d15e\\U0001d15e\\U0001d15e" 1012 "\\U0001d15e\\U0001d157\\U0001d165\\U0001d15e\\U0001d15e\\U0001d15e\\U0001d15e" 1013 "\\U0001d15e\\U0001d157\\U0001d165\\U0001d15e\\U0001d15e\\U0001d15e\\U0001d15e" 1014 "aaaaaaaaaaAAAAAAAAZZZZZZZZZZZZZZZZzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" 1015 "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" 1016 "ccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc" 1017 "ddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd" 1018 "\\u1100\\u116f\\u11aa\\uAD8B\\uAD8B\\u1100\\u116f\\u11aa" 1019 "\\u1E0C\\u031B\\u0307", 1020 1021 // some strings that may make a difference whether the compare function 1022 // case-folds or decomposes first 1023 // 34..41 1024 "\\u0360\\u0345\\u0334", 1025 "\\u0360\\u03b9\\u0334", 1026 1027 "\\u0360\\u1f80\\u0334", 1028 "\\u0360\\u03b1\\u0313\\u03b9\\u0334", 1029 1030 "\\u0360\\u1ffc\\u0334", 1031 "\\u0360\\u03c9\\u03b9\\u0334", 1032 1033 "a\\u0360\\u0345\\u0360\\u0345b", 1034 "a\\u0345\\u0360\\u0345\\u0360b", 1035 1036 // interesting cases for canonical caseless match with turkic i handling 1037 // 42..43 1038 "\\u00cc", 1039 "\\u0069\\u0300", 1040 1041 // strings with post-Unicode 3.2 normalization or normalization corrections 1042 // 44..45 1043 "\\u00e4\\u193b\\U0002f868", 1044 "\\u0061\\u193b\\u0308\\u36fc", 1045 1046 // empty string 1047 // 46 1048 "" 1049 }; 1050 1051 UnicodeString s[100]; // at least as many items as in strings[] ! 1052 1053 // all combinations of options 1054 // UNORM_INPUT_IS_FCD is set automatically if both input strings fulfill FCD conditions 1055 // set UNORM_UNICODE_3_2 in one additional combination 1056 static const struct { 1057 uint32_t options; 1058 const char *name; 1059 } opt[]={ 1060 { 0, "default" }, 1061 { U_COMPARE_CODE_POINT_ORDER, "c.p. order" }, 1062 { U_COMPARE_IGNORE_CASE, "ignore case" }, 1063 { U_COMPARE_CODE_POINT_ORDER|U_COMPARE_IGNORE_CASE, "c.p. order & ignore case" }, 1064 { U_COMPARE_IGNORE_CASE|U_FOLD_CASE_EXCLUDE_SPECIAL_I, "ignore case & special i" }, 1065 { U_COMPARE_CODE_POINT_ORDER|U_COMPARE_IGNORE_CASE|U_FOLD_CASE_EXCLUDE_SPECIAL_I, "c.p. order & ignore case & special i" }, 1066 { UNORM_UNICODE_3_2<<UNORM_COMPARE_NORM_OPTIONS_SHIFT, "Unicode 3.2" } 1067 }; 1068 1069 int32_t i, j, k, count=UPRV_LENGTHOF(strings); 1070 int32_t result, refResult; 1071 1072 UErrorCode errorCode; 1073 1074 // create the UnicodeStrings 1075 for(i=0; i<count; ++i) { 1076 s[i]=UnicodeString(strings[i], "").unescape(); 1077 } 1078 1079 // test them each with each other 1080 for(i=0; i<count; ++i) { 1081 for(j=i; j<count; ++j) { 1082 for(k=0; k<UPRV_LENGTHOF(opt); ++k) { 1083 // test Normalizer::compare 1084 errorCode=U_ZERO_ERROR; 1085 result=_norm_compare(s[i], s[j], opt[k].options, errorCode); 1086 refResult=ref_norm_compare(s[i], s[j], opt[k].options, errorCode); 1087 if(_sign(result)!=_sign(refResult)) { 1088 errln("Normalizer::compare(%d, %d, %s)%s should be %s %s", 1089 i, j, opt[k].name, _signString(result), _signString(refResult), 1090 U_SUCCESS(errorCode) ? "" : u_errorName(errorCode)); 1091 } 1092 1093 // test UnicodeString::caseCompare - same internal implementation function 1094 if(opt[k].options&U_COMPARE_IGNORE_CASE) { 1095 errorCode=U_ZERO_ERROR; 1096 result=s[i].caseCompare(s[j], opt[k].options); 1097 refResult=ref_case_compare(s[i], s[j], opt[k].options); 1098 if(_sign(result)!=_sign(refResult)) { 1099 errln("UniStr::caseCompare(%d, %d, %s)%s should be %s %s", 1100 i, j, opt[k].name, _signString(result), _signString(refResult), 1101 U_SUCCESS(errorCode) ? "" : u_errorName(errorCode)); 1102 } 1103 } 1104 } 1105 } 1106 } 1107 1108 // test cases with i and I to make sure Turkic works 1109 static const UChar iI[]={ 0x49, 0x69, 0x130, 0x131 }; 1110 UnicodeSet iSet, set; 1111 1112 UnicodeString s1, s2; 1113 1114 const Normalizer2Impl *nfcImpl=Normalizer2Factory::getNFCImpl(errorCode); 1115 if(U_FAILURE(errorCode) || !nfcImpl->ensureCanonIterData(errorCode)) { 1116 dataerrln("Normalizer2Factory::getNFCImpl().ensureCanonIterData() failed: %s", 1117 u_errorName(errorCode)); 1118 return; 1119 } 1120 1121 // collect all sets into one for contiguous output 1122 for(i=0; i<UPRV_LENGTHOF(iI); ++i) { 1123 if(nfcImpl->getCanonStartSet(iI[i], iSet)) { 1124 set.addAll(iSet); 1125 } 1126 } 1127 1128 // test all of these precomposed characters 1129 const Normalizer2 *nfcNorm2=Normalizer2::getNFCInstance(errorCode); 1130 UnicodeSetIterator it(set); 1131 while(it.next() && !it.isString()) { 1132 UChar32 c=it.getCodepoint(); 1133 if(!nfcNorm2->getDecomposition(c, s2)) { 1134 dataerrln("NFC.getDecomposition(i-composite U+%04lx) failed", (long)c); 1135 return; 1136 } 1137 1138 s1.setTo(c); 1139 for(k=0; k<UPRV_LENGTHOF(opt); ++k) { 1140 // test Normalizer::compare 1141 errorCode=U_ZERO_ERROR; 1142 result=_norm_compare(s1, s2, opt[k].options, errorCode); 1143 refResult=ref_norm_compare(s1, s2, opt[k].options, errorCode); 1144 if(_sign(result)!=_sign(refResult)) { 1145 errln("Normalizer::compare(U+%04x with its NFD, %s)%s should be %s %s", 1146 c, opt[k].name, _signString(result), _signString(refResult), 1147 U_SUCCESS(errorCode) ? "" : u_errorName(errorCode)); 1148 } 1149 1150 // test UnicodeString::caseCompare - same internal implementation function 1151 if(opt[k].options&U_COMPARE_IGNORE_CASE) { 1152 errorCode=U_ZERO_ERROR; 1153 result=s1.caseCompare(s2, opt[k].options); 1154 refResult=ref_case_compare(s1, s2, opt[k].options); 1155 if(_sign(result)!=_sign(refResult)) { 1156 errln("UniStr::caseCompare(U+%04x with its NFD, %s)%s should be %s %s", 1157 c, opt[k].name, _signString(result), _signString(refResult), 1158 U_SUCCESS(errorCode) ? "" : u_errorName(errorCode)); 1159 } 1160 } 1161 } 1162 } 1163 1164 // test getDecomposition() for some characters that do not decompose 1165 if( nfcNorm2->getDecomposition(0x20, s2) || 1166 nfcNorm2->getDecomposition(0x4e00, s2) || 1167 nfcNorm2->getDecomposition(0x20002, s2) 1168 ) { 1169 errln("NFC.getDecomposition() returns TRUE for characters which do not have decompositions"); 1170 } 1171 1172 // test getRawDecomposition() for some characters that do not decompose 1173 if( nfcNorm2->getRawDecomposition(0x20, s2) || 1174 nfcNorm2->getRawDecomposition(0x4e00, s2) || 1175 nfcNorm2->getRawDecomposition(0x20002, s2) 1176 ) { 1177 errln("NFC.getRawDecomposition() returns TRUE for characters which do not have decompositions"); 1178 } 1179 1180 // test composePair() for some pairs of characters that do not compose 1181 if( nfcNorm2->composePair(0x20, 0x301)>=0 || 1182 nfcNorm2->composePair(0x61, 0x305)>=0 || 1183 nfcNorm2->composePair(0x1100, 0x1160)>=0 || 1184 nfcNorm2->composePair(0xac00, 0x11a7)>=0 1185 ) { 1186 errln("NFC.composePair() incorrectly composes some pairs of characters"); 1187 } 1188 1189 // test FilteredNormalizer2::getDecomposition() 1190 UnicodeSet filter(UNICODE_STRING_SIMPLE("[^\\u00a0-\\u00ff]"), errorCode); 1191 FilteredNormalizer2 fn2(*nfcNorm2, filter); 1192 if( fn2.getDecomposition(0xe4, s1) || !fn2.getDecomposition(0x100, s2) || 1193 s2.length()!=2 || s2[0]!=0x41 || s2[1]!=0x304 1194 ) { 1195 errln("FilteredNormalizer2(NFC, ^A0-FF).getDecomposition() failed"); 1196 } 1197 1198 // test FilteredNormalizer2::getRawDecomposition() 1199 if( fn2.getRawDecomposition(0xe4, s1) || !fn2.getRawDecomposition(0x100, s2) || 1200 s2.length()!=2 || s2[0]!=0x41 || s2[1]!=0x304 1201 ) { 1202 errln("FilteredNormalizer2(NFC, ^A0-FF).getRawDecomposition() failed"); 1203 } 1204 1205 // test FilteredNormalizer2::composePair() 1206 if( 0x100!=fn2.composePair(0x41, 0x304) || 1207 fn2.composePair(0xc7, 0x301)>=0 // unfiltered result: U+1E08 1208 ) { 1209 errln("FilteredNormalizer2(NFC, ^A0-FF).composePair() failed"); 1210 } 1211} 1212 1213// verify that case-folding does not un-FCD strings 1214int32_t 1215BasicNormalizerTest::countFoldFCDExceptions(uint32_t foldingOptions) { 1216 UnicodeString s, fold, d; 1217 UChar32 c; 1218 int32_t count; 1219 uint8_t cc, trailCC, foldCC, foldTrailCC; 1220 UNormalizationCheckResult qcResult; 1221 int8_t category; 1222 UBool isNFD; 1223 UErrorCode errorCode; 1224 1225 logln("Test if case folding may un-FCD a string (folding options %04lx)", foldingOptions); 1226 1227 count=0; 1228 for(c=0; c<=0x10ffff; ++c) { 1229 errorCode = U_ZERO_ERROR; 1230 category=u_charType(c); 1231 if(category==U_UNASSIGNED) { 1232 continue; // skip unassigned code points 1233 } 1234 if(c==0xac00) { 1235 c=0xd7a3; // skip Hangul - no case folding there 1236 continue; 1237 } 1238 // skip Han blocks - no case folding there either 1239 if(c==0x3400) { 1240 c=0x4db5; 1241 continue; 1242 } 1243 if(c==0x4e00) { 1244 c=0x9fa5; 1245 continue; 1246 } 1247 if(c==0x20000) { 1248 c=0x2a6d6; 1249 continue; 1250 } 1251 1252 s.setTo(c); 1253 1254 // get leading and trailing cc for c 1255 Normalizer::decompose(s, FALSE, 0, d, errorCode); 1256 isNFD= s==d; 1257 cc=u_getCombiningClass(d.char32At(0)); 1258 trailCC=u_getCombiningClass(d.char32At(d.length()-1)); 1259 1260 // get leading and trailing cc for the case-folding of c 1261 s.foldCase(foldingOptions); 1262 Normalizer::decompose(s, FALSE, 0, d, errorCode); 1263 foldCC=u_getCombiningClass(d.char32At(0)); 1264 foldTrailCC=u_getCombiningClass(d.char32At(d.length()-1)); 1265 1266 qcResult=Normalizer::quickCheck(s, UNORM_FCD, errorCode); 1267 1268 if (U_FAILURE(errorCode)) { 1269 ++count; 1270 dataerrln("U+%04lx: Failed with error %s", u_errorName(errorCode)); 1271 } 1272 1273 // bad: 1274 // - character maps to empty string: adjacent characters may then need reordering 1275 // - folding has different leading/trailing cc's, and they don't become just 0 1276 // - folding itself is not FCD 1277 if( qcResult!=UNORM_YES || 1278 s.isEmpty() || 1279 (cc!=foldCC && foldCC!=0) || (trailCC!=foldTrailCC && foldTrailCC!=0) 1280 ) { 1281 ++count; 1282 dataerrln("U+%04lx: case-folding may un-FCD a string (folding options %04lx)", c, foldingOptions); 1283 dataerrln(" cc %02x trailCC %02x foldCC(U+%04lx) %02x foldTrailCC(U+%04lx) %02x quickCheck(folded)=%d", cc, trailCC, d.char32At(0), foldCC, d.char32At(d.length()-1), foldTrailCC, qcResult); 1284 continue; 1285 } 1286 1287 // also bad: 1288 // if a code point is in NFD but its case folding is not, then 1289 // unorm_compare will also fail 1290 if(isNFD && UNORM_YES!=Normalizer::quickCheck(s, UNORM_NFD, errorCode)) { 1291 ++count; 1292 errln("U+%04lx: case-folding un-NFDs this character (folding options %04lx)", c, foldingOptions); 1293 } 1294 } 1295 1296 logln("There are %ld code points for which case-folding may un-FCD a string (folding options %04lx)", count, foldingOptions); 1297 return count; 1298} 1299 1300void 1301BasicNormalizerTest::FindFoldFCDExceptions() { 1302 int32_t count; 1303 1304 count=countFoldFCDExceptions(0); 1305 count+=countFoldFCDExceptions(U_FOLD_CASE_EXCLUDE_SPECIAL_I); 1306 if(count>0) { 1307 /* 1308 * If case-folding un-FCDs any strings, then unorm_compare() must be 1309 * re-implemented. 1310 * It currently assumes that one can check for FCD then case-fold 1311 * and then still have FCD strings for raw decomposition without reordering. 1312 */ 1313 dataerrln("error: There are %ld code points for which case-folding may un-FCD a string for all folding options.\n" 1314 "See comment in BasicNormalizerTest::FindFoldFCDExceptions()!", count); 1315 } 1316} 1317 1318static void 1319initExpectedSkippables(UnicodeSet skipSets[UNORM_MODE_COUNT], UErrorCode &errorCode) { 1320 skipSets[UNORM_NFD].applyPattern( 1321 UNICODE_STRING_SIMPLE("[[:NFD_QC=Yes:]&[:ccc=0:]]"), errorCode); 1322 skipSets[UNORM_NFC].applyPattern( 1323 UNICODE_STRING_SIMPLE("[[:NFC_QC=Yes:]&[:ccc=0:]-[:HST=LV:]]"), errorCode); 1324 skipSets[UNORM_NFKD].applyPattern( 1325 UNICODE_STRING_SIMPLE("[[:NFKD_QC=Yes:]&[:ccc=0:]]"), errorCode); 1326 skipSets[UNORM_NFKC].applyPattern( 1327 UNICODE_STRING_SIMPLE("[[:NFKC_QC=Yes:]&[:ccc=0:]-[:HST=LV:]]"), errorCode); 1328 1329 // Remove from the NFC and NFKC sets all those characters that change 1330 // when a back-combining character is added. 1331 // First, get all of the back-combining characters and their combining classes. 1332 UnicodeSet combineBack("[:NFC_QC=Maybe:]", errorCode); 1333 int32_t numCombineBack=combineBack.size(); 1334 int32_t *combineBackCharsAndCc=new int32_t[numCombineBack*2]; 1335 UnicodeSetIterator iter(combineBack); 1336 for(int32_t i=0; i<numCombineBack; ++i) { 1337 iter.next(); 1338 UChar32 c=iter.getCodepoint(); 1339 combineBackCharsAndCc[2*i]=c; 1340 combineBackCharsAndCc[2*i+1]=u_getCombiningClass(c); 1341 } 1342 1343 // We need not look at control codes, Han characters nor Hangul LVT syllables because they 1344 // do not combine forward. LV syllables are already removed. 1345 UnicodeSet notInteresting("[[:C:][:Unified_Ideograph:][:HST=LVT:]]", errorCode); 1346 LocalPointer<UnicodeSet> unsure(&((UnicodeSet *)(skipSets[UNORM_NFC].clone()))->removeAll(notInteresting)); 1347 // System.out.format("unsure.size()=%d\n", unsure.size()); 1348 1349 // For each character about which we are unsure, see if it changes when we add 1350 // one of the back-combining characters. 1351 const Normalizer2 *norm2=Normalizer2::getNFCInstance(errorCode); 1352 UnicodeString s; 1353 iter.reset(*unsure); 1354 while(iter.next()) { 1355 UChar32 c=iter.getCodepoint(); 1356 s.setTo(c); 1357 int32_t cLength=s.length(); 1358 int32_t tccc=u_getIntPropertyValue(c, UCHAR_TRAIL_CANONICAL_COMBINING_CLASS); 1359 for(int32_t i=0; i<numCombineBack; ++i) { 1360 // If c's decomposition ends with a character with non-zero combining class, then 1361 // c can only change if it combines with a character with a non-zero combining class. 1362 int32_t cc2=combineBackCharsAndCc[2*i+1]; 1363 if(tccc==0 || cc2!=0) { 1364 UChar32 c2=combineBackCharsAndCc[2*i]; 1365 s.append(c2); 1366 if(!norm2->isNormalized(s, errorCode)) { 1367 // System.out.format("remove U+%04x (tccc=%d) + U+%04x (cc=%d)\n", c, tccc, c2, cc2); 1368 skipSets[UNORM_NFC].remove(c); 1369 skipSets[UNORM_NFKC].remove(c); 1370 break; 1371 } 1372 s.truncate(cLength); 1373 } 1374 } 1375 } 1376 delete [] combineBackCharsAndCc; 1377} 1378 1379void 1380BasicNormalizerTest::TestSkippable() { 1381 UnicodeSet diff, skipSets[UNORM_MODE_COUNT], expectSets[UNORM_MODE_COUNT]; 1382 UnicodeString s, pattern; 1383 1384 /* build NF*Skippable sets from runtime data */ 1385 IcuTestErrorCode errorCode(*this, "TestSkippable"); 1386 skipSets[UNORM_NFD].applyPattern(UNICODE_STRING_SIMPLE("[:NFD_Inert:]"), errorCode); 1387 skipSets[UNORM_NFKD].applyPattern(UNICODE_STRING_SIMPLE("[:NFKD_Inert:]"), errorCode); 1388 skipSets[UNORM_NFC].applyPattern(UNICODE_STRING_SIMPLE("[:NFC_Inert:]"), errorCode); 1389 skipSets[UNORM_NFKC].applyPattern(UNICODE_STRING_SIMPLE("[:NFKC_Inert:]"), errorCode); 1390 if(errorCode.logDataIfFailureAndReset("UnicodeSet(NF..._Inert) failed")) { 1391 return; 1392 } 1393 1394 /* get expected sets from hardcoded patterns */ 1395 initExpectedSkippables(expectSets, errorCode); 1396 errorCode.assertSuccess(); 1397 1398 for(int32_t i=UNORM_NONE; i<UNORM_MODE_COUNT; ++i) { 1399 if(skipSets[i]!=expectSets[i]) { 1400 errln("error: TestSkippable skipSets[%d]!=expectedSets[%d]\n", i, i); 1401 // Note: This used to depend on hardcoded UnicodeSet patterns generated by 1402 // Mark's unicodetools.com.ibm.text.UCD.NFSkippable, by 1403 // running com.ibm.text.UCD.Main with the option NFSkippable. 1404 // Since ICU 4.6/Unicode 6, we are generating the 1405 // expectSets ourselves in initSkippables(). 1406 1407 s=UNICODE_STRING_SIMPLE("skip-expect="); 1408 (diff=skipSets[i]).removeAll(expectSets[i]).toPattern(pattern, TRUE); 1409 s.append(pattern); 1410 1411 pattern.remove(); 1412 s.append(UNICODE_STRING_SIMPLE("\n\nexpect-skip=")); 1413 (diff=expectSets[i]).removeAll(skipSets[i]).toPattern(pattern, TRUE); 1414 s.append(pattern); 1415 s.append(UNICODE_STRING_SIMPLE("\n\n")); 1416 1417 errln(s); 1418 } 1419 } 1420} 1421 1422struct StringPair { const char *input, *expected; }; 1423 1424void 1425BasicNormalizerTest::TestCustomComp() { 1426 static const StringPair pairs[]={ 1427 { "\\uD801\\uE000\\uDFFE", "" }, 1428 { "\\uD800\\uD801\\uE000\\uDFFE\\uDFFF", "\\uD7FF\\uFFFF" }, 1429 { "\\uD800\\uD801\\uDFFE\\uDFFF", "\\uD7FF\\U000107FE\\uFFFF" }, 1430 { "\\uE001\\U000110B9\\u0345\\u0308\\u0327", "\\uE002\\U000110B9\\u0327\\u0345" }, 1431 { "\\uE010\\U000F0011\\uE012", "\\uE011\\uE012" }, 1432 { "\\uE010\\U000F0011\\U000F0011\\uE012", "\\uE011\\U000F0010" }, 1433 { "\\uE111\\u1161\\uE112\\u1162", "\\uAE4C\\u1102\\u0062\\u1162" }, 1434 { "\\uFFF3\\uFFF7\\U00010036\\U00010077", "\\U00010037\\U00010037\\uFFF6\\U00010037" } 1435 }; 1436 IcuTestErrorCode errorCode(*this, "BasicNormalizerTest/TestCustomComp"); 1437 const Normalizer2 *customNorm2= 1438 Normalizer2::getInstance(loadTestData(errorCode), "testnorm", 1439 UNORM2_COMPOSE, errorCode); 1440 if(errorCode.logDataIfFailureAndReset("unable to load testdata/testnorm.nrm")) { 1441 return; 1442 } 1443 for(int32_t i=0; i<UPRV_LENGTHOF(pairs); ++i) { 1444 const StringPair &pair=pairs[i]; 1445 UnicodeString input=UnicodeString(pair.input, -1, US_INV).unescape(); 1446 UnicodeString expected=UnicodeString(pair.expected, -1, US_INV).unescape(); 1447 UnicodeString result=customNorm2->normalize(input, errorCode); 1448 if(result!=expected) { 1449 errln("custom compose Normalizer2 did not normalize input %d as expected", i); 1450 } 1451 } 1452} 1453 1454void 1455BasicNormalizerTest::TestCustomFCC() { 1456 static const StringPair pairs[]={ 1457 { "\\uD801\\uE000\\uDFFE", "" }, 1458 { "\\uD800\\uD801\\uE000\\uDFFE\\uDFFF", "\\uD7FF\\uFFFF" }, 1459 { "\\uD800\\uD801\\uDFFE\\uDFFF", "\\uD7FF\\U000107FE\\uFFFF" }, 1460 // The following expected result is different from CustomComp 1461 // because of only-contiguous composition. 1462 { "\\uE001\\U000110B9\\u0345\\u0308\\u0327", "\\uE001\\U000110B9\\u0327\\u0308\\u0345" }, 1463 { "\\uE010\\U000F0011\\uE012", "\\uE011\\uE012" }, 1464 { "\\uE010\\U000F0011\\U000F0011\\uE012", "\\uE011\\U000F0010" }, 1465 { "\\uE111\\u1161\\uE112\\u1162", "\\uAE4C\\u1102\\u0062\\u1162" }, 1466 { "\\uFFF3\\uFFF7\\U00010036\\U00010077", "\\U00010037\\U00010037\\uFFF6\\U00010037" } 1467 }; 1468 IcuTestErrorCode errorCode(*this, "BasicNormalizerTest/TestCustomFCC"); 1469 const Normalizer2 *customNorm2= 1470 Normalizer2::getInstance(loadTestData(errorCode), "testnorm", 1471 UNORM2_COMPOSE_CONTIGUOUS, errorCode); 1472 if(errorCode.logDataIfFailureAndReset("unable to load testdata/testnorm.nrm")) { 1473 return; 1474 } 1475 for(int32_t i=0; i<UPRV_LENGTHOF(pairs); ++i) { 1476 const StringPair &pair=pairs[i]; 1477 UnicodeString input=UnicodeString(pair.input, -1, US_INV).unescape(); 1478 UnicodeString expected=UnicodeString(pair.expected, -1, US_INV).unescape(); 1479 UnicodeString result=customNorm2->normalize(input, errorCode); 1480 if(result!=expected) { 1481 errln("custom FCC Normalizer2 did not normalize input %d as expected", i); 1482 } 1483 } 1484} 1485 1486/* Improve code coverage of Normalizer2 */ 1487void 1488BasicNormalizerTest::TestFilteredNormalizer2Coverage() { 1489 UErrorCode errorCode = U_ZERO_ERROR; 1490 const Normalizer2 *nfcNorm2=Normalizer2::getNFCInstance(errorCode); 1491 if (U_FAILURE(errorCode)) { 1492 dataerrln("Normalizer2::getNFCInstance() call failed - %s", u_errorName(status)); 1493 return; 1494 } 1495 UnicodeSet filter(UNICODE_STRING_SIMPLE("[^\\u00a0-\\u00ff\\u0310-\\u031f]"), errorCode); 1496 FilteredNormalizer2 fn2(*nfcNorm2, filter); 1497 1498 UChar32 char32 = 0x0054; 1499 1500 if (fn2.isInert(char32)) { 1501 errln("FilteredNormalizer2.isInert() failed."); 1502 } 1503 1504 if (fn2.hasBoundaryAfter(char32)) { 1505 errln("FilteredNormalizer2.hasBoundaryAfter() failed."); 1506 } 1507 1508 UChar32 c; 1509 for(c=0; c<=0x3ff; ++c) { 1510 uint8_t expectedCC= filter.contains(c) ? nfcNorm2->getCombiningClass(c) : 0; 1511 uint8_t cc=fn2.getCombiningClass(c); 1512 if(cc!=expectedCC) { 1513 errln( 1514 UnicodeString("FilteredNormalizer2(NFC, ^A0-FF,310-31F).getCombiningClass(U+")+ 1515 hex(c)+ 1516 ")==filtered NFC.getCC()"); 1517 } 1518 } 1519 1520 UnicodeString newString1 = UNICODE_STRING_SIMPLE("[^\\u0100-\\u01ff]"); 1521 UnicodeString newString2 = UNICODE_STRING_SIMPLE("[^\\u0200-\\u02ff]"); 1522 fn2.append(newString1, newString2, errorCode); 1523 if (U_FAILURE(errorCode)) { 1524 errln("FilteredNormalizer2.append() failed."); 1525 } 1526} 1527 1528#endif /* #if !UCONFIG_NO_NORMALIZATION */ 1529