tstnorm.cpp revision 27f654740f2a26ad62a5c155af9199af9e69b889
1b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru/******************************************************************** 2b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru * COPYRIGHT: 350294ead5e5d23f5bbfed76e00e6b510bd41eee1claireho * Copyright (c) 1997-2010, International Business Machines Corporation and 4b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru * others. All Rights Reserved. 5b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru ********************************************************************/ 6b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 7b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru#include "unicode/utypes.h" 8b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 9b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru#if !UCONFIG_NO_NORMALIZATION 10b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 11b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru#include "unicode/uchar.h" 1250294ead5e5d23f5bbfed76e00e6b510bd41eee1claireho#include "unicode/errorcode.h" 13b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru#include "unicode/normlzr.h" 14b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru#include "unicode/uniset.h" 15b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru#include "unicode/usetiter.h" 16b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru#include "unicode/schriter.h" 17b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru#include "cstring.h" 1827f654740f2a26ad62a5c155af9199af9e69b889claireho#include "normalizer2impl.h" 19b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru#include "tstnorm.h" 20b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 21b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru#define LENGTHOF(array) ((int32_t)(sizeof(array)/sizeof((array)[0]))) 22b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru#define ARRAY_LENGTH(array) LENGTHOF(array) 23b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 24b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru#define CASE(id,test) case id: \ 25b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru name = #test; \ 26b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru if (exec) { \ 27b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru logln(#test "---"); \ 28b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru logln((UnicodeString)""); \ 29b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru test(); \ 30b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru } \ 31b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru break 32b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 33b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Querustatic UErrorCode status = U_ZERO_ERROR; 34b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 35b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queruvoid BasicNormalizerTest::runIndexedTest(int32_t index, UBool exec, 36b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru const char* &name, char* /*par*/) { 37b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru switch (index) { 38b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru CASE(0,TestDecomp); 39b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru CASE(1,TestCompatDecomp); 40b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru CASE(2,TestCanonCompose); 41b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru CASE(3,TestCompatCompose); 42b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru CASE(4,TestPrevious); 43b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru CASE(5,TestHangulDecomp); 44b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru CASE(6,TestHangulCompose); 45b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru CASE(7,TestTibetan); 46b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru CASE(8,TestCompositionExclusion); 47b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru CASE(9,TestZeroIndex); 48b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru CASE(10,TestVerisign); 49b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru CASE(11,TestPreviousNext); 50b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru CASE(12,TestNormalizerAPI); 51b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru CASE(13,TestConcatenate); 52b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru CASE(14,FindFoldFCDExceptions); 53b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru CASE(15,TestCompare); 54b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru CASE(16,TestSkippable); 5527f654740f2a26ad62a5c155af9199af9e69b889claireho#if !UCONFIG_NO_FILE_IO && !UCONFIG_NO_LEGACY_CONVERSION 5650294ead5e5d23f5bbfed76e00e6b510bd41eee1claireho CASE(17,TestCustomComp); 5750294ead5e5d23f5bbfed76e00e6b510bd41eee1claireho CASE(18,TestCustomFCC); 5827f654740f2a26ad62a5c155af9199af9e69b889claireho#endif 5927f654740f2a26ad62a5c155af9199af9e69b889claireho CASE(19,TestFilteredNormalizer2Coverage); 60b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru default: name = ""; break; 61b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru } 62b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru} 63b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 64b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru/** 65b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru * Convert Java-style strings with \u Unicode escapes into UnicodeString objects 66b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru */ 67b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Querustatic UnicodeString str(const char *input) 68b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru{ 69b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru UnicodeString str(input, ""); // Invariant conversion 70b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru return str.unescape(); 71b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru} 72b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 73b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 74b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste QueruBasicNormalizerTest::BasicNormalizerTest() 75b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru{ 76b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru // canonTest 77b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru // Input Decomposed Composed 78b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 79b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru canonTests[0][0] = str("cat"); canonTests[0][1] = str("cat"); canonTests[0][2] = str("cat"); 80b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 81b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru canonTests[1][0] = str("\\u00e0ardvark"); canonTests[1][1] = str("a\\u0300ardvark"); canonTests[1][2] = str("\\u00e0ardvark"); 82b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 83b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru canonTests[2][0] = str("\\u1e0a"); canonTests[2][1] = str("D\\u0307"); canonTests[2][2] = str("\\u1e0a"); // D-dot_above 84b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 85b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru canonTests[3][0] = str("D\\u0307"); canonTests[3][1] = str("D\\u0307"); canonTests[3][2] = str("\\u1e0a"); // D dot_above 86b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 87b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru canonTests[4][0] = str("\\u1e0c\\u0307"); canonTests[4][1] = str("D\\u0323\\u0307"); canonTests[4][2] = str("\\u1e0c\\u0307"); // D-dot_below dot_above 88b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 89b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru canonTests[5][0] = str("\\u1e0a\\u0323"); canonTests[5][1] = str("D\\u0323\\u0307"); canonTests[5][2] = str("\\u1e0c\\u0307"); // D-dot_above dot_below 90b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 91b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru canonTests[6][0] = str("D\\u0307\\u0323"); canonTests[6][1] = str("D\\u0323\\u0307"); canonTests[6][2] = str("\\u1e0c\\u0307"); // D dot_below dot_above 92b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 93b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru canonTests[7][0] = str("\\u1e10\\u0307\\u0323"); canonTests[7][1] = str("D\\u0327\\u0323\\u0307"); canonTests[7][2] = str("\\u1e10\\u0323\\u0307"); // D dot_below cedilla dot_above 94b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 95b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru canonTests[8][0] = str("D\\u0307\\u0328\\u0323"); canonTests[8][1] = str("D\\u0328\\u0323\\u0307"); canonTests[8][2] = str("\\u1e0c\\u0328\\u0307"); // D dot_above ogonek dot_below 96b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 97b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru canonTests[9][0] = str("\\u1E14"); canonTests[9][1] = str("E\\u0304\\u0300"); canonTests[9][2] = str("\\u1E14"); // E-macron-grave 98b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 99b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru canonTests[10][0] = str("\\u0112\\u0300"); canonTests[10][1] = str("E\\u0304\\u0300"); canonTests[10][2] = str("\\u1E14"); // E-macron + grave 100b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 101b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru canonTests[11][0] = str("\\u00c8\\u0304"); canonTests[11][1] = str("E\\u0300\\u0304"); canonTests[11][2] = str("\\u00c8\\u0304"); // E-grave + macron 102b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 103b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru canonTests[12][0] = str("\\u212b"); canonTests[12][1] = str("A\\u030a"); canonTests[12][2] = str("\\u00c5"); // angstrom_sign 104b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 105b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru canonTests[13][0] = str("\\u00c5"); canonTests[13][1] = str("A\\u030a"); canonTests[13][2] = str("\\u00c5"); // A-ring 106b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 107b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru canonTests[14][0] = str("\\u00C4ffin"); canonTests[14][1] = str("A\\u0308ffin"); canonTests[14][2] = str("\\u00C4ffin"); 108b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 109b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru canonTests[15][0] = str("\\u00C4\\uFB03n"); canonTests[15][1] = str("A\\u0308\\uFB03n"); canonTests[15][2] = str("\\u00C4\\uFB03n"); 110b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 111b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru canonTests[16][0] = str("Henry IV"); canonTests[16][1] = str("Henry IV"); canonTests[16][2] = str("Henry IV"); 112b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 113b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru canonTests[17][0] = str("Henry \\u2163"); canonTests[17][1] = str("Henry \\u2163"); canonTests[17][2] = str("Henry \\u2163"); 114b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 115b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru canonTests[18][0] = str("\\u30AC"); canonTests[18][1] = str("\\u30AB\\u3099"); canonTests[18][2] = str("\\u30AC"); // ga (Katakana) 116b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 117b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru canonTests[19][0] = str("\\u30AB\\u3099"); canonTests[19][1] = str("\\u30AB\\u3099"); canonTests[19][2] = str("\\u30AC"); // ka + ten 118b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 119b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru canonTests[20][0] = str("\\uFF76\\uFF9E"); canonTests[20][1] = str("\\uFF76\\uFF9E"); canonTests[20][2] = str("\\uFF76\\uFF9E"); // hw_ka + hw_ten 120b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 121b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru canonTests[21][0] = str("\\u30AB\\uFF9E"); canonTests[21][1] = str("\\u30AB\\uFF9E"); canonTests[21][2] = str("\\u30AB\\uFF9E"); // ka + hw_ten 122b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 123b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru canonTests[22][0] = str("\\uFF76\\u3099"); canonTests[22][1] = str("\\uFF76\\u3099"); canonTests[22][2] = str("\\uFF76\\u3099"); // hw_ka + ten 124b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 125b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru canonTests[23][0] = str("A\\u0300\\u0316"); canonTests[23][1] = str("A\\u0316\\u0300"); canonTests[23][2] = str("\\u00C0\\u0316"); 126b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 127b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru /* compatTest */ 128b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru // Input Decomposed Composed 129b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru compatTests[0][0] = str("cat"); compatTests[0][1] = str("cat"); compatTests[0][2] = str("cat") ; 130b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 131b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru compatTests[1][0] = str("\\uFB4f"); compatTests[1][1] = str("\\u05D0\\u05DC"); compatTests[1][2] = str("\\u05D0\\u05DC"); // Alef-Lamed vs. Alef, Lamed 132b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 133b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru compatTests[2][0] = str("\\u00C4ffin"); compatTests[2][1] = str("A\\u0308ffin"); compatTests[2][2] = str("\\u00C4ffin") ; 134b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 135b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru compatTests[3][0] = str("\\u00C4\\uFB03n"); compatTests[3][1] = str("A\\u0308ffin"); compatTests[3][2] = str("\\u00C4ffin") ; // ffi ligature -> f + f + i 136b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 137b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru compatTests[4][0] = str("Henry IV"); compatTests[4][1] = str("Henry IV"); compatTests[4][2] = str("Henry IV") ; 138b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 139b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru compatTests[5][0] = str("Henry \\u2163"); compatTests[5][1] = str("Henry IV"); compatTests[5][2] = str("Henry IV") ; 140b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 141b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru compatTests[6][0] = str("\\u30AC"); compatTests[6][1] = str("\\u30AB\\u3099"); compatTests[6][2] = str("\\u30AC") ; // ga (Katakana) 142b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 143b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru compatTests[7][0] = str("\\u30AB\\u3099"); compatTests[7][1] = str("\\u30AB\\u3099"); compatTests[7][2] = str("\\u30AC") ; // ka + ten 144b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 145b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru compatTests[8][0] = str("\\uFF76\\u3099"); compatTests[8][1] = str("\\u30AB\\u3099"); compatTests[8][2] = str("\\u30AC") ; // hw_ka + ten 146b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 147b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru /* These two are broken in Unicode 2.1.2 but fixed in 2.1.5 and later */ 148b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru compatTests[9][0] = str("\\uFF76\\uFF9E"); compatTests[9][1] = str("\\u30AB\\u3099"); compatTests[9][2] = str("\\u30AC") ; // hw_ka + hw_ten 149b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 150b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru compatTests[10][0] = str("\\u30AB\\uFF9E"); compatTests[10][1] = str("\\u30AB\\u3099"); compatTests[10][2] = str("\\u30AC") ; // ka + hw_ten 151b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 152b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru /* Hangul Canonical */ 153b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru // Input Decomposed Composed 154b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru hangulCanon[0][0] = str("\\ud4db"); hangulCanon[0][1] = str("\\u1111\\u1171\\u11b6"); hangulCanon[0][2] = str("\\ud4db") ; 155b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 156b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru hangulCanon[1][0] = str("\\u1111\\u1171\\u11b6"), hangulCanon[1][1] = str("\\u1111\\u1171\\u11b6"), hangulCanon[1][2] = str("\\ud4db"); 157b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru} 158b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 159b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste QueruBasicNormalizerTest::~BasicNormalizerTest() 160b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru{ 161b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru} 162b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 163b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queruvoid BasicNormalizerTest::TestPrevious() 164b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru{ 165b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru Normalizer* norm = new Normalizer("", UNORM_NFD); 166b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 167b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru logln("testing decomp..."); 168b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru uint32_t i; 169b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru for (i = 0; i < ARRAY_LENGTH(canonTests); i++) { 170b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru backAndForth(norm, canonTests[i][0]); 171b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru } 172b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 173b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru logln("testing compose..."); 174b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru norm->setMode(UNORM_NFC); 175b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru for (i = 0; i < ARRAY_LENGTH(canonTests); i++) { 176b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru backAndForth(norm, canonTests[i][0]); 177b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru } 178b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 179b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru delete norm; 180b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru} 181b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 182b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queruvoid BasicNormalizerTest::TestDecomp() 183b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru{ 184b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru Normalizer* norm = new Normalizer("", UNORM_NFD); 185b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru iterateTest(norm, canonTests, ARRAY_LENGTH(canonTests), 1); 186b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru staticTest(UNORM_NFD, 0, canonTests, ARRAY_LENGTH(canonTests), 1); 187b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru delete norm; 188b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru} 189b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 190b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queruvoid BasicNormalizerTest::TestCompatDecomp() 191b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru{ 192b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru Normalizer* norm = new Normalizer("", UNORM_NFKD); 193b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru iterateTest(norm, compatTests, ARRAY_LENGTH(compatTests), 1); 194b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 195b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru staticTest(UNORM_NFKD, 0, 196b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru compatTests, ARRAY_LENGTH(compatTests), 1); 197b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru delete norm; 198b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru} 199b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 200b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queruvoid BasicNormalizerTest::TestCanonCompose() 201b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru{ 202b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru Normalizer* norm = new Normalizer("", UNORM_NFC); 203b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru iterateTest(norm, canonTests, ARRAY_LENGTH(canonTests), 2); 204b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 205b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru staticTest(UNORM_NFC, 0, canonTests, 206b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru ARRAY_LENGTH(canonTests), 2); 207b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru delete norm; 208b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru} 209b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 210b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queruvoid BasicNormalizerTest::TestCompatCompose() 211b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru{ 212b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru Normalizer* norm = new Normalizer("", UNORM_NFKC); 213b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru iterateTest(norm, compatTests, ARRAY_LENGTH(compatTests), 2); 214b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 215b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru staticTest(UNORM_NFKC, 0, 216b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru compatTests, ARRAY_LENGTH(compatTests), 2); 217b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru delete norm; 218b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru} 219b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 220b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 221b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru//------------------------------------------------------------------------------- 222b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 223b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queruvoid BasicNormalizerTest::TestHangulCompose() 224b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru{ 225b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru // Make sure that the static composition methods work 226b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru logln("Canonical composition..."); 227b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru staticTest(UNORM_NFC, 0, hangulCanon, ARRAY_LENGTH(hangulCanon), 2); 228b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru logln("Compatibility composition..."); 229b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 230b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru // Now try iterative composition.... 231b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru logln("Static composition..."); 232b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru Normalizer* norm = new Normalizer("", UNORM_NFC); 233b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru iterateTest(norm, hangulCanon, ARRAY_LENGTH(hangulCanon), 2); 234b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru norm->setMode(UNORM_NFKC); 235b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 236b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru // And finally, make sure you can do it in reverse too 237b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru logln("Reverse iteration..."); 238b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru norm->setMode(UNORM_NFC); 239b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru for (uint32_t i = 0; i < ARRAY_LENGTH(hangulCanon); i++) { 240b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru backAndForth(norm, hangulCanon[i][0]); 241b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru } 242b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru delete norm; 243b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru} 244b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 245b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queruvoid BasicNormalizerTest::TestHangulDecomp() 246b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru{ 247b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru // Make sure that the static decomposition methods work 248b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru logln("Canonical decomposition..."); 249b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru staticTest(UNORM_NFD, 0, hangulCanon, ARRAY_LENGTH(hangulCanon), 1); 250b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru logln("Compatibility decomposition..."); 251b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 252b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru // Now the iterative decomposition methods... 253b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru logln("Iterative decomposition..."); 254b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru Normalizer* norm = new Normalizer("", UNORM_NFD); 255b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru iterateTest(norm, hangulCanon, ARRAY_LENGTH(hangulCanon), 1); 256b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru norm->setMode(UNORM_NFKD); 257b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 258b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru // And finally, make sure you can do it in reverse too 259b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru logln("Reverse iteration..."); 260b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru norm->setMode(UNORM_NFD); 261b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru for (uint32_t i = 0; i < ARRAY_LENGTH(hangulCanon); i++) { 262b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru backAndForth(norm, hangulCanon[i][0]); 263b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru } 264b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru delete norm; 265b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru} 266b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 267b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru/** 268b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru * The Tibetan vowel sign AA, 0f71, was messed up prior to Unicode version 2.1.9. 269b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru */ 270b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queruvoid BasicNormalizerTest::TestTibetan(void) { 271b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru UnicodeString decomp[1][3]; 272b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru decomp[0][0] = str("\\u0f77"); 273b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru decomp[0][1] = str("\\u0f77"); 274b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru decomp[0][2] = str("\\u0fb2\\u0f71\\u0f80"); 275b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 276b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru UnicodeString compose[1][3]; 277b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru compose[0][0] = str("\\u0fb2\\u0f71\\u0f80"); 278b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru compose[0][1] = str("\\u0fb2\\u0f71\\u0f80"); 279b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru compose[0][2] = str("\\u0fb2\\u0f71\\u0f80"); 280b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 281b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru staticTest(UNORM_NFD, 0, decomp, ARRAY_LENGTH(decomp), 1); 282b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru staticTest(UNORM_NFKD, 0, decomp, ARRAY_LENGTH(decomp), 2); 283b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru staticTest(UNORM_NFC, 0, compose, ARRAY_LENGTH(compose), 1); 284b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru staticTest(UNORM_NFKC, 0, compose, ARRAY_LENGTH(compose), 2); 285b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru} 286b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 287b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru/** 288b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru * Make sure characters in the CompositionExclusion.txt list do not get 289b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru * composed to. 290b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru */ 291b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queruvoid BasicNormalizerTest::TestCompositionExclusion(void) { 292b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru // This list is generated from CompositionExclusion.txt. 293b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru // Update whenever the normalizer tables are updated. Note 294b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru // that we test all characters listed, even those that can be 295b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru // derived from the Unicode DB and are therefore commented 296b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru // out. 297b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru // ### TODO read composition exclusion from source/data/unidata file 298b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru // and test against that 299b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru UnicodeString EXCLUDED = str( 300b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru "\\u0340\\u0341\\u0343\\u0344\\u0374\\u037E\\u0387\\u0958" 301b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru "\\u0959\\u095A\\u095B\\u095C\\u095D\\u095E\\u095F\\u09DC" 302b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru "\\u09DD\\u09DF\\u0A33\\u0A36\\u0A59\\u0A5A\\u0A5B\\u0A5E" 303b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru "\\u0B5C\\u0B5D\\u0F43\\u0F4D\\u0F52\\u0F57\\u0F5C\\u0F69" 304b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru "\\u0F73\\u0F75\\u0F76\\u0F78\\u0F81\\u0F93\\u0F9D\\u0FA2" 305b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru "\\u0FA7\\u0FAC\\u0FB9\\u1F71\\u1F73\\u1F75\\u1F77\\u1F79" 306b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru "\\u1F7B\\u1F7D\\u1FBB\\u1FBE\\u1FC9\\u1FCB\\u1FD3\\u1FDB" 307b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru "\\u1FE3\\u1FEB\\u1FEE\\u1FEF\\u1FF9\\u1FFB\\u1FFD\\u2000" 308b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru "\\u2001\\u2126\\u212A\\u212B\\u2329\\u232A\\uF900\\uFA10" 309b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru "\\uFA12\\uFA15\\uFA20\\uFA22\\uFA25\\uFA26\\uFA2A\\uFB1F" 310b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru "\\uFB2A\\uFB2B\\uFB2C\\uFB2D\\uFB2E\\uFB2F\\uFB30\\uFB31" 311b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru "\\uFB32\\uFB33\\uFB34\\uFB35\\uFB36\\uFB38\\uFB39\\uFB3A" 312b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru "\\uFB3B\\uFB3C\\uFB3E\\uFB40\\uFB41\\uFB43\\uFB44\\uFB46" 313b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru "\\uFB47\\uFB48\\uFB49\\uFB4A\\uFB4B\\uFB4C\\uFB4D\\uFB4E" 314b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru ); 315b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru for (int32_t i=0; i<EXCLUDED.length(); ++i) { 316b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru UnicodeString a(EXCLUDED.charAt(i)); 317b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru UnicodeString b; 318b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru UnicodeString c; 319b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru Normalizer::normalize(a, UNORM_NFKD, 0, b, status); 320b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru Normalizer::normalize(b, UNORM_NFC, 0, c, status); 321b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru if (c == a) { 322b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru errln("FAIL: " + hex(a) + " x DECOMP_COMPAT => " + 323b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru hex(b) + " x COMPOSE => " + 324b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru hex(c)); 325b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru } else if (verbose) { 326b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru logln("Ok: " + hex(a) + " x DECOMP_COMPAT => " + 327b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru hex(b) + " x COMPOSE => " + 328b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru hex(c)); 329b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru } 330b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru } 331b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru} 332b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 333b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru/** 334b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru * Test for a problem that showed up just before ICU 1.6 release 335b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru * having to do with combining characters with an index of zero. 336b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru * Such characters do not participate in any canonical 337b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru * decompositions. However, having an index of zero means that 338b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru * they all share one typeMask[] entry, that is, they all have to 339b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru * map to the same canonical class, which is not the case, in 340b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru * reality. 341b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru */ 342b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queruvoid BasicNormalizerTest::TestZeroIndex(void) { 343b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru const char* DATA[] = { 344b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru // Expect col1 x COMPOSE_COMPAT => col2 345b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru // Expect col2 x DECOMP => col3 346b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru "A\\u0316\\u0300", "\\u00C0\\u0316", "A\\u0316\\u0300", 347b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru "A\\u0300\\u0316", "\\u00C0\\u0316", "A\\u0316\\u0300", 348b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru "A\\u0327\\u0300", "\\u00C0\\u0327", "A\\u0327\\u0300", 349b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru "c\\u0321\\u0327", "c\\u0321\\u0327", "c\\u0321\\u0327", 350b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru "c\\u0327\\u0321", "\\u00E7\\u0321", "c\\u0327\\u0321", 351b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru }; 352b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru int32_t DATA_length = (int32_t)(sizeof(DATA) / sizeof(DATA[0])); 353b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 354b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru for (int32_t i=0; i<DATA_length; i+=3) { 355b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru UErrorCode status = U_ZERO_ERROR; 356b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru UnicodeString a(DATA[i], ""); 357b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru a = a.unescape(); 358b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru UnicodeString b; 359b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru Normalizer::normalize(a, UNORM_NFKC, 0, b, status); 36050294ead5e5d23f5bbfed76e00e6b510bd41eee1claireho if (U_FAILURE(status)) { 36150294ead5e5d23f5bbfed76e00e6b510bd41eee1claireho dataerrln("Error calling normalize UNORM_NFKC: %s", u_errorName(status)); 362b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru } else { 36350294ead5e5d23f5bbfed76e00e6b510bd41eee1claireho UnicodeString exp(DATA[i+1], ""); 36450294ead5e5d23f5bbfed76e00e6b510bd41eee1claireho exp = exp.unescape(); 36550294ead5e5d23f5bbfed76e00e6b510bd41eee1claireho if (b == exp) { 36650294ead5e5d23f5bbfed76e00e6b510bd41eee1claireho logln((UnicodeString)"Ok: " + hex(a) + " x COMPOSE_COMPAT => " + hex(b)); 36750294ead5e5d23f5bbfed76e00e6b510bd41eee1claireho } else { 36850294ead5e5d23f5bbfed76e00e6b510bd41eee1claireho errln((UnicodeString)"FAIL: " + hex(a) + " x COMPOSE_COMPAT => " + hex(b) + 36950294ead5e5d23f5bbfed76e00e6b510bd41eee1claireho ", expect " + hex(exp)); 37050294ead5e5d23f5bbfed76e00e6b510bd41eee1claireho } 371b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru } 372b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru Normalizer::normalize(b, UNORM_NFD, 0, a, status); 37350294ead5e5d23f5bbfed76e00e6b510bd41eee1claireho if (U_FAILURE(status)) { 37450294ead5e5d23f5bbfed76e00e6b510bd41eee1claireho dataerrln("Error calling normalize UNORM_NFD: %s", u_errorName(status)); 375b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru } else { 37650294ead5e5d23f5bbfed76e00e6b510bd41eee1claireho UnicodeString exp = UnicodeString(DATA[i+2], "").unescape(); 37750294ead5e5d23f5bbfed76e00e6b510bd41eee1claireho if (a == exp) { 37850294ead5e5d23f5bbfed76e00e6b510bd41eee1claireho logln((UnicodeString)"Ok: " + hex(b) + " x DECOMP => " + hex(a)); 37950294ead5e5d23f5bbfed76e00e6b510bd41eee1claireho } else { 38050294ead5e5d23f5bbfed76e00e6b510bd41eee1claireho errln((UnicodeString)"FAIL: " + hex(b) + " x DECOMP => " + hex(a) + 38150294ead5e5d23f5bbfed76e00e6b510bd41eee1claireho ", expect " + hex(exp)); 38250294ead5e5d23f5bbfed76e00e6b510bd41eee1claireho } 383b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru } 384b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru } 385b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru} 386b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 387b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru/** 388b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru * Run a few specific cases that are failing for Verisign. 389b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru */ 390b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queruvoid BasicNormalizerTest::TestVerisign(void) { 391b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru /* 392b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru > Their input: 393b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru > 05B8 05B9 05B1 0591 05C3 05B0 05AC 059F 394b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru > Their output (supposedly from ICU): 395b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru > 05B8 05B1 05B9 0591 05C3 05B0 05AC 059F 396b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru > My output from charlint: 397b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru > 05B1 05B8 05B9 0591 05C3 05B0 05AC 059F 398b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 399b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 05B8 05B9 05B1 0591 05C3 05B0 05AC 059F => 05B1 05B8 05B9 0591 05C3 05B0 400b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 05AC 059F 401b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 402b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru U+05B8 18 E HEBREW POINT QAMATS 403b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru U+05B9 19 F HEBREW POINT HOLAM 404b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru U+05B1 11 HEBREW POINT HATAF SEGOL 405b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru U+0591 220 HEBREW ACCENT ETNAHTA 406b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru U+05C3 0 HEBREW PUNCTUATION SOF PASUQ 407b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru U+05B0 10 HEBREW POINT SHEVA 408b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru U+05AC 230 HEBREW ACCENT ILUY 409b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru U+059F 230 HEBREW ACCENT QARNEY PARA 410b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 411b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru U+05B1 11 HEBREW POINT HATAF SEGOL 412b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru U+05B8 18 HEBREW POINT QAMATS 413b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru U+05B9 19 HEBREW POINT HOLAM 414b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru U+0591 220 HEBREW ACCENT ETNAHTA 415b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru U+05C3 0 HEBREW PUNCTUATION SOF PASUQ 416b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru U+05B0 10 HEBREW POINT SHEVA 417b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru U+05AC 230 HEBREW ACCENT ILUY 418b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru U+059F 230 HEBREW ACCENT QARNEY PARA 419b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 420b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru Wrong result: 421b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru U+05B8 18 HEBREW POINT QAMATS 422b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru U+05B1 11 HEBREW POINT HATAF SEGOL 423b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru U+05B9 19 HEBREW POINT HOLAM 424b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru U+0591 220 HEBREW ACCENT ETNAHTA 425b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru U+05C3 0 HEBREW PUNCTUATION SOF PASUQ 426b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru U+05B0 10 HEBREW POINT SHEVA 427b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru U+05AC 230 HEBREW ACCENT ILUY 428b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru U+059F 230 HEBREW ACCENT QARNEY PARA 429b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 430b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 431b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru > Their input: 432b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru >0592 05B7 05BC 05A5 05B0 05C0 05C4 05AD 433b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru >Their output (supposedly from ICU): 434b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru >0592 05B0 05B7 05BC 05A5 05C0 05AD 05C4 435b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru >My output from charlint: 436b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru >05B0 05B7 05BC 05A5 0592 05C0 05AD 05C4 437b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 438b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 0592 05B7 05BC 05A5 05B0 05C0 05C4 05AD => 05B0 05B7 05BC 05A5 0592 05C0 439b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 05AD 05C4 440b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 441b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru U+0592 230 HEBREW ACCENT SEGOL 442b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru U+05B7 17 HEBREW POINT PATAH 443b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru U+05BC 21 HEBREW POINT DAGESH OR MAPIQ 444b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru U+05A5 220 HEBREW ACCENT MERKHA 445b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru U+05B0 10 HEBREW POINT SHEVA 446b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru U+05C0 0 HEBREW PUNCTUATION PASEQ 447b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru U+05C4 230 HEBREW MARK UPPER DOT 448b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru U+05AD 222 HEBREW ACCENT DEHI 449b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 450b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru U+05B0 10 HEBREW POINT SHEVA 451b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru U+05B7 17 HEBREW POINT PATAH 452b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru U+05BC 21 HEBREW POINT DAGESH OR MAPIQ 453b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru U+05A5 220 HEBREW ACCENT MERKHA 454b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru U+0592 230 HEBREW ACCENT SEGOL 455b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru U+05C0 0 HEBREW PUNCTUATION PASEQ 456b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru U+05AD 222 HEBREW ACCENT DEHI 457b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru U+05C4 230 HEBREW MARK UPPER DOT 458b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 459b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru Wrong result: 460b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru U+0592 230 HEBREW ACCENT SEGOL 461b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru U+05B0 10 HEBREW POINT SHEVA 462b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru U+05B7 17 HEBREW POINT PATAH 463b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru U+05BC 21 HEBREW POINT DAGESH OR MAPIQ 464b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru U+05A5 220 HEBREW ACCENT MERKHA 465b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru U+05C0 0 HEBREW PUNCTUATION PASEQ 466b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru U+05AD 222 HEBREW ACCENT DEHI 467b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru U+05C4 230 HEBREW MARK UPPER DOT 468b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru */ 469b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru UnicodeString data[2][3]; 470b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru data[0][0] = str("\\u05B8\\u05B9\\u05B1\\u0591\\u05C3\\u05B0\\u05AC\\u059F"); 471b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru data[0][1] = str("\\u05B1\\u05B8\\u05B9\\u0591\\u05C3\\u05B0\\u05AC\\u059F"); 472b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru data[0][2] = str(""); 473b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru data[1][0] = str("\\u0592\\u05B7\\u05BC\\u05A5\\u05B0\\u05C0\\u05C4\\u05AD"); 474b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru data[1][1] = str("\\u05B0\\u05B7\\u05BC\\u05A5\\u0592\\u05C0\\u05AD\\u05C4"); 475b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru data[1][2] = str(""); 476b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 477b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru staticTest(UNORM_NFD, 0, data, ARRAY_LENGTH(data), 1); 478b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru staticTest(UNORM_NFC, 0, data, ARRAY_LENGTH(data), 1); 479b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru} 480b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 481b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru//------------------------------------------------------------------------ 482b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru// Internal utilities 483b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru// 484b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 485b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste QueruUnicodeString BasicNormalizerTest::hex(UChar ch) { 486b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru UnicodeString result; 487b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru return appendHex(ch, 4, result); 488b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru} 489b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 490b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste QueruUnicodeString BasicNormalizerTest::hex(const UnicodeString& s) { 491b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru UnicodeString result; 492b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru for (int i = 0; i < s.length(); ++i) { 493b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru if (i != 0) result += (UChar)0x2c/*,*/; 494b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru appendHex(s[i], 4, result); 495b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru } 496b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru return result; 497b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru} 498b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 499b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 500b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queruinline static void insert(UnicodeString& dest, int pos, UChar32 ch) 501b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru{ 502b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru dest.replace(pos, 0, ch); 503b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru} 504b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 505b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queruvoid BasicNormalizerTest::backAndForth(Normalizer* iter, const UnicodeString& input) 506b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru{ 507b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru UChar32 ch; 508b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru iter->setText(input, status); 509b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 510b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru // Run through the iterator forwards and stick it into a StringBuffer 511b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru UnicodeString forward; 512b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru for (ch = iter->first(); ch != iter->DONE; ch = iter->next()) { 513b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru forward += ch; 514b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru } 515b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 516b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru // Now do it backwards 517b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru UnicodeString reverse; 518b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru for (ch = iter->last(); ch != iter->DONE; ch = iter->previous()) { 519b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru insert(reverse, 0, ch); 520b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru } 521b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 522b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru if (forward != reverse) { 523b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru errln("Forward/reverse mismatch for input " + hex(input) 524b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru + ", forward: " + hex(forward) + ", backward: " + hex(reverse)); 525b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru } 526b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru} 527b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 528b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queruvoid BasicNormalizerTest::staticTest(UNormalizationMode mode, int options, 529b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru UnicodeString tests[][3], int length, 530b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru int outCol) 531b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru{ 532b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru for (int i = 0; i < length; i++) 533b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru { 534b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru UnicodeString& input = tests[i][0]; 535b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru UnicodeString& expect = tests[i][outCol]; 536b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 537b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru logln("Normalizing '" + input + "' (" + hex(input) + ")" ); 538b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 539b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru UnicodeString output; 540b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru Normalizer::normalize(input, mode, options, output, status); 541b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 542b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru if (output != expect) { 54350294ead5e5d23f5bbfed76e00e6b510bd41eee1claireho dataerrln(UnicodeString("ERROR: case ") + i + " normalized " + hex(input) + "\n" 544b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru + " expected " + hex(expect) + "\n" 545b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru + " static got " + hex(output) ); 546b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru } 547b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru } 548b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru} 549b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 550b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queruvoid BasicNormalizerTest::iterateTest(Normalizer* iter, 551b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru UnicodeString tests[][3], int length, 552b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru int outCol) 553b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru{ 554b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru for (int i = 0; i < length; i++) 555b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru { 556b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru UnicodeString& input = tests[i][0]; 557b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru UnicodeString& expect = tests[i][outCol]; 558b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 559b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru logln("Normalizing '" + input + "' (" + hex(input) + ")" ); 560b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 561b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru iter->setText(input, status); 562b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru assertEqual(input, expect, iter, UnicodeString("ERROR: case ") + i + " "); 563b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru } 564b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru} 565b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 566b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queruvoid BasicNormalizerTest::assertEqual(const UnicodeString& input, 567b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru const UnicodeString& expected, 568b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru Normalizer* iter, 569b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru const UnicodeString& errPrefix) 570b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru{ 571b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru UnicodeString result; 572b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 573b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru for (UChar32 ch = iter->first(); ch != iter->DONE; ch = iter->next()) { 574b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru result += ch; 575b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru } 576b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru if (result != expected) { 57750294ead5e5d23f5bbfed76e00e6b510bd41eee1claireho dataerrln(errPrefix + "normalized " + hex(input) + "\n" 578b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru + " expected " + hex(expected) + "\n" 579b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru + " iterate got " + hex(result) ); 580b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru } 581b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru} 582b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 583b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru// helper class for TestPreviousNext() 584b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru// simple UTF-32 character iterator 585b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queruclass UChar32Iterator { 586b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Querupublic: 587b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru UChar32Iterator(const UChar32 *text, int32_t len, int32_t index) : 588b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru s(text), length(len), i(index) {} 589b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 590b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru UChar32 current() { 591b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru if(i<length) { 592b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru return s[i]; 593b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru } else { 594b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru return 0xffff; 595b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru } 596b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru } 597b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 598b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru UChar32 next() { 599b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru if(i<length) { 600b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru return s[i++]; 601b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru } else { 602b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru return 0xffff; 603b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru } 604b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru } 605b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 606b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru UChar32 previous() { 607b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru if(i>0) { 608b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru return s[--i]; 609b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru } else { 610b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru return 0xffff; 611b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru } 612b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru } 613b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 614b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru int32_t getIndex() { 615b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru return i; 616b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru } 617b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queruprivate: 618b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru const UChar32 *s; 619b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru int32_t length, i; 620b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru}; 621b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 622b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queruvoid 623b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste QueruBasicNormalizerTest::TestPreviousNext(const UChar *src, int32_t srcLength, 624b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru const UChar32 *expect, int32_t expectLength, 625b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru const int32_t *expectIndex, // its length=expectLength+1 626b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru int32_t srcMiddle, int32_t expectMiddle, 627b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru const char *moves, 628b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru UNormalizationMode mode, 629b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru const char *name) { 630b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru // iterators 631b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru Normalizer iter(src, srcLength, mode); 632b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 633b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru // test getStaticClassID and getDynamicClassID 634b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru if(iter.getDynamicClassID() != Normalizer::getStaticClassID()) { 635b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru errln("getStaticClassID != getDynamicClassID for Normalizer."); 636b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru } 637b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 638b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru UChar32Iterator iter32(expect, expectLength, expectMiddle); 639b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 640b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru UChar32 c1, c2; 641b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru char m; 642b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 643b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru // initially set the indexes into the middle of the strings 644b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru iter.setIndexOnly(srcMiddle); 645b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 646b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru // move around and compare the iteration code points with 647b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru // the expected ones 648b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru const char *move=moves; 649b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru while((m=*move++)!=0) { 650b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru if(m=='-') { 651b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru c1=iter.previous(); 652b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru c2=iter32.previous(); 653b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru } else if(m=='0') { 654b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru c1=iter.current(); 655b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru c2=iter32.current(); 656b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru } else /* m=='+' */ { 657b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru c1=iter.next(); 658b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru c2=iter32.next(); 659b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru } 660b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 661b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru // compare results 662b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru if(c1!=c2) { 663b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru // copy the moves until the current (m) move, and terminate 664b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru char history[64]; 665b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru uprv_strcpy(history, moves); 666b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru history[move-moves]=0; 66750294ead5e5d23f5bbfed76e00e6b510bd41eee1claireho dataerrln("error: mismatch in Normalizer iteration (%s) at %s: " 66850294ead5e5d23f5bbfed76e00e6b510bd41eee1claireho "got c1=U+%04lx != expected c2=U+%04lx", 669b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru name, history, c1, c2); 670b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru break; 671b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru } 672b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 673b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru // compare indexes 674b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru if(iter.getIndex()!=expectIndex[iter32.getIndex()]) { 675b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru // copy the moves until the current (m) move, and terminate 676b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru char history[64]; 677b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru uprv_strcpy(history, moves); 678b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru history[move-moves]=0; 679b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru errln("error: index mismatch in Normalizer iteration (%s) at %s: " 680b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru "Normalizer index %ld expected %ld\n", 681b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru name, history, iter.getIndex(), expectIndex[iter32.getIndex()]); 682b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru break; 683b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru } 684b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru } 685b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru} 686b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 687b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queruvoid 688b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste QueruBasicNormalizerTest::TestPreviousNext() { 689b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru // src and expect strings 690b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru static const UChar src[]={ 691b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru UTF16_LEAD(0x2f999), UTF16_TRAIL(0x2f999), 692b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru UTF16_LEAD(0x1d15f), UTF16_TRAIL(0x1d15f), 693b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 0xc4, 694b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 0x1ed0 695b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru }; 696b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru static const UChar32 expect[]={ 697b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 0x831d, 698b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 0x1d158, 0x1d165, 699b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 0x41, 0x308, 700b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 0x4f, 0x302, 0x301 701b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru }; 702b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 703b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru // expected src indexes corresponding to expect indexes 704b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru static const int32_t expectIndex[]={ 705b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 0, 706b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 2, 2, 707b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 4, 4, 708b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 5, 5, 5, 709b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 6 // behind last character 710b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru }; 711b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 712b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru // src and expect strings for regression test for j2911 713b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru static const UChar src_j2911[]={ 714b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru UTF16_LEAD(0x2f999), UTF16_TRAIL(0x2f999), 715b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 0xdd00, 0xd900, // unpaired surrogates - regression test for j2911 716b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 0xc4, 717b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 0x4f, 0x302, 0x301 718b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru }; 719b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru static const UChar32 expect_j2911[]={ 720b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 0x831d, 721b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 0xdd00, 0xd900, // unpaired surrogates - regression test for j2911 722b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 0xc4, 723b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 0x1ed0 724b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru }; 725b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 726b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru // expected src indexes corresponding to expect indexes 727b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru static const int32_t expectIndex_j2911[]={ 728b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 0, 729b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 2, 3, 730b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 4, 731b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 5, 732b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 8 // behind last character 733b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru }; 734b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 735b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru // initial indexes into the src and expect strings 736b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru // for both sets of test data 737b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru enum { 738b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru SRC_MIDDLE=4, 739b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru EXPECT_MIDDLE=3, 740b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru SRC_MIDDLE_2=2, 741b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru EXPECT_MIDDLE_2=1 742b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru }; 743b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 744b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru // movement vector 745b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru // - for previous(), 0 for current(), + for next() 746b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru // for both sets of test data 747b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru static const char *const moves="0+0+0--0-0-+++0--+++++++0--------"; 748b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 749b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru TestPreviousNext(src, LENGTHOF(src), 750b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru expect, LENGTHOF(expect), 751b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru expectIndex, 752b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru SRC_MIDDLE, EXPECT_MIDDLE, 753b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru moves, UNORM_NFD, "basic"); 754b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 755b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru TestPreviousNext(src_j2911, LENGTHOF(src_j2911), 756b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru expect_j2911, LENGTHOF(expect_j2911), 757b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru expectIndex_j2911, 758b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru SRC_MIDDLE, EXPECT_MIDDLE, 759b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru moves, UNORM_NFKC, "j2911"); 760b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 761b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru // try again from different "middle" indexes 762b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru TestPreviousNext(src, LENGTHOF(src), 763b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru expect, LENGTHOF(expect), 764b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru expectIndex, 765b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru SRC_MIDDLE_2, EXPECT_MIDDLE_2, 766b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru moves, UNORM_NFD, "basic_2"); 767b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 768b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru TestPreviousNext(src_j2911, LENGTHOF(src_j2911), 769b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru expect_j2911, LENGTHOF(expect_j2911), 770b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru expectIndex_j2911, 771b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru SRC_MIDDLE_2, EXPECT_MIDDLE_2, 772b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru moves, UNORM_NFKC, "j2911_2"); 773b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru} 774b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 775b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queruvoid BasicNormalizerTest::TestConcatenate() { 776b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru static const char *const 777b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru cases[][4]={ 778b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru /* mode, left, right, result */ 779b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru { 780b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru "C", 781b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru "re", 782b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru "\\u0301sum\\u00e9", 783b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru "r\\u00e9sum\\u00e9" 784b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru }, 785b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru { 786b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru "C", 787b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru "a\\u1100", 788b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru "\\u1161bcdefghijk", 789b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru "a\\uac00bcdefghijk" 790b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru }, 791b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru /* ### TODO: add more interesting cases */ 792b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru { 79350294ead5e5d23f5bbfed76e00e6b510bd41eee1claireho "D", 79450294ead5e5d23f5bbfed76e00e6b510bd41eee1claireho "\\u03B1\\u0345", 79550294ead5e5d23f5bbfed76e00e6b510bd41eee1claireho "\\u0C4D\\U000110BA\\U0001D169", 79650294ead5e5d23f5bbfed76e00e6b510bd41eee1claireho "\\u03B1\\U0001D169\\U000110BA\\u0C4D\\u0345" 797b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru } 798b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru }; 799b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 800b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru UnicodeString left, right, expect, result, r; 801b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru UErrorCode errorCode; 802b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru UNormalizationMode mode; 803b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru int32_t i; 804b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 805b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru /* test concatenation */ 806b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru for(i=0; i<(int32_t)(sizeof(cases)/sizeof(cases[0])); ++i) { 807b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru switch(*cases[i][0]) { 808b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru case 'C': mode=UNORM_NFC; break; 809b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru case 'D': mode=UNORM_NFD; break; 810b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru case 'c': mode=UNORM_NFKC; break; 811b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru case 'd': mode=UNORM_NFKD; break; 812b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru default: mode=UNORM_NONE; break; 813b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru } 814b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 815b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru left=UnicodeString(cases[i][1], "").unescape(); 816b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru right=UnicodeString(cases[i][2], "").unescape(); 817b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru expect=UnicodeString(cases[i][3], "").unescape(); 818b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 819b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru //result=r=UnicodeString(); 820b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru errorCode=U_ZERO_ERROR; 821b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 822b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru r=Normalizer::concatenate(left, right, result, mode, 0, errorCode); 823b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru if(U_FAILURE(errorCode) || /*result!=r ||*/ result!=expect) { 82450294ead5e5d23f5bbfed76e00e6b510bd41eee1claireho dataerrln("error in Normalizer::concatenate(), cases[] fails with "+ 825b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru UnicodeString(u_errorName(errorCode))+", result==expect: expected: "+ 826b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru hex(expect)+" =========> got: " + hex(result)); 827b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru } 828b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru } 829b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 830b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru /* test error cases */ 831b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 832b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru /* left.getBuffer()==result.getBuffer() */ 833b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru result=r=expect=UnicodeString("zz", ""); 834b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru errorCode=U_UNEXPECTED_TOKEN; 835b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru r=Normalizer::concatenate(left, right, result, mode, 0, errorCode); 836b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru if(errorCode!=U_UNEXPECTED_TOKEN || result!=r || !result.isBogus()) { 837b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru errln("error in Normalizer::concatenate(), violates UErrorCode protocol"); 838b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru } 839b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 840b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru left.setToBogus(); 841b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru errorCode=U_ZERO_ERROR; 842b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru r=Normalizer::concatenate(left, right, result, mode, 0, errorCode); 843b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru if(errorCode!=U_ILLEGAL_ARGUMENT_ERROR || result!=r || !result.isBogus()) { 844b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru errln("error in Normalizer::concatenate(), does not detect left.isBogus()"); 845b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru } 846b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru} 847b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 848b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru// reference implementation of Normalizer::compare 849b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Querustatic int32_t 850b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queruref_norm_compare(const UnicodeString &s1, const UnicodeString &s2, uint32_t options, UErrorCode &errorCode) { 851b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru UnicodeString r1, r2, t1, t2; 852b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru int32_t normOptions=(int32_t)(options>>UNORM_COMPARE_NORM_OPTIONS_SHIFT); 853b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 854b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru if(options&U_COMPARE_IGNORE_CASE) { 855b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru Normalizer::decompose(s1, FALSE, normOptions, r1, errorCode); 856b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru Normalizer::decompose(s2, FALSE, normOptions, r2, errorCode); 857b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 858b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru r1.foldCase(options); 859b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru r2.foldCase(options); 860b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru } else { 861b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru r1=s1; 862b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru r2=s2; 863b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru } 864b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 865b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru Normalizer::decompose(r1, FALSE, normOptions, t1, errorCode); 866b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru Normalizer::decompose(r2, FALSE, normOptions, t2, errorCode); 867b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 868b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru if(options&U_COMPARE_CODE_POINT_ORDER) { 869b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru return t1.compareCodePointOrder(t2); 870b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru } else { 871b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru return t1.compare(t2); 872b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru } 873b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru} 874b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 875b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru// test wrapper for Normalizer::compare, sets UNORM_INPUT_IS_FCD appropriately 876b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Querustatic int32_t 877b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru_norm_compare(const UnicodeString &s1, const UnicodeString &s2, uint32_t options, UErrorCode &errorCode) { 878b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru int32_t normOptions=(int32_t)(options>>UNORM_COMPARE_NORM_OPTIONS_SHIFT); 879b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 880b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru if( UNORM_YES==Normalizer::quickCheck(s1, UNORM_FCD, normOptions, errorCode) && 881b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru UNORM_YES==Normalizer::quickCheck(s2, UNORM_FCD, normOptions, errorCode)) { 882b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru options|=UNORM_INPUT_IS_FCD; 883b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru } 884b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 885b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru return Normalizer::compare(s1, s2, options, errorCode); 886b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru} 887b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 888b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru// reference implementation of UnicodeString::caseCompare 889b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Querustatic int32_t 890b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queruref_case_compare(const UnicodeString &s1, const UnicodeString &s2, uint32_t options) { 891b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru UnicodeString t1, t2; 892b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 893b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru t1=s1; 894b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru t2=s2; 895b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 896b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru t1.foldCase(options); 897b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru t2.foldCase(options); 898b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 899b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru if(options&U_COMPARE_CODE_POINT_ORDER) { 900b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru return t1.compareCodePointOrder(t2); 901b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru } else { 902b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru return t1.compare(t2); 903b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru } 904b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru} 905b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 906b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru// reduce an integer to -1/0/1 907b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Querustatic inline int32_t 908b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru_sign(int32_t value) { 909b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru if(value==0) { 910b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru return 0; 911b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru } else { 912b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru return (value>>31)|1; 913b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru } 914b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru} 915b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 916b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Querustatic const char * 917b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru_signString(int32_t value) { 918b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru if(value<0) { 919b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru return "<0"; 920b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru } else if(value==0) { 921b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru return "=0"; 922b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru } else /* value>0 */ { 923b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru return ">0"; 924b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru } 925b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru} 926b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 927b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queruvoid 928b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste QueruBasicNormalizerTest::TestCompare() { 929b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru // test Normalizer::compare and unorm_compare (thinly wrapped by the former) 930b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru // by comparing it with its semantic equivalent 931b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru // since we trust the pieces, this is sufficient 932b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 933b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru // test each string with itself and each other 934b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru // each time with all options 935b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru static const char *const 936b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru strings[]={ 937b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru // some cases from NormalizationTest.txt 938b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru // 0..3 939b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru "D\\u031B\\u0307\\u0323", 940b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru "\\u1E0C\\u031B\\u0307", 941b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru "D\\u031B\\u0323\\u0307", 942b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru "d\\u031B\\u0323\\u0307", 943b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 944b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru // 4..6 945b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru "\\u00E4", 946b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru "a\\u0308", 947b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru "A\\u0308", 948b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 949b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru // Angstrom sign = A ring 950b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru // 7..10 951b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru "\\u212B", 952b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru "\\u00C5", 953b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru "A\\u030A", 954b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru "a\\u030A", 955b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 956b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru // 11.14 957b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru "a\\u059A\\u0316\\u302A\\u032Fb", 958b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru "a\\u302A\\u0316\\u032F\\u059Ab", 959b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru "a\\u302A\\u0316\\u032F\\u059Ab", 960b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru "A\\u059A\\u0316\\u302A\\u032Fb", 961b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 962b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru // from ICU case folding tests 963b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru // 15..20 964b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru "A\\u00df\\u00b5\\ufb03\\U0001040c\\u0131", 965b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru "ass\\u03bcffi\\U00010434i", 966b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru "\\u0061\\u0042\\u0131\\u03a3\\u00df\\ufb03\\ud93f\\udfff", 967b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru "\\u0041\\u0062\\u0069\\u03c3\\u0073\\u0053\\u0046\\u0066\\u0049\\ud93f\\udfff", 968b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru "\\u0041\\u0062\\u0131\\u03c3\\u0053\\u0073\\u0066\\u0046\\u0069\\ud93f\\udfff", 969b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru "\\u0041\\u0062\\u0069\\u03c3\\u0073\\u0053\\u0046\\u0066\\u0049\\ud93f\\udffd", 970b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 971b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru // U+d800 U+10001 see implementation comment in unorm_cmpEquivFold 972b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru // vs. U+10000 at bottom - code point order 973b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru // 21..22 974b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru "\\ud800\\ud800\\udc01", 975b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru "\\ud800\\udc00", 976b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 977b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru // other code point order tests from ustrtest.cpp 978b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru // 23..31 979b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru "\\u20ac\\ud801", 980b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru "\\u20ac\\ud800\\udc00", 981b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru "\\ud800", 982b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru "\\ud800\\uff61", 983b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru "\\udfff", 984b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru "\\uff61\\udfff", 985b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru "\\uff61\\ud800\\udc02", 986b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru "\\ud800\\udc02", 987b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru "\\ud84d\\udc56", 988b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 989b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru // long strings, see cnormtst.c/TestNormCoverage() 990b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru // equivalent if case-insensitive 991b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru // 32..33 992b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru "\\uAD8B\\uAD8B\\uAD8B\\uAD8B" 993b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru "\\U0001d15e\\U0001d157\\U0001d165\\U0001d15e\\U0001d15e\\U0001d15e\\U0001d15e" 994b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru "\\U0001d15e\\U0001d157\\U0001d165\\U0001d15e\\U0001d15e\\U0001d15e\\U0001d15e" 995b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru "\\U0001d15e\\U0001d157\\U0001d165\\U0001d15e\\U0001d15e\\U0001d15e\\U0001d15e" 996b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru "\\U0001d157\\U0001d165\\U0001d15e\\U0001d15e\\U0001d15e\\U0001d15e\\U0001d15e" 997b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru "\\U0001d157\\U0001d165\\U0001d15e\\U0001d15e\\U0001d15e\\U0001d15e\\U0001d15e" 998b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru "aaaaaaaaaaaaaaaaaazzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" 999b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" 1000b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru "ccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc" 1001b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru "ddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd" 1002b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru "\\uAD8B\\uAD8B\\uAD8B\\uAD8B" 1003b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru "d\\u031B\\u0307\\u0323", 1004b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 1005b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru "\\u1100\\u116f\\u11aa\\uAD8B\\uAD8B\\u1100\\u116f\\u11aa" 1006b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru "\\U0001d157\\U0001d165\\U0001d15e\\U0001d15e\\U0001d15e\\U0001d15e\\U0001d15e" 1007b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru "\\U0001d157\\U0001d165\\U0001d15e\\U0001d15e\\U0001d15e\\U0001d15e\\U0001d15e" 1008b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru "\\U0001d157\\U0001d165\\U0001d15e\\U0001d15e\\U0001d15e\\U0001d15e\\U0001d15e" 1009b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru "\\U0001d15e\\U0001d157\\U0001d165\\U0001d15e\\U0001d15e\\U0001d15e\\U0001d15e" 1010b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru "\\U0001d15e\\U0001d157\\U0001d165\\U0001d15e\\U0001d15e\\U0001d15e\\U0001d15e" 1011b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru "aaaaaaaaaaAAAAAAAAZZZZZZZZZZZZZZZZzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" 1012b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" 1013b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru "ccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc" 1014b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru "ddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd" 1015b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru "\\u1100\\u116f\\u11aa\\uAD8B\\uAD8B\\u1100\\u116f\\u11aa" 1016b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru "\\u1E0C\\u031B\\u0307", 1017b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 1018b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru // some strings that may make a difference whether the compare function 1019b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru // case-folds or decomposes first 1020b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru // 34..41 1021b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru "\\u0360\\u0345\\u0334", 1022b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru "\\u0360\\u03b9\\u0334", 1023b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 1024b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru "\\u0360\\u1f80\\u0334", 1025b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru "\\u0360\\u03b1\\u0313\\u03b9\\u0334", 1026b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 1027b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru "\\u0360\\u1ffc\\u0334", 1028b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru "\\u0360\\u03c9\\u03b9\\u0334", 1029b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 1030b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru "a\\u0360\\u0345\\u0360\\u0345b", 1031b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru "a\\u0345\\u0360\\u0345\\u0360b", 1032b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 1033b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru // interesting cases for canonical caseless match with turkic i handling 1034b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru // 42..43 1035b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru "\\u00cc", 1036b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru "\\u0069\\u0300", 1037b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 1038b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru // strings with post-Unicode 3.2 normalization or normalization corrections 1039b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru // 44..45 1040b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru "\\u00e4\\u193b\\U0002f868", 1041b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru "\\u0061\\u193b\\u0308\\u36fc", 1042b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 1043b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru // empty string 1044b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru // 46 1045b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru "" 1046b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru }; 1047b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 1048b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru UnicodeString s[100]; // at least as many items as in strings[] ! 1049b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 1050b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru // all combinations of options 1051b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru // UNORM_INPUT_IS_FCD is set automatically if both input strings fulfill FCD conditions 1052b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru // set UNORM_UNICODE_3_2 in one additional combination 1053b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru static const struct { 1054b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru uint32_t options; 1055b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru const char *name; 1056b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru } opt[]={ 1057b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru { 0, "default" }, 1058b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru { U_COMPARE_CODE_POINT_ORDER, "c.p. order" }, 1059b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru { U_COMPARE_IGNORE_CASE, "ignore case" }, 1060b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru { U_COMPARE_CODE_POINT_ORDER|U_COMPARE_IGNORE_CASE, "c.p. order & ignore case" }, 1061b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru { U_COMPARE_IGNORE_CASE|U_FOLD_CASE_EXCLUDE_SPECIAL_I, "ignore case & special i" }, 1062b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru { U_COMPARE_CODE_POINT_ORDER|U_COMPARE_IGNORE_CASE|U_FOLD_CASE_EXCLUDE_SPECIAL_I, "c.p. order & ignore case & special i" }, 1063b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru { UNORM_UNICODE_3_2<<UNORM_COMPARE_NORM_OPTIONS_SHIFT, "Unicode 3.2" } 1064b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru }; 1065b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 1066b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru int32_t i, j, k, count=LENGTHOF(strings); 1067b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru int32_t result, refResult; 1068b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 1069b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru UErrorCode errorCode; 1070b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 1071b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru // create the UnicodeStrings 1072b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru for(i=0; i<count; ++i) { 1073b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru s[i]=UnicodeString(strings[i], "").unescape(); 1074b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru } 1075b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 1076b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru // test them each with each other 1077b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru for(i=0; i<count; ++i) { 1078b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru for(j=i; j<count; ++j) { 1079b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru for(k=0; k<LENGTHOF(opt); ++k) { 1080b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru // test Normalizer::compare 1081b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru errorCode=U_ZERO_ERROR; 1082b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru result=_norm_compare(s[i], s[j], opt[k].options, errorCode); 1083b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru refResult=ref_norm_compare(s[i], s[j], opt[k].options, errorCode); 1084b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru if(_sign(result)!=_sign(refResult)) { 1085b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru errln("Normalizer::compare(%d, %d, %s)%s should be %s %s", 1086b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru i, j, opt[k].name, _signString(result), _signString(refResult), 1087b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru U_SUCCESS(errorCode) ? "" : u_errorName(errorCode)); 1088b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru } 1089b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 1090b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru // test UnicodeString::caseCompare - same internal implementation function 1091b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru if(opt[k].options&U_COMPARE_IGNORE_CASE) { 1092b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru errorCode=U_ZERO_ERROR; 1093b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru result=s[i].caseCompare(s[j], opt[k].options); 1094b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru refResult=ref_case_compare(s[i], s[j], opt[k].options); 1095b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru if(_sign(result)!=_sign(refResult)) { 1096b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru errln("UniStr::caseCompare(%d, %d, %s)%s should be %s %s", 1097b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru i, j, opt[k].name, _signString(result), _signString(refResult), 1098b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru U_SUCCESS(errorCode) ? "" : u_errorName(errorCode)); 1099b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru } 1100b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru } 1101b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru } 1102b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru } 1103b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru } 1104b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 1105b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru // test cases with i and I to make sure Turkic works 1106b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru static const UChar iI[]={ 0x49, 0x69, 0x130, 0x131 }; 110727f654740f2a26ad62a5c155af9199af9e69b889claireho UnicodeSet iSet, set; 1108b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 1109b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru UnicodeString s1, s2; 111027f654740f2a26ad62a5c155af9199af9e69b889claireho 111127f654740f2a26ad62a5c155af9199af9e69b889claireho const Normalizer2Impl *nfcImpl=Normalizer2Factory::getNFCImpl(errorCode); 111227f654740f2a26ad62a5c155af9199af9e69b889claireho if(U_FAILURE(errorCode) || !nfcImpl->ensureCanonIterData(errorCode)) { 111327f654740f2a26ad62a5c155af9199af9e69b889claireho dataerrln("Normalizer2Factory::getNFCImpl().ensureCanonIterData() failed: %s", 111427f654740f2a26ad62a5c155af9199af9e69b889claireho u_errorName(errorCode)); 111527f654740f2a26ad62a5c155af9199af9e69b889claireho return; 111627f654740f2a26ad62a5c155af9199af9e69b889claireho } 1117b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 1118b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru // collect all sets into one for contiguous output 1119b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru for(i=0; i<LENGTHOF(iI); ++i) { 112027f654740f2a26ad62a5c155af9199af9e69b889claireho if(nfcImpl->getCanonStartSet(iI[i], iSet)) { 112127f654740f2a26ad62a5c155af9199af9e69b889claireho set.addAll(iSet); 1122b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru } 1123b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru } 1124b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 1125b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru // test all of these precomposed characters 112627f654740f2a26ad62a5c155af9199af9e69b889claireho const Normalizer2 *nfcNorm2=Normalizer2Factory::getNFCInstance(errorCode); 1127b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru UnicodeSetIterator it(set); 112827f654740f2a26ad62a5c155af9199af9e69b889claireho while(it.next() && !it.isString()) { 112927f654740f2a26ad62a5c155af9199af9e69b889claireho UChar32 c=it.getCodepoint(); 113027f654740f2a26ad62a5c155af9199af9e69b889claireho if(!nfcNorm2->getDecomposition(c, s2)) { 113127f654740f2a26ad62a5c155af9199af9e69b889claireho dataerrln("NFC.getDecomposition(i-composite U+%04lx) failed", (long)c); 113227f654740f2a26ad62a5c155af9199af9e69b889claireho return; 113327f654740f2a26ad62a5c155af9199af9e69b889claireho } 113427f654740f2a26ad62a5c155af9199af9e69b889claireho 113527f654740f2a26ad62a5c155af9199af9e69b889claireho s1.setTo(c); 113627f654740f2a26ad62a5c155af9199af9e69b889claireho for(k=0; k<LENGTHOF(opt); ++k) { 113727f654740f2a26ad62a5c155af9199af9e69b889claireho // test Normalizer::compare 1138b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru errorCode=U_ZERO_ERROR; 113927f654740f2a26ad62a5c155af9199af9e69b889claireho result=_norm_compare(s1, s2, opt[k].options, errorCode); 114027f654740f2a26ad62a5c155af9199af9e69b889claireho refResult=ref_norm_compare(s1, s2, opt[k].options, errorCode); 114127f654740f2a26ad62a5c155af9199af9e69b889claireho if(_sign(result)!=_sign(refResult)) { 114227f654740f2a26ad62a5c155af9199af9e69b889claireho errln("Normalizer::compare(U+%04x with its NFD, %s)%s should be %s %s", 114327f654740f2a26ad62a5c155af9199af9e69b889claireho c, opt[k].name, _signString(result), _signString(refResult), 114427f654740f2a26ad62a5c155af9199af9e69b889claireho U_SUCCESS(errorCode) ? "" : u_errorName(errorCode)); 1145b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru } 1146b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 114727f654740f2a26ad62a5c155af9199af9e69b889claireho // test UnicodeString::caseCompare - same internal implementation function 114827f654740f2a26ad62a5c155af9199af9e69b889claireho if(opt[k].options&U_COMPARE_IGNORE_CASE) { 1149b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru errorCode=U_ZERO_ERROR; 115027f654740f2a26ad62a5c155af9199af9e69b889claireho result=s1.caseCompare(s2, opt[k].options); 115127f654740f2a26ad62a5c155af9199af9e69b889claireho refResult=ref_case_compare(s1, s2, opt[k].options); 1152b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru if(_sign(result)!=_sign(refResult)) { 115327f654740f2a26ad62a5c155af9199af9e69b889claireho errln("UniStr::caseCompare(U+%04x with its NFD, %s)%s should be %s %s", 115427f654740f2a26ad62a5c155af9199af9e69b889claireho c, opt[k].name, _signString(result), _signString(refResult), 1155b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru U_SUCCESS(errorCode) ? "" : u_errorName(errorCode)); 1156b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru } 1157b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru } 1158b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru } 1159b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru } 116027f654740f2a26ad62a5c155af9199af9e69b889claireho 116127f654740f2a26ad62a5c155af9199af9e69b889claireho // test getDecomposition() for some characters that do not decompose 116227f654740f2a26ad62a5c155af9199af9e69b889claireho if( nfcNorm2->getDecomposition(0x20, s2) || 116327f654740f2a26ad62a5c155af9199af9e69b889claireho nfcNorm2->getDecomposition(0x4e00, s2) || 116427f654740f2a26ad62a5c155af9199af9e69b889claireho nfcNorm2->getDecomposition(0x20002, s2) 116527f654740f2a26ad62a5c155af9199af9e69b889claireho ) { 116627f654740f2a26ad62a5c155af9199af9e69b889claireho errln("NFC.getDecomposition() returns TRUE for characters which do not have decompositions"); 116727f654740f2a26ad62a5c155af9199af9e69b889claireho } 116827f654740f2a26ad62a5c155af9199af9e69b889claireho 116927f654740f2a26ad62a5c155af9199af9e69b889claireho // test FilteredNormalizer2::getDecomposition() 117027f654740f2a26ad62a5c155af9199af9e69b889claireho UnicodeSet filter(UNICODE_STRING_SIMPLE("[^\\u00a0-\\u00ff]"), errorCode); 117127f654740f2a26ad62a5c155af9199af9e69b889claireho FilteredNormalizer2 fn2(*nfcNorm2, filter); 117227f654740f2a26ad62a5c155af9199af9e69b889claireho if( fn2.getDecomposition(0xe4, s1) || !fn2.getDecomposition(0x100, s2) || 117327f654740f2a26ad62a5c155af9199af9e69b889claireho s2.length()!=2 || s2[0]!=0x41 || s2[1]!=0x304 117427f654740f2a26ad62a5c155af9199af9e69b889claireho ) { 117527f654740f2a26ad62a5c155af9199af9e69b889claireho errln("FilteredNormalizer2(NFC, ^A0-FF).getDecomposition() failed"); 117627f654740f2a26ad62a5c155af9199af9e69b889claireho } 1177b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru} 1178b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 1179b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru// verify that case-folding does not un-FCD strings 1180b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queruint32_t 1181b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste QueruBasicNormalizerTest::countFoldFCDExceptions(uint32_t foldingOptions) { 1182b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru UnicodeString s, fold, d; 1183b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru UChar32 c; 1184b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru int32_t count; 1185b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru uint8_t cc, trailCC, foldCC, foldTrailCC; 1186b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru UNormalizationCheckResult qcResult; 1187b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru int8_t category; 1188b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru UBool isNFD; 1189b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru UErrorCode errorCode; 1190b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 1191b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru logln("Test if case folding may un-FCD a string (folding options %04lx)", foldingOptions); 1192b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 1193b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru count=0; 1194b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru for(c=0; c<=0x10ffff; ++c) { 1195b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru errorCode = U_ZERO_ERROR; 1196b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru category=u_charType(c); 1197b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru if(category==U_UNASSIGNED) { 1198b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru continue; // skip unassigned code points 1199b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru } 1200b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru if(c==0xac00) { 1201b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru c=0xd7a3; // skip Hangul - no case folding there 1202b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru continue; 1203b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru } 1204b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru // skip Han blocks - no case folding there either 1205b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru if(c==0x3400) { 1206b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru c=0x4db5; 1207b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru continue; 1208b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru } 1209b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru if(c==0x4e00) { 1210b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru c=0x9fa5; 1211b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru continue; 1212b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru } 1213b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru if(c==0x20000) { 1214b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru c=0x2a6d6; 1215b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru continue; 1216b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru } 1217b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 1218b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru s.setTo(c); 1219b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 1220b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru // get leading and trailing cc for c 1221b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru Normalizer::decompose(s, FALSE, 0, d, errorCode); 1222b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru isNFD= s==d; 1223b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru cc=u_getCombiningClass(d.char32At(0)); 1224b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru trailCC=u_getCombiningClass(d.char32At(d.length()-1)); 1225b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 1226b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru // get leading and trailing cc for the case-folding of c 1227b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru s.foldCase(foldingOptions); 1228b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru Normalizer::decompose(s, FALSE, 0, d, errorCode); 1229b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru foldCC=u_getCombiningClass(d.char32At(0)); 1230b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru foldTrailCC=u_getCombiningClass(d.char32At(d.length()-1)); 1231b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 1232b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru qcResult=Normalizer::quickCheck(s, UNORM_FCD, errorCode); 1233b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 1234b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru if (U_FAILURE(errorCode)) { 1235b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru ++count; 123650294ead5e5d23f5bbfed76e00e6b510bd41eee1claireho dataerrln("U+%04lx: Failed with error %s", u_errorName(errorCode)); 1237b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru } 1238b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 1239b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru // bad: 1240b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru // - character maps to empty string: adjacent characters may then need reordering 1241b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru // - folding has different leading/trailing cc's, and they don't become just 0 1242b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru // - folding itself is not FCD 1243b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru if( qcResult!=UNORM_YES || 1244b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru s.isEmpty() || 1245b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru (cc!=foldCC && foldCC!=0) || (trailCC!=foldTrailCC && foldTrailCC!=0) 1246b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru ) { 1247b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru ++count; 124850294ead5e5d23f5bbfed76e00e6b510bd41eee1claireho dataerrln("U+%04lx: case-folding may un-FCD a string (folding options %04lx)", c, foldingOptions); 124950294ead5e5d23f5bbfed76e00e6b510bd41eee1claireho dataerrln(" cc %02x trailCC %02x foldCC(U+%04lx) %02x foldTrailCC(U+%04lx) %02x quickCheck(folded)=%d", cc, trailCC, d.char32At(0), foldCC, d.char32At(d.length()-1), foldTrailCC, qcResult); 1250b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru continue; 1251b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru } 1252b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 1253b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru // also bad: 1254b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru // if a code point is in NFD but its case folding is not, then 1255b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru // unorm_compare will also fail 1256b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru if(isNFD && UNORM_YES!=Normalizer::quickCheck(s, UNORM_NFD, errorCode)) { 1257b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru ++count; 1258b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru errln("U+%04lx: case-folding un-NFDs this character (folding options %04lx)", c, foldingOptions); 1259b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru } 1260b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru } 1261b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 1262b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru logln("There are %ld code points for which case-folding may un-FCD a string (folding options %04lx)", count, foldingOptions); 1263b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru return count; 1264b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru} 1265b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 1266b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queruvoid 1267b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste QueruBasicNormalizerTest::FindFoldFCDExceptions() { 1268b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru int32_t count; 1269b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 1270b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru count=countFoldFCDExceptions(0); 1271b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru count+=countFoldFCDExceptions(U_FOLD_CASE_EXCLUDE_SPECIAL_I); 1272b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru if(count>0) { 1273b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru /* 1274b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru * If case-folding un-FCDs any strings, then unorm_compare() must be 1275b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru * re-implemented. 1276b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru * It currently assumes that one can check for FCD then case-fold 1277b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru * and then still have FCD strings for raw decomposition without reordering. 1278b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru */ 127950294ead5e5d23f5bbfed76e00e6b510bd41eee1claireho dataerrln("error: There are %ld code points for which case-folding may un-FCD a string for all folding options.\n" 1280b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru "See comment in BasicNormalizerTest::FindFoldFCDExceptions()!", count); 1281b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru } 1282b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru} 1283b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 1284b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Querustatic void 128527f654740f2a26ad62a5c155af9199af9e69b889clairehoinitExpectedSkippables(UnicodeSet skipSets[UNORM_MODE_COUNT], UErrorCode &errorCode) { 128627f654740f2a26ad62a5c155af9199af9e69b889claireho skipSets[UNORM_NFD].applyPattern( 128727f654740f2a26ad62a5c155af9199af9e69b889claireho UNICODE_STRING_SIMPLE("[[:NFD_QC=Yes:]&[:ccc=0:]]"), errorCode); 128827f654740f2a26ad62a5c155af9199af9e69b889claireho skipSets[UNORM_NFC].applyPattern( 128927f654740f2a26ad62a5c155af9199af9e69b889claireho UNICODE_STRING_SIMPLE("[[:NFC_QC=Yes:]&[:ccc=0:]-[:HST=LV:]]"), errorCode); 129027f654740f2a26ad62a5c155af9199af9e69b889claireho skipSets[UNORM_NFKD].applyPattern( 129127f654740f2a26ad62a5c155af9199af9e69b889claireho UNICODE_STRING_SIMPLE("[[:NFKD_QC=Yes:]&[:ccc=0:]]"), errorCode); 129227f654740f2a26ad62a5c155af9199af9e69b889claireho skipSets[UNORM_NFKC].applyPattern( 129327f654740f2a26ad62a5c155af9199af9e69b889claireho UNICODE_STRING_SIMPLE("[[:NFKC_QC=Yes:]&[:ccc=0:]-[:HST=LV:]]"), errorCode); 129427f654740f2a26ad62a5c155af9199af9e69b889claireho 129527f654740f2a26ad62a5c155af9199af9e69b889claireho // Remove from the NFC and NFKC sets all those characters that change 129627f654740f2a26ad62a5c155af9199af9e69b889claireho // when a back-combining character is added. 129727f654740f2a26ad62a5c155af9199af9e69b889claireho // First, get all of the back-combining characters and their combining classes. 129827f654740f2a26ad62a5c155af9199af9e69b889claireho UnicodeSet combineBack("[:NFC_QC=Maybe:]", errorCode); 129927f654740f2a26ad62a5c155af9199af9e69b889claireho int32_t numCombineBack=combineBack.size(); 130027f654740f2a26ad62a5c155af9199af9e69b889claireho int32_t *combineBackCharsAndCc=new int32_t[numCombineBack*2]; 130127f654740f2a26ad62a5c155af9199af9e69b889claireho UnicodeSetIterator iter(combineBack); 130227f654740f2a26ad62a5c155af9199af9e69b889claireho for(int32_t i=0; i<numCombineBack; ++i) { 130327f654740f2a26ad62a5c155af9199af9e69b889claireho iter.next(); 130427f654740f2a26ad62a5c155af9199af9e69b889claireho UChar32 c=iter.getCodepoint(); 130527f654740f2a26ad62a5c155af9199af9e69b889claireho combineBackCharsAndCc[2*i]=c; 130627f654740f2a26ad62a5c155af9199af9e69b889claireho combineBackCharsAndCc[2*i+1]=u_getCombiningClass(c); 130727f654740f2a26ad62a5c155af9199af9e69b889claireho } 1308b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 130927f654740f2a26ad62a5c155af9199af9e69b889claireho // We need not look at control codes, Han characters nor Hangul LVT syllables because they 131027f654740f2a26ad62a5c155af9199af9e69b889claireho // do not combine forward. LV syllables are already removed. 131127f654740f2a26ad62a5c155af9199af9e69b889claireho UnicodeSet notInteresting("[[:C:][:Unified_Ideograph:][:HST=LVT:]]", errorCode); 131227f654740f2a26ad62a5c155af9199af9e69b889claireho LocalPointer<UnicodeSet> unsure(&((UnicodeSet *)(skipSets[UNORM_NFC].clone()))->removeAll(notInteresting)); 131327f654740f2a26ad62a5c155af9199af9e69b889claireho // System.out.format("unsure.size()=%d\n", unsure.size()); 131427f654740f2a26ad62a5c155af9199af9e69b889claireho 131527f654740f2a26ad62a5c155af9199af9e69b889claireho // For each character about which we are unsure, see if it changes when we add 131627f654740f2a26ad62a5c155af9199af9e69b889claireho // one of the back-combining characters. 131727f654740f2a26ad62a5c155af9199af9e69b889claireho const Normalizer2 *norm2=Normalizer2::getInstance(NULL, "nfc", UNORM2_COMPOSE, errorCode); 131827f654740f2a26ad62a5c155af9199af9e69b889claireho UnicodeString s; 131927f654740f2a26ad62a5c155af9199af9e69b889claireho iter.reset(*unsure); 132027f654740f2a26ad62a5c155af9199af9e69b889claireho while(iter.next()) { 132127f654740f2a26ad62a5c155af9199af9e69b889claireho UChar32 c=iter.getCodepoint(); 132227f654740f2a26ad62a5c155af9199af9e69b889claireho s.setTo(c); 132327f654740f2a26ad62a5c155af9199af9e69b889claireho int32_t cLength=s.length(); 132427f654740f2a26ad62a5c155af9199af9e69b889claireho int32_t tccc=u_getIntPropertyValue(c, UCHAR_TRAIL_CANONICAL_COMBINING_CLASS); 132527f654740f2a26ad62a5c155af9199af9e69b889claireho for(int32_t i=0; i<numCombineBack; ++i) { 132627f654740f2a26ad62a5c155af9199af9e69b889claireho // If c's decomposition ends with a character with non-zero combining class, then 132727f654740f2a26ad62a5c155af9199af9e69b889claireho // c can only change if it combines with a character with a non-zero combining class. 132827f654740f2a26ad62a5c155af9199af9e69b889claireho int32_t cc2=combineBackCharsAndCc[2*i+1]; 132927f654740f2a26ad62a5c155af9199af9e69b889claireho if(tccc==0 || cc2!=0) { 133027f654740f2a26ad62a5c155af9199af9e69b889claireho UChar32 c2=combineBackCharsAndCc[2*i]; 133127f654740f2a26ad62a5c155af9199af9e69b889claireho s.append(c2); 133227f654740f2a26ad62a5c155af9199af9e69b889claireho if(!norm2->isNormalized(s, errorCode)) { 133327f654740f2a26ad62a5c155af9199af9e69b889claireho // System.out.format("remove U+%04x (tccc=%d) + U+%04x (cc=%d)\n", c, tccc, c2, cc2); 133427f654740f2a26ad62a5c155af9199af9e69b889claireho skipSets[UNORM_NFC].remove(c); 133527f654740f2a26ad62a5c155af9199af9e69b889claireho skipSets[UNORM_NFKC].remove(c); 133627f654740f2a26ad62a5c155af9199af9e69b889claireho break; 133727f654740f2a26ad62a5c155af9199af9e69b889claireho } 133827f654740f2a26ad62a5c155af9199af9e69b889claireho s.truncate(cLength); 133927f654740f2a26ad62a5c155af9199af9e69b889claireho } 134027f654740f2a26ad62a5c155af9199af9e69b889claireho } 134127f654740f2a26ad62a5c155af9199af9e69b889claireho } 134227f654740f2a26ad62a5c155af9199af9e69b889claireho delete [] combineBackCharsAndCc; 1343b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru} 1344b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 1345b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queruvoid 1346b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste QueruBasicNormalizerTest::TestSkippable() { 134750294ead5e5d23f5bbfed76e00e6b510bd41eee1claireho UnicodeSet diff, skipSets[UNORM_MODE_COUNT], expectSets[UNORM_MODE_COUNT]; 1348b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru UnicodeString s, pattern; 1349b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 1350b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru /* build NF*Skippable sets from runtime data */ 135150294ead5e5d23f5bbfed76e00e6b510bd41eee1claireho IcuTestErrorCode errorCode(*this, "TestSkippable"); 135250294ead5e5d23f5bbfed76e00e6b510bd41eee1claireho skipSets[UNORM_NFD].applyPattern(UNICODE_STRING_SIMPLE("[:NFD_Inert:]"), errorCode); 135350294ead5e5d23f5bbfed76e00e6b510bd41eee1claireho skipSets[UNORM_NFKD].applyPattern(UNICODE_STRING_SIMPLE("[:NFKD_Inert:]"), errorCode); 135450294ead5e5d23f5bbfed76e00e6b510bd41eee1claireho skipSets[UNORM_NFC].applyPattern(UNICODE_STRING_SIMPLE("[:NFC_Inert:]"), errorCode); 135550294ead5e5d23f5bbfed76e00e6b510bd41eee1claireho skipSets[UNORM_NFKC].applyPattern(UNICODE_STRING_SIMPLE("[:NFKC_Inert:]"), errorCode); 135650294ead5e5d23f5bbfed76e00e6b510bd41eee1claireho if(errorCode.logDataIfFailureAndReset("UnicodeSet(NF..._Inert) failed")) { 1357b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru return; 1358b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru } 1359b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 1360b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru /* get expected sets from hardcoded patterns */ 136127f654740f2a26ad62a5c155af9199af9e69b889claireho initExpectedSkippables(expectSets, errorCode); 136227f654740f2a26ad62a5c155af9199af9e69b889claireho errorCode.assertSuccess(); 1363b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 136450294ead5e5d23f5bbfed76e00e6b510bd41eee1claireho for(int32_t i=UNORM_NONE; i<UNORM_MODE_COUNT; ++i) { 1365b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru if(skipSets[i]!=expectSets[i]) { 136627f654740f2a26ad62a5c155af9199af9e69b889claireho errln("error: TestSkippable skipSets[%d]!=expectedSets[%d]\n", i, i); 136727f654740f2a26ad62a5c155af9199af9e69b889claireho // Note: This used to depend on hardcoded UnicodeSet patterns generated by 136827f654740f2a26ad62a5c155af9199af9e69b889claireho // Mark's unicodetools.com.ibm.text.UCD.NFSkippable, by 136927f654740f2a26ad62a5c155af9199af9e69b889claireho // running com.ibm.text.UCD.Main with the option NFSkippable. 137027f654740f2a26ad62a5c155af9199af9e69b889claireho // Since ICU 4.6/Unicode 6, we are generating the 137127f654740f2a26ad62a5c155af9199af9e69b889claireho // expectSets ourselves in initSkippables(). 1372b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 1373b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru s=UNICODE_STRING_SIMPLE("skip-expect="); 1374b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru (diff=skipSets[i]).removeAll(expectSets[i]).toPattern(pattern, TRUE); 1375b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru s.append(pattern); 1376b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 1377b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru pattern.remove(); 1378b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru s.append(UNICODE_STRING_SIMPLE("\n\nexpect-skip=")); 1379b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru (diff=expectSets[i]).removeAll(skipSets[i]).toPattern(pattern, TRUE); 1380b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru s.append(pattern); 1381b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru s.append(UNICODE_STRING_SIMPLE("\n\n")); 1382b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 1383b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru errln(s); 1384b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru } 1385b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru } 1386b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru} 1387b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru 138850294ead5e5d23f5bbfed76e00e6b510bd41eee1clairehostruct StringPair { const char *input, *expected; }; 138950294ead5e5d23f5bbfed76e00e6b510bd41eee1claireho 139050294ead5e5d23f5bbfed76e00e6b510bd41eee1clairehovoid 139150294ead5e5d23f5bbfed76e00e6b510bd41eee1clairehoBasicNormalizerTest::TestCustomComp() { 139250294ead5e5d23f5bbfed76e00e6b510bd41eee1claireho static const StringPair pairs[]={ 139350294ead5e5d23f5bbfed76e00e6b510bd41eee1claireho { "\\uD801\\uE000\\uDFFE", "" }, 139450294ead5e5d23f5bbfed76e00e6b510bd41eee1claireho { "\\uD800\\uD801\\uE000\\uDFFE\\uDFFF", "\\uD7FF\\uFFFF" }, 139550294ead5e5d23f5bbfed76e00e6b510bd41eee1claireho { "\\uD800\\uD801\\uDFFE\\uDFFF", "\\uD7FF\\U000107FE\\uFFFF" }, 139650294ead5e5d23f5bbfed76e00e6b510bd41eee1claireho { "\\uE001\\U000110B9\\u0345\\u0308\\u0327", "\\uE002\\U000110B9\\u0327\\u0345" }, 139750294ead5e5d23f5bbfed76e00e6b510bd41eee1claireho { "\\uE010\\U000F0011\\uE012", "\\uE011\\uE012" }, 139850294ead5e5d23f5bbfed76e00e6b510bd41eee1claireho { "\\uE010\\U000F0011\\U000F0011\\uE012", "\\uE011\\U000F0010" }, 139950294ead5e5d23f5bbfed76e00e6b510bd41eee1claireho { "\\uE111\\u1161\\uE112\\u1162", "\\uAE4C\\u1102\\u0062\\u1162" }, 140050294ead5e5d23f5bbfed76e00e6b510bd41eee1claireho { "\\uFFF3\\uFFF7\\U00010036\\U00010077", "\\U00010037\\U00010037\\uFFF6\\U00010037" } 140150294ead5e5d23f5bbfed76e00e6b510bd41eee1claireho }; 140250294ead5e5d23f5bbfed76e00e6b510bd41eee1claireho IcuTestErrorCode errorCode(*this, "BasicNormalizerTest/TestCustomComp"); 140350294ead5e5d23f5bbfed76e00e6b510bd41eee1claireho const Normalizer2 *customNorm2= 140450294ead5e5d23f5bbfed76e00e6b510bd41eee1claireho Normalizer2::getInstance(loadTestData(errorCode), "testnorm", 140550294ead5e5d23f5bbfed76e00e6b510bd41eee1claireho UNORM2_COMPOSE, errorCode); 140627f654740f2a26ad62a5c155af9199af9e69b889claireho if(errorCode.logDataIfFailureAndReset("unable to load testdata/testnorm.nrm")) { 140750294ead5e5d23f5bbfed76e00e6b510bd41eee1claireho return; 140850294ead5e5d23f5bbfed76e00e6b510bd41eee1claireho } 140950294ead5e5d23f5bbfed76e00e6b510bd41eee1claireho for(int32_t i=0; i<LENGTHOF(pairs); ++i) { 141050294ead5e5d23f5bbfed76e00e6b510bd41eee1claireho const StringPair &pair=pairs[i]; 141150294ead5e5d23f5bbfed76e00e6b510bd41eee1claireho UnicodeString input=UnicodeString(pair.input, -1, US_INV).unescape(); 141250294ead5e5d23f5bbfed76e00e6b510bd41eee1claireho UnicodeString expected=UnicodeString(pair.expected, -1, US_INV).unescape(); 141350294ead5e5d23f5bbfed76e00e6b510bd41eee1claireho UnicodeString result=customNorm2->normalize(input, errorCode); 141450294ead5e5d23f5bbfed76e00e6b510bd41eee1claireho if(result!=expected) { 141550294ead5e5d23f5bbfed76e00e6b510bd41eee1claireho errln("custom compose Normalizer2 did not normalize input %d as expected", i); 141650294ead5e5d23f5bbfed76e00e6b510bd41eee1claireho } 141750294ead5e5d23f5bbfed76e00e6b510bd41eee1claireho } 141850294ead5e5d23f5bbfed76e00e6b510bd41eee1claireho} 141950294ead5e5d23f5bbfed76e00e6b510bd41eee1claireho 142050294ead5e5d23f5bbfed76e00e6b510bd41eee1clairehovoid 142150294ead5e5d23f5bbfed76e00e6b510bd41eee1clairehoBasicNormalizerTest::TestCustomFCC() { 142250294ead5e5d23f5bbfed76e00e6b510bd41eee1claireho static const StringPair pairs[]={ 142350294ead5e5d23f5bbfed76e00e6b510bd41eee1claireho { "\\uD801\\uE000\\uDFFE", "" }, 142450294ead5e5d23f5bbfed76e00e6b510bd41eee1claireho { "\\uD800\\uD801\\uE000\\uDFFE\\uDFFF", "\\uD7FF\\uFFFF" }, 142550294ead5e5d23f5bbfed76e00e6b510bd41eee1claireho { "\\uD800\\uD801\\uDFFE\\uDFFF", "\\uD7FF\\U000107FE\\uFFFF" }, 142650294ead5e5d23f5bbfed76e00e6b510bd41eee1claireho // The following expected result is different from CustomComp 142750294ead5e5d23f5bbfed76e00e6b510bd41eee1claireho // because of only-contiguous composition. 142850294ead5e5d23f5bbfed76e00e6b510bd41eee1claireho { "\\uE001\\U000110B9\\u0345\\u0308\\u0327", "\\uE001\\U000110B9\\u0327\\u0308\\u0345" }, 142950294ead5e5d23f5bbfed76e00e6b510bd41eee1claireho { "\\uE010\\U000F0011\\uE012", "\\uE011\\uE012" }, 143050294ead5e5d23f5bbfed76e00e6b510bd41eee1claireho { "\\uE010\\U000F0011\\U000F0011\\uE012", "\\uE011\\U000F0010" }, 143150294ead5e5d23f5bbfed76e00e6b510bd41eee1claireho { "\\uE111\\u1161\\uE112\\u1162", "\\uAE4C\\u1102\\u0062\\u1162" }, 143250294ead5e5d23f5bbfed76e00e6b510bd41eee1claireho { "\\uFFF3\\uFFF7\\U00010036\\U00010077", "\\U00010037\\U00010037\\uFFF6\\U00010037" } 143350294ead5e5d23f5bbfed76e00e6b510bd41eee1claireho }; 143450294ead5e5d23f5bbfed76e00e6b510bd41eee1claireho IcuTestErrorCode errorCode(*this, "BasicNormalizerTest/TestCustomFCC"); 143550294ead5e5d23f5bbfed76e00e6b510bd41eee1claireho const Normalizer2 *customNorm2= 143650294ead5e5d23f5bbfed76e00e6b510bd41eee1claireho Normalizer2::getInstance(loadTestData(errorCode), "testnorm", 143750294ead5e5d23f5bbfed76e00e6b510bd41eee1claireho UNORM2_COMPOSE_CONTIGUOUS, errorCode); 143827f654740f2a26ad62a5c155af9199af9e69b889claireho if(errorCode.logDataIfFailureAndReset("unable to load testdata/testnorm.nrm")) { 143950294ead5e5d23f5bbfed76e00e6b510bd41eee1claireho return; 144050294ead5e5d23f5bbfed76e00e6b510bd41eee1claireho } 144150294ead5e5d23f5bbfed76e00e6b510bd41eee1claireho for(int32_t i=0; i<LENGTHOF(pairs); ++i) { 144250294ead5e5d23f5bbfed76e00e6b510bd41eee1claireho const StringPair &pair=pairs[i]; 144350294ead5e5d23f5bbfed76e00e6b510bd41eee1claireho UnicodeString input=UnicodeString(pair.input, -1, US_INV).unescape(); 144450294ead5e5d23f5bbfed76e00e6b510bd41eee1claireho UnicodeString expected=UnicodeString(pair.expected, -1, US_INV).unescape(); 144550294ead5e5d23f5bbfed76e00e6b510bd41eee1claireho UnicodeString result=customNorm2->normalize(input, errorCode); 144650294ead5e5d23f5bbfed76e00e6b510bd41eee1claireho if(result!=expected) { 144750294ead5e5d23f5bbfed76e00e6b510bd41eee1claireho errln("custom FCC Normalizer2 did not normalize input %d as expected", i); 144850294ead5e5d23f5bbfed76e00e6b510bd41eee1claireho } 144950294ead5e5d23f5bbfed76e00e6b510bd41eee1claireho } 145050294ead5e5d23f5bbfed76e00e6b510bd41eee1claireho} 145150294ead5e5d23f5bbfed76e00e6b510bd41eee1claireho 145227f654740f2a26ad62a5c155af9199af9e69b889claireho/* Improve code coverage of Normalizer2 */ 145327f654740f2a26ad62a5c155af9199af9e69b889clairehovoid 145427f654740f2a26ad62a5c155af9199af9e69b889clairehoBasicNormalizerTest::TestFilteredNormalizer2Coverage() { 145527f654740f2a26ad62a5c155af9199af9e69b889claireho UErrorCode errorCode = U_ZERO_ERROR; 145627f654740f2a26ad62a5c155af9199af9e69b889claireho const Normalizer2 *nfcNorm2=Normalizer2Factory::getNFCInstance(errorCode); 145727f654740f2a26ad62a5c155af9199af9e69b889claireho if (U_FAILURE(errorCode)) { 145827f654740f2a26ad62a5c155af9199af9e69b889claireho dataerrln("Normalizer2Factory::getNFCInstance() call failed - %s", u_errorName(status)); 145927f654740f2a26ad62a5c155af9199af9e69b889claireho return; 146027f654740f2a26ad62a5c155af9199af9e69b889claireho } 146127f654740f2a26ad62a5c155af9199af9e69b889claireho UnicodeSet filter(UNICODE_STRING_SIMPLE("[^\\u00a0-\\u00ff]"), errorCode); 146227f654740f2a26ad62a5c155af9199af9e69b889claireho UnicodeString newString1 = UNICODE_STRING_SIMPLE("[^\\u0100-\\u01ff]"); 146327f654740f2a26ad62a5c155af9199af9e69b889claireho UnicodeString newString2 = UNICODE_STRING_SIMPLE("[^\\u0200-\\u02ff]"); 146427f654740f2a26ad62a5c155af9199af9e69b889claireho FilteredNormalizer2 fn2(*nfcNorm2, filter); 146527f654740f2a26ad62a5c155af9199af9e69b889claireho 146627f654740f2a26ad62a5c155af9199af9e69b889claireho UChar32 char32 = 0x0054; 146727f654740f2a26ad62a5c155af9199af9e69b889claireho 146827f654740f2a26ad62a5c155af9199af9e69b889claireho if (fn2.isInert(char32)) { 146927f654740f2a26ad62a5c155af9199af9e69b889claireho errln("FilteredNormalizer2.isInert() failed."); 147027f654740f2a26ad62a5c155af9199af9e69b889claireho } 147127f654740f2a26ad62a5c155af9199af9e69b889claireho 147227f654740f2a26ad62a5c155af9199af9e69b889claireho if (fn2.hasBoundaryAfter(char32)) { 147327f654740f2a26ad62a5c155af9199af9e69b889claireho errln("FilteredNormalizer2.hasBoundaryAfter() failed."); 147427f654740f2a26ad62a5c155af9199af9e69b889claireho } 147527f654740f2a26ad62a5c155af9199af9e69b889claireho 147627f654740f2a26ad62a5c155af9199af9e69b889claireho fn2.append(newString1, newString2, errorCode); 147727f654740f2a26ad62a5c155af9199af9e69b889claireho if (U_FAILURE(errorCode)) { 147827f654740f2a26ad62a5c155af9199af9e69b889claireho errln("FilteredNormalizer2.append() failed."); 147927f654740f2a26ad62a5c155af9199af9e69b889claireho } 148027f654740f2a26ad62a5c155af9199af9e69b889claireho} 148127f654740f2a26ad62a5c155af9199af9e69b889claireho 1482b13da9df870a61b11249bf741347908dbea0edd8Jean-Baptiste Queru#endif /* #if !UCONFIG_NO_NORMALIZATION */ 1483