1f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles)/* 2f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles)************************************************************************ 3f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles)* Copyright (c) 1997-2010, International Business Machines 4f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles)* Corporation and others. All Rights Reserved. 5f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles)************************************************************************ 6f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles)*/ 7f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) 8f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles)#include "unicode/utypes.h" 9f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) 10f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles)#if !UCONFIG_NO_NORMALIZATION 11f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) 12f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles)#include "unicode/uchar.h" 13f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles)#include "unicode/normlzr.h" 14f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles)#include "unicode/uniset.h" 15f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles)#include "unicode/putil.h" 16f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles)#include "cstring.h" 17f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles)#include "filestrm.h" 18f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles)#include "normconf.h" 19f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles)#include <stdio.h> 20f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) 21f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles)#define ARRAY_LENGTH(array) (sizeof(array) / sizeof(array[0])) 22f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) 23f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles)#define CASE(id,test,exec) case id: \ 24f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) name = #test; \ 25f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) if (exec) { \ 26f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) logln(#test "---"); \ 27f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) logln((UnicodeString)""); \ 28f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) test(); \ 29f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) } \ 30f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) break 31f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) 32f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles)void NormalizerConformanceTest::runIndexedTest(int32_t index, UBool exec, const char* &name, char* /*par*/) { 33f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) switch (index) { 34f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) CASE(0, TestConformance, exec); 35f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles)#if !UCONFIG_NO_FILE_IO && !UCONFIG_NO_LEGACY_CONVERSION 36f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) CASE(1, TestConformance32, exec); 37f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles)#endif 38f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) // CASE(2, TestCase6); 39f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) default: name = ""; break; 40f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) } 41f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles)} 42f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) 43f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles)#define FIELD_COUNT 5 44f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) 45f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles)NormalizerConformanceTest::NormalizerConformanceTest() : 46f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) normalizer(UnicodeString(), UNORM_NFC) {} 47f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) 48f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles)NormalizerConformanceTest::~NormalizerConformanceTest() {} 49f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) 50f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles)// more interesting conformance test cases, not in the unicode.org NormalizationTest.txt 51f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles)static const char *moreCases[]={ 52f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) // Markus 2001aug30 53f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) "0061 0332 0308;00E4 0332;0061 0332 0308;00E4 0332;0061 0332 0308; # Markus 0", 54f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) 55f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) // Markus 2001oct26 - test edge case for iteration: U+0f73.cc==0 but decomposition.lead.cc==129 56f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) "0061 0301 0F73;00E1 0F71 0F72;0061 0F71 0F72 0301;00E1 0F71 0F72;0061 0F71 0F72 0301; # Markus 1" 57f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles)}; 58f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) 59f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles)void NormalizerConformanceTest::compare(const UnicodeString& s1, const UnicodeString& s2){ 60f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) UErrorCode status=U_ZERO_ERROR; 61f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) // TODO: Re-enable this tests after UTC fixes UAX 21 62f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) if(s1.indexOf((UChar32)0x0345)>=0)return; 63f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) if(Normalizer::compare(s1,s2,U_FOLD_CASE_DEFAULT,status)!=0){ 64f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) errln("Normalizer::compare() failed for s1: " + prettify(s1) + " s2: " +prettify(s2)); 65f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) } 66f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles)} 67f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) 68f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles)FileStream * 69f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles)NormalizerConformanceTest::openNormalizationTestFile(const char *filename) { 70f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) char unidataPath[2000]; 71f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) const char *folder; 72f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) FileStream *input; 73f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) UErrorCode errorCode; 74f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) 75f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) // look inside ICU_DATA first 76f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) folder=pathToDataDirectory(); 77f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) if(folder!=NULL) { 78f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) strcpy(unidataPath, folder); 79f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) strcat(unidataPath, "unidata" U_FILE_SEP_STRING); 80f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) strcat(unidataPath, filename); 81f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) input=T_FileStream_open(unidataPath, "rb"); 82f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) if(input!=NULL) { 83f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) return input; 84f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) } 85f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) } 86f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) 87f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) // find icu/source/data/unidata relative to the test data 88f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) errorCode=U_ZERO_ERROR; 89f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) folder=loadTestData(errorCode); 90f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) if(U_SUCCESS(errorCode)) { 91f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) strcpy(unidataPath, folder); 92f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) strcat(unidataPath, U_FILE_SEP_STRING ".." U_FILE_SEP_STRING ".." 93f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) U_FILE_SEP_STRING ".." U_FILE_SEP_STRING ".." 94f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) U_FILE_SEP_STRING "data" U_FILE_SEP_STRING "unidata" U_FILE_SEP_STRING); 95f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) strcat(unidataPath, filename); 96f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) input=T_FileStream_open(unidataPath, "rb"); 97f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) if(input!=NULL) { 98f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) return input; 99f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) } 100f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) } 101f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) 102f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) // look in icu/source/test/testdata/out/build 103f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) errorCode=U_ZERO_ERROR; 104f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) folder=loadTestData(errorCode); 105f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) if(U_SUCCESS(errorCode)) { 106f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) strcpy(unidataPath, folder); 107f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) strcat(unidataPath, U_FILE_SEP_STRING); 108f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) strcat(unidataPath, filename); 109f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) input=T_FileStream_open(unidataPath, "rb"); 110f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) if(input!=NULL) { 111f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) return input; 112f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) } 113f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) } 114f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) 115f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) // look in icu/source/test/testdata 116f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) errorCode=U_ZERO_ERROR; 117f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) folder=loadTestData(errorCode); 118f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) if(U_SUCCESS(errorCode)) { 119f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) strcpy(unidataPath, folder); 120f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) strcat(unidataPath, U_FILE_SEP_STRING ".." U_FILE_SEP_STRING ".." U_FILE_SEP_STRING); 121f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) strcat(unidataPath, filename); 122f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) input=T_FileStream_open(unidataPath, "rb"); 123f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) if(input!=NULL) { 124f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) return input; 125f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) } 126f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) } 127f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) 128f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) // find icu/source/data/unidata relative to U_TOPSRCDIR 129f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles)#if defined(U_TOPSRCDIR) 130f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) strcpy(unidataPath, U_TOPSRCDIR U_FILE_SEP_STRING "data" U_FILE_SEP_STRING "unidata" U_FILE_SEP_STRING); 131f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) strcat(unidataPath, filename); 132f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) input=T_FileStream_open(unidataPath, "rb"); 133f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) if(input!=NULL) { 134f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) return input; 135f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) } 136f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) 137f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) strcpy(unidataPath, U_TOPSRCDIR U_FILE_SEP_STRING "test" U_FILE_SEP_STRING "testdata" U_FILE_SEP_STRING); 138f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) strcat(unidataPath, filename); 139f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) input=T_FileStream_open(unidataPath, "rb"); 140f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) if(input!=NULL) { 141f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) return input; 142f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) } 143f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles)#endif 144f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) 145f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) dataerrln("Failed to open %s", filename); 146f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) return NULL; 147f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles)} 148f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) 149f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles)/** 150f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) * Test the conformance of Normalizer to 151f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) * http://www.unicode.org/Public/UNIDATA/NormalizationTest.txt 152f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) */ 153f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles)void NormalizerConformanceTest::TestConformance() { 154f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) TestConformance(openNormalizationTestFile("NormalizationTest.txt"), 0); 155f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles)} 156f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) 157f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles)void NormalizerConformanceTest::TestConformance32() { 158f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) TestConformance(openNormalizationTestFile("NormalizationTest-3.2.0.txt"), UNORM_UNICODE_3_2); 159f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles)} 160f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) 161f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles)void NormalizerConformanceTest::TestConformance(FileStream *input, int32_t options) { 162f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) enum { BUF_SIZE = 1024 }; 163f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) char lineBuf[BUF_SIZE]; 164f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) UnicodeString fields[FIELD_COUNT]; 165f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) UErrorCode status = U_ZERO_ERROR; 166f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) int32_t passCount = 0; 167f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) int32_t failCount = 0; 168f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) UChar32 c; 169f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) 170f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) if(input==NULL) { 171f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) return; 172f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) } 173f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) 174f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) // UnicodeSet for all code points that are not mentioned in NormalizationTest.txt 175f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) UnicodeSet other(0, 0x10ffff); 176f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) 177f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) int32_t count, countMoreCases = sizeof(moreCases)/sizeof(moreCases[0]); 178f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) for (count = 1;;++count) { 179f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) if (!T_FileStream_eof(input)) { 180f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) T_FileStream_readLine(input, lineBuf, (int32_t)sizeof(lineBuf)); 181f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) } else { 182f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) // once NormalizationTest.txt is finished, use moreCases[] 183f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) if(count > countMoreCases) { 184f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) count = 0; 185f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) } else if(count == countMoreCases) { 186f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) // all done 187f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) break; 188f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) } 189f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) uprv_strcpy(lineBuf, moreCases[count]); 190f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) } 191f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) if (lineBuf[0] == 0 || lineBuf[0] == '\n' || lineBuf[0] == '\r') continue; 192f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) 193f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) // Expect 5 columns of this format: 194f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) // 1E0C;1E0C;0044 0323;1E0C;0044 0323; # <comments> 195f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) 196f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) // Parse out the comment. 197f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) if (lineBuf[0] == '#') continue; 198f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) 199f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) // Read separator lines starting with '@' 200f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) if (lineBuf[0] == '@') { 201f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) logln(lineBuf); 202f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) continue; 203f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) } 204f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) 205f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) // Parse out the fields 206f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) if (!hexsplit(lineBuf, ';', fields, FIELD_COUNT)) { 207f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) errln((UnicodeString)"Unable to parse line " + count); 208f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) break; // Syntax error 209f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) } 210f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) 211f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) // Remove a single code point from the "other" UnicodeSet 212f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) if(fields[0].length()==fields[0].moveIndex32(0, 1)) { 213f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) c=fields[0].char32At(0); 214f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) if(0xac20<=c && c<=0xd73f && quick) { 215f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) // not an exhaustive test run: skip most Hangul syllables 216f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) if(c==0xac20) { 217f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) other.remove(0xac20, 0xd73f); 218f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) } 219f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) continue; 220f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) } 221f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) other.remove(c); 222f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) } 223f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) 224f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) if (checkConformance(fields, lineBuf, options, status)) { 225f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) ++passCount; 226f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) } else { 227f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) ++failCount; 228f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) if(status == U_FILE_ACCESS_ERROR) { 229f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) dataerrln("Something is wrong with the normalizer, skipping the rest of the test."); 230f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) break; 231f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) } 232f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) } 233f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) if ((count % 1000) == 0) { 234f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) logln("Line %d", count); 235f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) } 236f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) } 237f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) 238f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) T_FileStream_close(input); 239f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) 240f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) /* 241f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) * Test that all characters that are not mentioned 242f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) * as single code points in column 1 243f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) * do not change under any normalization. 244f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) */ 245f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) 246f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) // remove U+ffff because that is the end-of-iteration sentinel value 247f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) other.remove(0xffff); 248f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) 249f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) for(c=0; c<=0x10ffff; quick ? c+=113 : ++c) { 250f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) if(0x30000<=c && c<0xe0000) { 251f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) c=0xe0000; 252f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) } 253f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) if(!other.contains(c)) { 254f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) continue; 255f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) } 256f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) 257f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) fields[0]=fields[1]=fields[2]=fields[3]=fields[4].setTo(c); 258f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) sprintf(lineBuf, "not mentioned code point U+%04lx", (long)c); 259f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) 260f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) if (checkConformance(fields, lineBuf, options, status)) { 261f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) ++passCount; 262f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) } else { 263f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) ++failCount; 264f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) if(status == U_FILE_ACCESS_ERROR) { 265f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) dataerrln("Something is wrong with the normalizer, skipping the rest of the test.: %s", u_errorName(status)); 266f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) break; 267f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) } 268f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) } 269f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) if ((c % 0x1000) == 0) { 270f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) logln("Code point U+%04lx", c); 271f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) } 272f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) } 273f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) 274f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) if (failCount != 0) { 275f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) dataerrln((UnicodeString)"Total: " + failCount + " lines/code points failed, " + 276f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) passCount + " lines/code points passed"); 277f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) } else { 278f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) logln((UnicodeString)"Total: " + passCount + " lines/code points passed"); 279f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) } 280f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles)} 281f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) 282f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles)/** 283f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) * Verify the conformance of the given line of the Unicode 284f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) * normalization (UTR 15) test suite file. For each line, 285f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) * there are five columns, corresponding to field[0]..field[4]. 286f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) * 287f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) * The following invariants must be true for all conformant implementations 288f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) * c2 == NFC(c1) == NFC(c2) == NFC(c3) 289f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) * c3 == NFD(c1) == NFD(c2) == NFD(c3) 290f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) * c4 == NFKC(c1) == NFKC(c2) == NFKC(c3) == NFKC(c4) == NFKC(c5) 291f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) * c5 == NFKD(c1) == NFKD(c2) == NFKD(c3) == NFKD(c4) == NFKD(c5) 292f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) * 293f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) * @param field the 5 columns 294f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) * @param line the source line from the test suite file 295f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) * @return true if the test passes 296f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) */ 297f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles)UBool NormalizerConformanceTest::checkConformance(const UnicodeString* field, 298f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) const char *line, 299f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) int32_t options, 300f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) UErrorCode &status) { 301f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) UBool pass = TRUE, result; 302f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) //UErrorCode status = U_ZERO_ERROR; 303f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) UnicodeString out, fcd; 304f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) int32_t fieldNum; 305f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) 306f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) for (int32_t i=0; i<FIELD_COUNT; ++i) { 307f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) fieldNum = i+1; 308f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) if (i<3) { 309f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) Normalizer::normalize(field[i], UNORM_NFC, options, out, status); 310f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) if (U_FAILURE(status)) { 311f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) dataerrln("Error running normalize UNORM_NFC: %s", u_errorName(status)); 312f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) } else { 313f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) pass &= assertEqual("C", field[i], out, field[1], "c2!=C(c", fieldNum); 314f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) iterativeNorm(field[i], UNORM_NFC, options, out, +1); 315f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) pass &= assertEqual("C(+1)", field[i], out, field[1], "c2!=C(c", fieldNum); 316f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) iterativeNorm(field[i], UNORM_NFC, options, out, -1); 317f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) pass &= assertEqual("C(-1)", field[i], out, field[1], "c2!=C(c", fieldNum); 318f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) } 319f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) 320f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) Normalizer::normalize(field[i], UNORM_NFD, options, out, status); 321f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) if (U_FAILURE(status)) { 322f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) dataerrln("Error running normalize UNORM_NFD: %s", u_errorName(status)); 323f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) } else { 324f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) pass &= assertEqual("D", field[i], out, field[2], "c3!=D(c", fieldNum); 325f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) iterativeNorm(field[i], UNORM_NFD, options, out, +1); 326f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) pass &= assertEqual("D(+1)", field[i], out, field[2], "c3!=D(c", fieldNum); 327f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) iterativeNorm(field[i], UNORM_NFD, options, out, -1); 328f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) pass &= assertEqual("D(-1)", field[i], out, field[2], "c3!=D(c", fieldNum); 329f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) } 330f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) } 331f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) Normalizer::normalize(field[i], UNORM_NFKC, options, out, status); 332f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) if (U_FAILURE(status)) { 333f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) dataerrln("Error running normalize UNORM_NFKC: %s", u_errorName(status)); 334f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) } else { 335f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) pass &= assertEqual("KC", field[i], out, field[3], "c4!=KC(c", fieldNum); 336f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) iterativeNorm(field[i], UNORM_NFKC, options, out, +1); 337f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) pass &= assertEqual("KC(+1)", field[i], out, field[3], "c4!=KC(c", fieldNum); 338f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) iterativeNorm(field[i], UNORM_NFKC, options, out, -1); 339f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) pass &= assertEqual("KC(-1)", field[i], out, field[3], "c4!=KC(c", fieldNum); 340f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) } 341f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) 342f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) Normalizer::normalize(field[i], UNORM_NFKD, options, out, status); 343f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) if (U_FAILURE(status)) { 344f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) dataerrln("Error running normalize UNORM_NFKD: %s", u_errorName(status)); 345f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) } else { 346f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) pass &= assertEqual("KD", field[i], out, field[4], "c5!=KD(c", fieldNum); 347f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) iterativeNorm(field[i], UNORM_NFKD, options, out, +1); 348f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) pass &= assertEqual("KD(+1)", field[i], out, field[4], "c5!=KD(c", fieldNum); 349f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) iterativeNorm(field[i], UNORM_NFKD, options, out, -1); 350f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) pass &= assertEqual("KD(-1)", field[i], out, field[4], "c5!=KD(c", fieldNum); 351f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) } 352f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) } 353f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) compare(field[1],field[2]); 354f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) compare(field[0],field[1]); 355f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) // test quick checks 356f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) if(UNORM_NO == Normalizer::quickCheck(field[1], UNORM_NFC, options, status)) { 357f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) errln("Normalizer error: quickCheck(NFC(s), UNORM_NFC) is UNORM_NO"); 358f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) pass = FALSE; 359f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) } 360f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) if(UNORM_NO == Normalizer::quickCheck(field[2], UNORM_NFD, options, status)) { 361f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) errln("Normalizer error: quickCheck(NFD(s), UNORM_NFD) is UNORM_NO"); 362f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) pass = FALSE; 363f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) } 364f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) if(UNORM_NO == Normalizer::quickCheck(field[3], UNORM_NFKC, options, status)) { 365f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) errln("Normalizer error: quickCheck(NFKC(s), UNORM_NFKC) is UNORM_NO"); 366f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) pass = FALSE; 367f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) } 368f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) if(UNORM_NO == Normalizer::quickCheck(field[4], UNORM_NFKD, options, status)) { 369f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) errln("Normalizer error: quickCheck(NFKD(s), UNORM_NFKD) is UNORM_NO"); 370f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) pass = FALSE; 371f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) } 372f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) 373f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) // branch on options==0 for better code coverage 374f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) if(options==0) { 375f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) result = Normalizer::isNormalized(field[1], UNORM_NFC, status); 376f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) } else { 377f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) result = Normalizer::isNormalized(field[1], UNORM_NFC, options, status); 378f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) } 379f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) if(!result) { 380f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) dataerrln("Normalizer error: isNormalized(NFC(s), UNORM_NFC) is FALSE"); 381f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) pass = FALSE; 382f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) } 383f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) if(field[0]!=field[1] && Normalizer::isNormalized(field[0], UNORM_NFC, options, status)) { 384f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) errln("Normalizer error: isNormalized(s, UNORM_NFC) is TRUE"); 385f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) pass = FALSE; 386f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) } 387f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) if(!Normalizer::isNormalized(field[3], UNORM_NFKC, options, status)) { 388f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) dataerrln("Normalizer error: isNormalized(NFKC(s), UNORM_NFKC) is FALSE"); 389f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) pass = FALSE; 390f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) } 391f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) if(field[0]!=field[3] && Normalizer::isNormalized(field[0], UNORM_NFKC, options, status)) { 392f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) errln("Normalizer error: isNormalized(s, UNORM_NFKC) is TRUE"); 393f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) pass = FALSE; 394f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) } 395f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) 396f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) // test FCD quick check and "makeFCD" 397f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) Normalizer::normalize(field[0], UNORM_FCD, options, fcd, status); 398f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) if(UNORM_NO == Normalizer::quickCheck(fcd, UNORM_FCD, options, status)) { 399f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) errln("Normalizer error: quickCheck(FCD(s), UNORM_FCD) is UNORM_NO"); 400f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) pass = FALSE; 401f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) } 402f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) if(UNORM_NO == Normalizer::quickCheck(field[2], UNORM_FCD, options, status)) { 403f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) errln("Normalizer error: quickCheck(NFD(s), UNORM_FCD) is UNORM_NO"); 404f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) pass = FALSE; 405f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) } 406f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) if(UNORM_NO == Normalizer::quickCheck(field[4], UNORM_FCD, options, status)) { 407f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) errln("Normalizer error: quickCheck(NFKD(s), UNORM_FCD) is UNORM_NO"); 408f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) pass = FALSE; 409f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) } 410f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) 411f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) Normalizer::normalize(fcd, UNORM_NFD, options, out, status); 412f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) if(out != field[2]) { 413f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) dataerrln("Normalizer error: NFD(FCD(s))!=NFD(s)"); 414f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) pass = FALSE; 415f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) } 416f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) 417f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) if (U_FAILURE(status)) { 418f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) dataerrln("Normalizer::normalize returned error status: %s", u_errorName(status)); 419f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) pass = FALSE; 420f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) } 421f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) 422f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) if(field[0]!=field[2]) { 423f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) // two strings that are canonically equivalent must test 424f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) // equal under a canonical caseless match 425f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) // see UAX #21 Case Mappings and Jitterbug 2021 and 426f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) // Unicode Technical Committee meeting consensus 92-C31 427f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) int32_t rc; 428f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) 429f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) status=U_ZERO_ERROR; 430f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) rc=Normalizer::compare(field[0], field[2], (options<<UNORM_COMPARE_NORM_OPTIONS_SHIFT)|U_COMPARE_IGNORE_CASE, status); 431f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) if(U_FAILURE(status)) { 432f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) dataerrln("Normalizer::compare(case-insensitive) sets %s", u_errorName(status)); 433f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) pass=FALSE; 434f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) } else if(rc!=0) { 435f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) errln("Normalizer::compare(original, NFD, case-insensitive) returned %d instead of 0 for equal", rc); 436f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) pass=FALSE; 437f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) } 438f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) } 439f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) 440f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) if (!pass) { 441f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) dataerrln("FAIL: %s", line); 442f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) } 443f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) return pass; 444f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles)} 445f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) 446f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles)/** 447f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) * Do a normalization using the iterative API in the given direction. 448f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) * @param dir either +1 or -1 449f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) */ 450f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles)void NormalizerConformanceTest::iterativeNorm(const UnicodeString& str, 451f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) UNormalizationMode mode, int32_t options, 452f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) UnicodeString& result, 453f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) int8_t dir) { 454f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) UErrorCode status = U_ZERO_ERROR; 455f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) normalizer.setText(str, status); 456f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) normalizer.setMode(mode); 457f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) normalizer.setOption(-1, 0); // reset all options 458f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) normalizer.setOption(options, 1); // set desired options 459f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) result.truncate(0); 460f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) if (U_FAILURE(status)) { 461f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) return; 462f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) } 463f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) UChar32 ch; 464f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) if (dir > 0) { 465f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) for (ch = normalizer.first(); ch != Normalizer::DONE; 466f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) ch = normalizer.next()) { 467f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) result.append(ch); 468f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) } 469f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) } else { 470f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) for (ch = normalizer.last(); ch != Normalizer::DONE; 471f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) ch = normalizer.previous()) { 472f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) result.insert(0, ch); 473f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) } 474f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) } 475f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles)} 476f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) 477f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles)/** 478f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) * @param op name of normalization form, e.g., "KC" 479f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) * @param s string being normalized 480f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) * @param got value received 481f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) * @param exp expected value 482f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) * @param msg description of this test 483f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) * @param return true if got == exp 484f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) */ 485f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles)UBool NormalizerConformanceTest::assertEqual(const char *op, 486f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) const UnicodeString& s, 487f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) const UnicodeString& got, 488f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) const UnicodeString& exp, 489f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) const char *msg, 490f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) int32_t field) 491f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles){ 492f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) if (exp == got) 493f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) return TRUE; 494f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) 495f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) char *sChars, *gotChars, *expChars; 496f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) UnicodeString sPretty(prettify(s)); 497f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) UnicodeString gotPretty(prettify(got)); 498f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) UnicodeString expPretty(prettify(exp)); 499f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) 500f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) sChars = new char[sPretty.length() + 1]; 501f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) gotChars = new char[gotPretty.length() + 1]; 502f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) expChars = new char[expPretty.length() + 1]; 503f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) 504f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) sPretty.extract(0, sPretty.length(), sChars, sPretty.length() + 1); 505f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) sChars[sPretty.length()] = 0; 506f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) gotPretty.extract(0, gotPretty.length(), gotChars, gotPretty.length() + 1); 507f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) gotChars[gotPretty.length()] = 0; 508f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) expPretty.extract(0, expPretty.length(), expChars, expPretty.length() + 1); 509f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) expChars[expPretty.length()] = 0; 510f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) 511f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) errln(" %s%d)%s(%s)=%s, exp. %s", msg, field, op, sChars, gotChars, expChars); 512f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) 513f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) delete []sChars; 514f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) delete []gotChars; 515f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) delete []expChars; 516f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) return FALSE; 517f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles)} 518f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) 519f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles)/** 520f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) * Split a string into pieces based on the given delimiter 521f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) * character. Then, parse the resultant fields from hex into 522f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) * characters. That is, "0040 0400;0C00;0899" -> new String[] { 523f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) * "\u0040\u0400", "\u0C00", "\u0899" }. The output is assumed to 524f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) * be of the proper length already, and exactly output.length 525f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) * fields are parsed. If there are too few an exception is 526f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) * thrown. If there are too many the extras are ignored. 527f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) * 528f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) * @return FALSE upon failure 529f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) */ 530f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles)UBool NormalizerConformanceTest::hexsplit(const char *s, char delimiter, 531f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) UnicodeString output[], int32_t outputLength) { 532f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) const char *t = s; 533f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) char *end = NULL; 534f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) UChar32 c; 535f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) int32_t i; 536f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) for (i=0; i<outputLength; ++i) { 537f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) // skip whitespace 538f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) while(*t == ' ' || *t == '\t') { 539f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) ++t; 540f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) } 541f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) 542f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) // read a sequence of code points 543f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) output[i].remove(); 544f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) for(;;) { 545f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) c = (UChar32)uprv_strtoul(t, &end, 16); 546f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) 547f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) if( (char *)t == end || 548f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) (uint32_t)c > 0x10ffff || 549f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) (*end != ' ' && *end != '\t' && *end != delimiter) 550f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) ) { 551f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) errln(UnicodeString("Bad field ", "") + (i + 1) + " in " + UnicodeString(s, "")); 552f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) return FALSE; 553f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) } 554f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) 555f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) output[i].append(c); 556f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) 557f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) t = (const char *)end; 558f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) 559f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) // skip whitespace 560f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) while(*t == ' ' || *t == '\t') { 561f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) ++t; 562f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) } 563f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) 564f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) if(*t == delimiter) { 565f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) ++t; 566f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) break; 567f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) } 568f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) if(*t == 0) { 569f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) if((i + 1) == outputLength) { 570f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) return TRUE; 571f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) } else { 572f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) errln(UnicodeString("Missing field(s) in ", "") + s + " only " + (i + 1) + " out of " + outputLength); 573f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) return FALSE; 574f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) } 575f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) } 576f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) } 577f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) } 578f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) return TRUE; 579f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles)} 580f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) 581f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles)// Specific tests for debugging. These are generally failures taken from 582f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles)// the conformance file, but culled out to make debugging easier. 583f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) 584f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles)void NormalizerConformanceTest::TestCase6(void) { 585f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) _testOneLine("0385;0385;00A8 0301;0020 0308 0301;0020 0308 0301;"); 586f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles)} 587f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) 588f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles)void NormalizerConformanceTest::_testOneLine(const char *line) { 589f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) UErrorCode status = U_ZERO_ERROR; 590f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) UnicodeString fields[FIELD_COUNT]; 591f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) if (!hexsplit(line, ';', fields, FIELD_COUNT)) { 592f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) errln((UnicodeString)"Unable to parse line " + line); 593f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) } else { 594f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) checkConformance(fields, line, 0, status); 595f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) } 596f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles)} 597f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles) 598f4ed1cf5d184064c4cf0e4359c6d5d8aadb50afaTorne (Richard Coles)#endif /* #if !UCONFIG_NO_NORMALIZATION */ 599