1/*
2************************************************************************
3* Copyright (c) 1997-2010, International Business Machines
4* Corporation and others.  All Rights Reserved.
5************************************************************************
6*/
7
8#include "unicode/utypes.h"
9
10#if !UCONFIG_NO_NORMALIZATION
11
12#include "unicode/uchar.h"
13#include "unicode/normlzr.h"
14#include "unicode/uniset.h"
15#include "unicode/putil.h"
16#include "cstring.h"
17#include "filestrm.h"
18#include "normconf.h"
19#include <stdio.h>
20
21#define ARRAY_LENGTH(array) (sizeof(array) / sizeof(array[0]))
22
23#define CASE(id,test,exec) case id:                          \
24                          name = #test;                 \
25                          if (exec) {                   \
26                              logln(#test "---");       \
27                              logln((UnicodeString)""); \
28                              test();                   \
29                          }                             \
30                          break
31
32void NormalizerConformanceTest::runIndexedTest(int32_t index, UBool exec, const char* &name, char* /*par*/) {
33    switch (index) {
34        CASE(0, TestConformance, exec);
35#if !UCONFIG_NO_FILE_IO && !UCONFIG_NO_LEGACY_CONVERSION
36        CASE(1, TestConformance32, exec);
37#endif
38        // CASE(2, TestCase6);
39        default: name = ""; break;
40    }
41}
42
43#define FIELD_COUNT 5
44
45NormalizerConformanceTest::NormalizerConformanceTest() :
46    normalizer(UnicodeString(), UNORM_NFC) {}
47
48NormalizerConformanceTest::~NormalizerConformanceTest() {}
49
50// more interesting conformance test cases, not in the unicode.org NormalizationTest.txt
51static const char *moreCases[]={
52    // Markus 2001aug30
53    "0061 0332 0308;00E4 0332;0061 0332 0308;00E4 0332;0061 0332 0308; # Markus 0",
54
55    // Markus 2001oct26 - test edge case for iteration: U+0f73.cc==0 but decomposition.lead.cc==129
56    "0061 0301 0F73;00E1 0F71 0F72;0061 0F71 0F72 0301;00E1 0F71 0F72;0061 0F71 0F72 0301; # Markus 1"
57};
58
59void NormalizerConformanceTest::compare(const UnicodeString& s1, const UnicodeString& s2){
60    UErrorCode status=U_ZERO_ERROR;
61     // TODO: Re-enable this tests after UTC fixes UAX 21
62    if(s1.indexOf((UChar32)0x0345)>=0)return;
63    if(Normalizer::compare(s1,s2,U_FOLD_CASE_DEFAULT,status)!=0){
64        errln("Normalizer::compare() failed for s1: " + prettify(s1) + " s2: " +prettify(s2));
65    }
66}
67
68FileStream *
69NormalizerConformanceTest::openNormalizationTestFile(const char *filename) {
70    char unidataPath[2000];
71    const char *folder;
72    FileStream *input;
73    UErrorCode errorCode;
74
75    // look inside ICU_DATA first
76    folder=pathToDataDirectory();
77    if(folder!=NULL) {
78        strcpy(unidataPath, folder);
79        strcat(unidataPath, "unidata" U_FILE_SEP_STRING);
80        strcat(unidataPath, filename);
81        input=T_FileStream_open(unidataPath, "rb");
82        if(input!=NULL) {
83            return input;
84        }
85    }
86
87    // find icu/source/data/unidata relative to the test data
88    errorCode=U_ZERO_ERROR;
89    folder=loadTestData(errorCode);
90    if(U_SUCCESS(errorCode)) {
91        strcpy(unidataPath, folder);
92        strcat(unidataPath, U_FILE_SEP_STRING ".." U_FILE_SEP_STRING ".."
93                     U_FILE_SEP_STRING ".." U_FILE_SEP_STRING ".."
94                     U_FILE_SEP_STRING "data" U_FILE_SEP_STRING "unidata" U_FILE_SEP_STRING);
95        strcat(unidataPath, filename);
96        input=T_FileStream_open(unidataPath, "rb");
97        if(input!=NULL) {
98            return input;
99        }
100    }
101
102    // look in icu/source/test/testdata/out/build
103    errorCode=U_ZERO_ERROR;
104    folder=loadTestData(errorCode);
105    if(U_SUCCESS(errorCode)) {
106        strcpy(unidataPath, folder);
107        strcat(unidataPath, U_FILE_SEP_STRING);
108        strcat(unidataPath, filename);
109        input=T_FileStream_open(unidataPath, "rb");
110        if(input!=NULL) {
111            return input;
112        }
113    }
114
115    // look in icu/source/test/testdata
116    errorCode=U_ZERO_ERROR;
117    folder=loadTestData(errorCode);
118    if(U_SUCCESS(errorCode)) {
119        strcpy(unidataPath, folder);
120        strcat(unidataPath, U_FILE_SEP_STRING ".." U_FILE_SEP_STRING ".." U_FILE_SEP_STRING);
121        strcat(unidataPath, filename);
122        input=T_FileStream_open(unidataPath, "rb");
123        if(input!=NULL) {
124            return input;
125        }
126    }
127
128    // find icu/source/data/unidata relative to U_TOPSRCDIR
129#if defined(U_TOPSRCDIR)
130    strcpy(unidataPath, U_TOPSRCDIR U_FILE_SEP_STRING "data" U_FILE_SEP_STRING "unidata" U_FILE_SEP_STRING);
131    strcat(unidataPath, filename);
132    input=T_FileStream_open(unidataPath, "rb");
133    if(input!=NULL) {
134        return input;
135    }
136
137    strcpy(unidataPath, U_TOPSRCDIR U_FILE_SEP_STRING "test" U_FILE_SEP_STRING "testdata" U_FILE_SEP_STRING);
138    strcat(unidataPath, filename);
139    input=T_FileStream_open(unidataPath, "rb");
140    if(input!=NULL) {
141        return input;
142    }
143#endif
144
145    dataerrln("Failed to open %s", filename);
146    return NULL;
147}
148
149/**
150 * Test the conformance of Normalizer to
151 * http://www.unicode.org/Public/UNIDATA/NormalizationTest.txt
152 */
153void NormalizerConformanceTest::TestConformance() {
154    TestConformance(openNormalizationTestFile("NormalizationTest.txt"), 0);
155}
156
157void NormalizerConformanceTest::TestConformance32() {
158    TestConformance(openNormalizationTestFile("NormalizationTest-3.2.0.txt"), UNORM_UNICODE_3_2);
159}
160
161void NormalizerConformanceTest::TestConformance(FileStream *input, int32_t options) {
162    enum { BUF_SIZE = 1024 };
163    char lineBuf[BUF_SIZE];
164    UnicodeString fields[FIELD_COUNT];
165    UErrorCode status = U_ZERO_ERROR;
166    int32_t passCount = 0;
167    int32_t failCount = 0;
168    UChar32 c;
169
170    if(input==NULL) {
171        return;
172    }
173
174    // UnicodeSet for all code points that are not mentioned in NormalizationTest.txt
175    UnicodeSet other(0, 0x10ffff);
176
177    int32_t count, countMoreCases = sizeof(moreCases)/sizeof(moreCases[0]);
178    for (count = 1;;++count) {
179        if (!T_FileStream_eof(input)) {
180            T_FileStream_readLine(input, lineBuf, (int32_t)sizeof(lineBuf));
181        } else {
182            // once NormalizationTest.txt is finished, use moreCases[]
183            if(count > countMoreCases) {
184                count = 0;
185            } else if(count == countMoreCases) {
186                // all done
187                break;
188            }
189            uprv_strcpy(lineBuf, moreCases[count]);
190        }
191        if (lineBuf[0] == 0 || lineBuf[0] == '\n' || lineBuf[0] == '\r') continue;
192
193        // Expect 5 columns of this format:
194        // 1E0C;1E0C;0044 0323;1E0C;0044 0323; # <comments>
195
196        // Parse out the comment.
197        if (lineBuf[0] == '#') continue;
198
199        // Read separator lines starting with '@'
200        if (lineBuf[0] == '@') {
201            logln(lineBuf);
202            continue;
203        }
204
205        // Parse out the fields
206        if (!hexsplit(lineBuf, ';', fields, FIELD_COUNT)) {
207            errln((UnicodeString)"Unable to parse line " + count);
208            break; // Syntax error
209        }
210
211        // Remove a single code point from the "other" UnicodeSet
212        if(fields[0].length()==fields[0].moveIndex32(0, 1)) {
213            c=fields[0].char32At(0);
214            if(0xac20<=c && c<=0xd73f && quick) {
215                // not an exhaustive test run: skip most Hangul syllables
216                if(c==0xac20) {
217                    other.remove(0xac20, 0xd73f);
218                }
219                continue;
220            }
221            other.remove(c);
222        }
223
224        if (checkConformance(fields, lineBuf, options, status)) {
225            ++passCount;
226        } else {
227            ++failCount;
228            if(status == U_FILE_ACCESS_ERROR) {
229              dataerrln("Something is wrong with the normalizer, skipping the rest of the test.");
230              break;
231            }
232        }
233        if ((count % 1000) == 0) {
234            logln("Line %d", count);
235        }
236    }
237
238    T_FileStream_close(input);
239
240    /*
241     * Test that all characters that are not mentioned
242     * as single code points in column 1
243     * do not change under any normalization.
244     */
245
246    // remove U+ffff because that is the end-of-iteration sentinel value
247    other.remove(0xffff);
248
249    for(c=0; c<=0x10ffff; quick ? c+=113 : ++c) {
250        if(0x30000<=c && c<0xe0000) {
251            c=0xe0000;
252        }
253        if(!other.contains(c)) {
254            continue;
255        }
256
257        fields[0]=fields[1]=fields[2]=fields[3]=fields[4].setTo(c);
258        sprintf(lineBuf, "not mentioned code point U+%04lx", (long)c);
259
260        if (checkConformance(fields, lineBuf, options, status)) {
261            ++passCount;
262        } else {
263            ++failCount;
264            if(status == U_FILE_ACCESS_ERROR) {
265              dataerrln("Something is wrong with the normalizer, skipping the rest of the test.: %s", u_errorName(status));
266              break;
267            }
268        }
269        if ((c % 0x1000) == 0) {
270            logln("Code point U+%04lx", c);
271        }
272    }
273
274    if (failCount != 0) {
275        dataerrln((UnicodeString)"Total: " + failCount + " lines/code points failed, " +
276              passCount + " lines/code points passed");
277    } else {
278        logln((UnicodeString)"Total: " + passCount + " lines/code points passed");
279    }
280}
281
282/**
283 * Verify the conformance of the given line of the Unicode
284 * normalization (UTR 15) test suite file.  For each line,
285 * there are five columns, corresponding to field[0]..field[4].
286 *
287 * The following invariants must be true for all conformant implementations
288 *  c2 == NFC(c1) == NFC(c2) == NFC(c3)
289 *  c3 == NFD(c1) == NFD(c2) == NFD(c3)
290 *  c4 == NFKC(c1) == NFKC(c2) == NFKC(c3) == NFKC(c4) == NFKC(c5)
291 *  c5 == NFKD(c1) == NFKD(c2) == NFKD(c3) == NFKD(c4) == NFKD(c5)
292 *
293 * @param field the 5 columns
294 * @param line the source line from the test suite file
295 * @return true if the test passes
296 */
297UBool NormalizerConformanceTest::checkConformance(const UnicodeString* field,
298                                                  const char *line,
299                                                  int32_t options,
300                                                  UErrorCode &status) {
301    UBool pass = TRUE, result;
302    //UErrorCode status = U_ZERO_ERROR;
303    UnicodeString out, fcd;
304    int32_t fieldNum;
305
306    for (int32_t i=0; i<FIELD_COUNT; ++i) {
307        fieldNum = i+1;
308        if (i<3) {
309            Normalizer::normalize(field[i], UNORM_NFC, options, out, status);
310            if (U_FAILURE(status)) {
311                dataerrln("Error running normalize UNORM_NFC: %s", u_errorName(status));
312            } else {
313                pass &= assertEqual("C", field[i], out, field[1], "c2!=C(c", fieldNum);
314                iterativeNorm(field[i], UNORM_NFC, options, out, +1);
315                pass &= assertEqual("C(+1)", field[i], out, field[1], "c2!=C(c", fieldNum);
316                iterativeNorm(field[i], UNORM_NFC, options, out, -1);
317                pass &= assertEqual("C(-1)", field[i], out, field[1], "c2!=C(c", fieldNum);
318            }
319
320            Normalizer::normalize(field[i], UNORM_NFD, options, out, status);
321            if (U_FAILURE(status)) {
322                dataerrln("Error running normalize UNORM_NFD: %s", u_errorName(status));
323            } else {
324                pass &= assertEqual("D", field[i], out, field[2], "c3!=D(c", fieldNum);
325                iterativeNorm(field[i], UNORM_NFD, options, out, +1);
326                pass &= assertEqual("D(+1)", field[i], out, field[2], "c3!=D(c", fieldNum);
327                iterativeNorm(field[i], UNORM_NFD, options, out, -1);
328                pass &= assertEqual("D(-1)", field[i], out, field[2], "c3!=D(c", fieldNum);
329            }
330        }
331        Normalizer::normalize(field[i], UNORM_NFKC, options, out, status);
332        if (U_FAILURE(status)) {
333            dataerrln("Error running normalize UNORM_NFKC: %s", u_errorName(status));
334        } else {
335            pass &= assertEqual("KC", field[i], out, field[3], "c4!=KC(c", fieldNum);
336            iterativeNorm(field[i], UNORM_NFKC, options, out, +1);
337            pass &= assertEqual("KC(+1)", field[i], out, field[3], "c4!=KC(c", fieldNum);
338            iterativeNorm(field[i], UNORM_NFKC, options, out, -1);
339            pass &= assertEqual("KC(-1)", field[i], out, field[3], "c4!=KC(c", fieldNum);
340        }
341
342        Normalizer::normalize(field[i], UNORM_NFKD, options, out, status);
343        if (U_FAILURE(status)) {
344            dataerrln("Error running normalize UNORM_NFKD: %s", u_errorName(status));
345        } else {
346            pass &= assertEqual("KD", field[i], out, field[4], "c5!=KD(c", fieldNum);
347            iterativeNorm(field[i], UNORM_NFKD, options, out, +1);
348            pass &= assertEqual("KD(+1)", field[i], out, field[4], "c5!=KD(c", fieldNum);
349            iterativeNorm(field[i], UNORM_NFKD, options, out, -1);
350            pass &= assertEqual("KD(-1)", field[i], out, field[4], "c5!=KD(c", fieldNum);
351        }
352    }
353    compare(field[1],field[2]);
354    compare(field[0],field[1]);
355    // test quick checks
356    if(UNORM_NO == Normalizer::quickCheck(field[1], UNORM_NFC, options, status)) {
357        errln("Normalizer error: quickCheck(NFC(s), UNORM_NFC) is UNORM_NO");
358        pass = FALSE;
359    }
360    if(UNORM_NO == Normalizer::quickCheck(field[2], UNORM_NFD, options, status)) {
361        errln("Normalizer error: quickCheck(NFD(s), UNORM_NFD) is UNORM_NO");
362        pass = FALSE;
363    }
364    if(UNORM_NO == Normalizer::quickCheck(field[3], UNORM_NFKC, options, status)) {
365        errln("Normalizer error: quickCheck(NFKC(s), UNORM_NFKC) is UNORM_NO");
366        pass = FALSE;
367    }
368    if(UNORM_NO == Normalizer::quickCheck(field[4], UNORM_NFKD, options, status)) {
369        errln("Normalizer error: quickCheck(NFKD(s), UNORM_NFKD) is UNORM_NO");
370        pass = FALSE;
371    }
372
373    // branch on options==0 for better code coverage
374    if(options==0) {
375        result = Normalizer::isNormalized(field[1], UNORM_NFC, status);
376    } else {
377        result = Normalizer::isNormalized(field[1], UNORM_NFC, options, status);
378    }
379    if(!result) {
380        dataerrln("Normalizer error: isNormalized(NFC(s), UNORM_NFC) is FALSE");
381        pass = FALSE;
382    }
383    if(field[0]!=field[1] && Normalizer::isNormalized(field[0], UNORM_NFC, options, status)) {
384        errln("Normalizer error: isNormalized(s, UNORM_NFC) is TRUE");
385        pass = FALSE;
386    }
387    if(!Normalizer::isNormalized(field[3], UNORM_NFKC, options, status)) {
388        dataerrln("Normalizer error: isNormalized(NFKC(s), UNORM_NFKC) is FALSE");
389        pass = FALSE;
390    }
391    if(field[0]!=field[3] && Normalizer::isNormalized(field[0], UNORM_NFKC, options, status)) {
392        errln("Normalizer error: isNormalized(s, UNORM_NFKC) is TRUE");
393        pass = FALSE;
394    }
395
396    // test FCD quick check and "makeFCD"
397    Normalizer::normalize(field[0], UNORM_FCD, options, fcd, status);
398    if(UNORM_NO == Normalizer::quickCheck(fcd, UNORM_FCD, options, status)) {
399        errln("Normalizer error: quickCheck(FCD(s), UNORM_FCD) is UNORM_NO");
400        pass = FALSE;
401    }
402    if(UNORM_NO == Normalizer::quickCheck(field[2], UNORM_FCD, options, status)) {
403        errln("Normalizer error: quickCheck(NFD(s), UNORM_FCD) is UNORM_NO");
404        pass = FALSE;
405    }
406    if(UNORM_NO == Normalizer::quickCheck(field[4], UNORM_FCD, options, status)) {
407        errln("Normalizer error: quickCheck(NFKD(s), UNORM_FCD) is UNORM_NO");
408        pass = FALSE;
409    }
410
411    Normalizer::normalize(fcd, UNORM_NFD, options, out, status);
412    if(out != field[2]) {
413        dataerrln("Normalizer error: NFD(FCD(s))!=NFD(s)");
414        pass = FALSE;
415    }
416
417    if (U_FAILURE(status)) {
418        dataerrln("Normalizer::normalize returned error status: %s", u_errorName(status));
419        pass = FALSE;
420    }
421
422    if(field[0]!=field[2]) {
423        // two strings that are canonically equivalent must test
424        // equal under a canonical caseless match
425        // see UAX #21 Case Mappings and Jitterbug 2021 and
426        // Unicode Technical Committee meeting consensus 92-C31
427        int32_t rc;
428
429        status=U_ZERO_ERROR;
430        rc=Normalizer::compare(field[0], field[2], (options<<UNORM_COMPARE_NORM_OPTIONS_SHIFT)|U_COMPARE_IGNORE_CASE, status);
431        if(U_FAILURE(status)) {
432            dataerrln("Normalizer::compare(case-insensitive) sets %s", u_errorName(status));
433            pass=FALSE;
434        } else if(rc!=0) {
435            errln("Normalizer::compare(original, NFD, case-insensitive) returned %d instead of 0 for equal", rc);
436            pass=FALSE;
437        }
438    }
439
440    if (!pass) {
441        dataerrln("FAIL: %s", line);
442    }
443    return pass;
444}
445
446/**
447 * Do a normalization using the iterative API in the given direction.
448 * @param dir either +1 or -1
449 */
450void NormalizerConformanceTest::iterativeNorm(const UnicodeString& str,
451                                              UNormalizationMode mode, int32_t options,
452                                              UnicodeString& result,
453                                              int8_t dir) {
454    UErrorCode status = U_ZERO_ERROR;
455    normalizer.setText(str, status);
456    normalizer.setMode(mode);
457    normalizer.setOption(-1, 0);        // reset all options
458    normalizer.setOption(options, 1);   // set desired options
459    result.truncate(0);
460    if (U_FAILURE(status)) {
461        return;
462    }
463    UChar32 ch;
464    if (dir > 0) {
465        for (ch = normalizer.first(); ch != Normalizer::DONE;
466             ch = normalizer.next()) {
467            result.append(ch);
468        }
469    } else {
470        for (ch = normalizer.last(); ch != Normalizer::DONE;
471             ch = normalizer.previous()) {
472            result.insert(0, ch);
473        }
474    }
475}
476
477/**
478 * @param op name of normalization form, e.g., "KC"
479 * @param s string being normalized
480 * @param got value received
481 * @param exp expected value
482 * @param msg description of this test
483 * @param return true if got == exp
484 */
485UBool NormalizerConformanceTest::assertEqual(const char *op,
486                                             const UnicodeString& s,
487                                             const UnicodeString& got,
488                                             const UnicodeString& exp,
489                                             const char *msg,
490                                             int32_t field)
491{
492    if (exp == got)
493        return TRUE;
494
495    char *sChars, *gotChars, *expChars;
496    UnicodeString sPretty(prettify(s));
497    UnicodeString gotPretty(prettify(got));
498    UnicodeString expPretty(prettify(exp));
499
500    sChars = new char[sPretty.length() + 1];
501    gotChars = new char[gotPretty.length() + 1];
502    expChars = new char[expPretty.length() + 1];
503
504    sPretty.extract(0, sPretty.length(), sChars, sPretty.length() + 1);
505    sChars[sPretty.length()] = 0;
506    gotPretty.extract(0, gotPretty.length(), gotChars, gotPretty.length() + 1);
507    gotChars[gotPretty.length()] = 0;
508    expPretty.extract(0, expPretty.length(), expChars, expPretty.length() + 1);
509    expChars[expPretty.length()] = 0;
510
511    errln("    %s%d)%s(%s)=%s, exp. %s", msg, field, op, sChars, gotChars, expChars);
512
513    delete []sChars;
514    delete []gotChars;
515    delete []expChars;
516    return FALSE;
517}
518
519/**
520 * Split a string into pieces based on the given delimiter
521 * character.  Then, parse the resultant fields from hex into
522 * characters.  That is, "0040 0400;0C00;0899" -> new String[] {
523 * "\u0040\u0400", "\u0C00", "\u0899" }.  The output is assumed to
524 * be of the proper length already, and exactly output.length
525 * fields are parsed.  If there are too few an exception is
526 * thrown.  If there are too many the extras are ignored.
527 *
528 * @return FALSE upon failure
529 */
530UBool NormalizerConformanceTest::hexsplit(const char *s, char delimiter,
531                                          UnicodeString output[], int32_t outputLength) {
532    const char *t = s;
533    char *end = NULL;
534    UChar32 c;
535    int32_t i;
536    for (i=0; i<outputLength; ++i) {
537        // skip whitespace
538        while(*t == ' ' || *t == '\t') {
539            ++t;
540        }
541
542        // read a sequence of code points
543        output[i].remove();
544        for(;;) {
545            c = (UChar32)uprv_strtoul(t, &end, 16);
546
547            if( (char *)t == end ||
548                (uint32_t)c > 0x10ffff ||
549                (*end != ' ' && *end != '\t' && *end != delimiter)
550            ) {
551                errln(UnicodeString("Bad field ", "") + (i + 1) + " in " + UnicodeString(s, ""));
552                return FALSE;
553            }
554
555            output[i].append(c);
556
557            t = (const char *)end;
558
559            // skip whitespace
560            while(*t == ' ' || *t == '\t') {
561                ++t;
562            }
563
564            if(*t == delimiter) {
565                ++t;
566                break;
567            }
568            if(*t == 0) {
569                if((i + 1) == outputLength) {
570                    return TRUE;
571                } else {
572                    errln(UnicodeString("Missing field(s) in ", "") + s + " only " + (i + 1) + " out of " + outputLength);
573                    return FALSE;
574                }
575            }
576        }
577    }
578    return TRUE;
579}
580
581// Specific tests for debugging.  These are generally failures taken from
582// the conformance file, but culled out to make debugging easier.
583
584void NormalizerConformanceTest::TestCase6(void) {
585    _testOneLine("0385;0385;00A8 0301;0020 0308 0301;0020 0308 0301;");
586}
587
588void NormalizerConformanceTest::_testOneLine(const char *line) {
589  UErrorCode status = U_ZERO_ERROR;
590    UnicodeString fields[FIELD_COUNT];
591    if (!hexsplit(line, ';', fields, FIELD_COUNT)) {
592        errln((UnicodeString)"Unable to parse line " + line);
593    } else {
594        checkConformance(fields, line, 0, status);
595    }
596}
597
598#endif /* #if !UCONFIG_NO_NORMALIZATION */
599