1/********************************************************************
2 * COPYRIGHT:
3 * Copyright (c) 1997-2013, International Business Machines Corporation and
4 * others. All Rights Reserved.
5 ********************************************************************/
6
7#include "unicode/ustring.h"
8#include "unicode/uchar.h"
9#include "unicode/uniset.h"
10#include "unicode/putil.h"
11#include "unicode/uscript.h"
12#include "cstring.h"
13#include "hash.h"
14#include "patternprops.h"
15#include "normalizer2impl.h"
16#include "uparse.h"
17#include "ucdtest.h"
18
19#define LENGTHOF(array) (int32_t)(sizeof(array)/sizeof(array[0]))
20
21static const char *ignorePropNames[]={
22    "FC_NFKC",
23    "NFD_QC",
24    "NFC_QC",
25    "NFKD_QC",
26    "NFKC_QC",
27    "Expands_On_NFD",
28    "Expands_On_NFC",
29    "Expands_On_NFKD",
30    "Expands_On_NFKC",
31    "NFKC_CF"
32};
33
34UnicodeTest::UnicodeTest()
35{
36    UErrorCode errorCode=U_ZERO_ERROR;
37    unknownPropertyNames=new U_NAMESPACE_QUALIFIER Hashtable(errorCode);
38    if(U_FAILURE(errorCode)) {
39        delete unknownPropertyNames;
40        unknownPropertyNames=NULL;
41    }
42    // Ignore some property names altogether.
43    for(int32_t i=0; i<LENGTHOF(ignorePropNames); ++i) {
44        unknownPropertyNames->puti(UnicodeString(ignorePropNames[i], -1, US_INV), 1, errorCode);
45    }
46}
47
48UnicodeTest::~UnicodeTest()
49{
50    delete unknownPropertyNames;
51}
52
53void UnicodeTest::runIndexedTest( int32_t index, UBool exec, const char* &name, char* /*par*/ )
54{
55    if(exec) {
56        logln("TestSuite UnicodeTest: ");
57    }
58    TESTCASE_AUTO_BEGIN;
59    TESTCASE_AUTO(TestAdditionalProperties);
60    TESTCASE_AUTO(TestBinaryValues);
61    TESTCASE_AUTO(TestConsistency);
62    TESTCASE_AUTO(TestPatternProperties);
63    TESTCASE_AUTO(TestScriptMetadata);
64    TESTCASE_AUTO_END;
65}
66
67//====================================================
68// private data used by the tests
69//====================================================
70
71// test DerivedCoreProperties.txt -------------------------------------------
72
73// copied from genprops.c
74static int32_t
75getTokenIndex(const char *const tokens[], int32_t countTokens, const char *s) {
76    const char *t, *z;
77    int32_t i, j;
78
79    s=u_skipWhitespace(s);
80    for(i=0; i<countTokens; ++i) {
81        t=tokens[i];
82        if(t!=NULL) {
83            for(j=0;; ++j) {
84                if(t[j]!=0) {
85                    if(s[j]!=t[j]) {
86                        break;
87                    }
88                } else {
89                    z=u_skipWhitespace(s+j);
90                    if(*z==';' || *z==0) {
91                        return i;
92                    } else {
93                        break;
94                    }
95                }
96            }
97        }
98    }
99    return -1;
100}
101
102static const char *const
103derivedPropsNames[]={
104    "Math",
105    "Alphabetic",
106    "Lowercase",
107    "Uppercase",
108    "ID_Start",
109    "ID_Continue",
110    "XID_Start",
111    "XID_Continue",
112    "Default_Ignorable_Code_Point",
113    "Full_Composition_Exclusion",
114    "Grapheme_Extend",
115    "Grapheme_Link", /* Unicode 5 moves this property here from PropList.txt */
116    "Grapheme_Base",
117    "Cased",
118    "Case_Ignorable",
119    "Changes_When_Lowercased",
120    "Changes_When_Uppercased",
121    "Changes_When_Titlecased",
122    "Changes_When_Casefolded",
123    "Changes_When_Casemapped",
124    "Changes_When_NFKC_Casefolded"
125};
126
127static const UProperty
128derivedPropsIndex[]={
129    UCHAR_MATH,
130    UCHAR_ALPHABETIC,
131    UCHAR_LOWERCASE,
132    UCHAR_UPPERCASE,
133    UCHAR_ID_START,
134    UCHAR_ID_CONTINUE,
135    UCHAR_XID_START,
136    UCHAR_XID_CONTINUE,
137    UCHAR_DEFAULT_IGNORABLE_CODE_POINT,
138    UCHAR_FULL_COMPOSITION_EXCLUSION,
139    UCHAR_GRAPHEME_EXTEND,
140    UCHAR_GRAPHEME_LINK,
141    UCHAR_GRAPHEME_BASE,
142    UCHAR_CASED,
143    UCHAR_CASE_IGNORABLE,
144    UCHAR_CHANGES_WHEN_LOWERCASED,
145    UCHAR_CHANGES_WHEN_UPPERCASED,
146    UCHAR_CHANGES_WHEN_TITLECASED,
147    UCHAR_CHANGES_WHEN_CASEFOLDED,
148    UCHAR_CHANGES_WHEN_CASEMAPPED,
149    UCHAR_CHANGES_WHEN_NFKC_CASEFOLDED
150};
151
152static int32_t numErrors[LENGTHOF(derivedPropsIndex)]={ 0 };
153
154enum { MAX_ERRORS=50 };
155
156U_CFUNC void U_CALLCONV
157derivedPropsLineFn(void *context,
158                   char *fields[][2], int32_t /* fieldCount */,
159                   UErrorCode *pErrorCode)
160{
161    UnicodeTest *me=(UnicodeTest *)context;
162    uint32_t start, end;
163    int32_t i;
164
165    u_parseCodePointRange(fields[0][0], &start, &end, pErrorCode);
166    if(U_FAILURE(*pErrorCode)) {
167        me->errln("UnicodeTest: syntax error in DerivedCoreProperties.txt or DerivedNormalizationProps.txt field 0 at %s\n", fields[0][0]);
168        return;
169    }
170
171    /* parse derived binary property name, ignore unknown names */
172    i=getTokenIndex(derivedPropsNames, LENGTHOF(derivedPropsNames), fields[1][0]);
173    if(i<0) {
174        UnicodeString propName(fields[1][0], (int32_t)(fields[1][1]-fields[1][0]));
175        propName.trim();
176        if(me->unknownPropertyNames->find(propName)==NULL) {
177            UErrorCode errorCode=U_ZERO_ERROR;
178            me->unknownPropertyNames->puti(propName, 1, errorCode);
179            me->errln("UnicodeTest warning: unknown property name '%s' in DerivedCoreProperties.txt or DerivedNormalizationProps.txt\n", fields[1][0]);
180        }
181        return;
182    }
183
184    me->derivedProps[i].add(start, end);
185}
186
187void UnicodeTest::TestAdditionalProperties() {
188#if !UCONFIG_NO_NORMALIZATION
189    // test DerivedCoreProperties.txt and DerivedNormalizationProps.txt
190    if(LENGTHOF(derivedProps)<LENGTHOF(derivedPropsNames)) {
191        errln("error: UnicodeTest::derivedProps[] too short, need at least %d UnicodeSets\n",
192              LENGTHOF(derivedPropsNames));
193        return;
194    }
195    if(LENGTHOF(derivedPropsIndex)!=LENGTHOF(derivedPropsNames)) {
196        errln("error in ucdtest.cpp: LENGTHOF(derivedPropsIndex)!=LENGTHOF(derivedPropsNames)\n");
197        return;
198    }
199
200    char newPath[256];
201    char backupPath[256];
202    char *fields[2][2];
203    UErrorCode errorCode=U_ZERO_ERROR;
204
205    /* Look inside ICU_DATA first */
206    strcpy(newPath, pathToDataDirectory());
207    strcat(newPath, "unidata" U_FILE_SEP_STRING "DerivedCoreProperties.txt");
208
209    // As a fallback, try to guess where the source data was located
210    // at the time ICU was built, and look there.
211#   ifdef U_TOPSRCDIR
212        strcpy(backupPath, U_TOPSRCDIR  U_FILE_SEP_STRING "data");
213#   else
214        strcpy(backupPath, loadTestData(errorCode));
215        strcat(backupPath, U_FILE_SEP_STRING ".." U_FILE_SEP_STRING ".." U_FILE_SEP_STRING ".." U_FILE_SEP_STRING ".." U_FILE_SEP_STRING "data");
216#   endif
217    strcat(backupPath, U_FILE_SEP_STRING);
218    strcat(backupPath, "unidata" U_FILE_SEP_STRING "DerivedCoreProperties.txt");
219
220    char *path=newPath;
221    u_parseDelimitedFile(newPath, ';', fields, 2, derivedPropsLineFn, this, &errorCode);
222
223    if(errorCode==U_FILE_ACCESS_ERROR) {
224        errorCode=U_ZERO_ERROR;
225        path=backupPath;
226        u_parseDelimitedFile(backupPath, ';', fields, 2, derivedPropsLineFn, this, &errorCode);
227    }
228    if(U_FAILURE(errorCode)) {
229        errln("error parsing DerivedCoreProperties.txt: %s\n", u_errorName(errorCode));
230        return;
231    }
232    char *basename=path+strlen(path)-strlen("DerivedCoreProperties.txt");
233    strcpy(basename, "DerivedNormalizationProps.txt");
234    u_parseDelimitedFile(path, ';', fields, 2, derivedPropsLineFn, this, &errorCode);
235    if(U_FAILURE(errorCode)) {
236        errln("error parsing DerivedNormalizationProps.txt: %s\n", u_errorName(errorCode));
237        return;
238    }
239
240    // now we have all derived core properties in the UnicodeSets
241    // run them all through the API
242    int32_t rangeCount, range;
243    uint32_t i;
244    UChar32 start, end;
245
246    // test all TRUE properties
247    for(i=0; i<LENGTHOF(derivedPropsNames); ++i) {
248        rangeCount=derivedProps[i].getRangeCount();
249        for(range=0; range<rangeCount && numErrors[i]<MAX_ERRORS; ++range) {
250            start=derivedProps[i].getRangeStart(range);
251            end=derivedProps[i].getRangeEnd(range);
252            for(; start<=end; ++start) {
253                if(!u_hasBinaryProperty(start, derivedPropsIndex[i])) {
254                    dataerrln("UnicodeTest error: u_hasBinaryProperty(U+%04lx, %s)==FALSE is wrong", start, derivedPropsNames[i]);
255                    if(++numErrors[i]>=MAX_ERRORS) {
256                      dataerrln("Too many errors, moving to the next test");
257                      break;
258                    }
259                }
260            }
261        }
262    }
263
264    // invert all properties
265    for(i=0; i<LENGTHOF(derivedPropsNames); ++i) {
266        derivedProps[i].complement();
267    }
268
269    // test all FALSE properties
270    for(i=0; i<LENGTHOF(derivedPropsNames); ++i) {
271        rangeCount=derivedProps[i].getRangeCount();
272        for(range=0; range<rangeCount && numErrors[i]<MAX_ERRORS; ++range) {
273            start=derivedProps[i].getRangeStart(range);
274            end=derivedProps[i].getRangeEnd(range);
275            for(; start<=end; ++start) {
276                if(u_hasBinaryProperty(start, derivedPropsIndex[i])) {
277                    errln("UnicodeTest error: u_hasBinaryProperty(U+%04lx, %s)==TRUE is wrong\n", start, derivedPropsNames[i]);
278                    if(++numErrors[i]>=MAX_ERRORS) {
279                      errln("Too many errors, moving to the next test");
280                      break;
281                    }
282                }
283            }
284        }
285    }
286#endif /* !UCONFIG_NO_NORMALIZATION */
287}
288
289void UnicodeTest::TestBinaryValues() {
290    /*
291     * Unicode 5.1 explicitly defines binary property value aliases.
292     * Verify that they are all recognized.
293     */
294    UErrorCode errorCode=U_ZERO_ERROR;
295    UnicodeSet alpha(UNICODE_STRING_SIMPLE("[:Alphabetic:]"), errorCode);
296    if(U_FAILURE(errorCode)) {
297        dataerrln("UnicodeSet([:Alphabetic:]) failed - %s", u_errorName(errorCode));
298        return;
299    }
300
301    static const char *const falseValues[]={ "N", "No", "F", "False" };
302    static const char *const trueValues[]={ "Y", "Yes", "T", "True" };
303    int32_t i;
304    for(i=0; i<LENGTHOF(falseValues); ++i) {
305        UnicodeString pattern=UNICODE_STRING_SIMPLE("[:Alphabetic=:]");
306        pattern.insert(pattern.length()-2, UnicodeString(falseValues[i], -1, US_INV));
307        errorCode=U_ZERO_ERROR;
308        UnicodeSet set(pattern, errorCode);
309        if(U_FAILURE(errorCode)) {
310            errln("UnicodeSet([:Alphabetic=%s:]) failed - %s\n", falseValues[i], u_errorName(errorCode));
311            continue;
312        }
313        set.complement();
314        if(set!=alpha) {
315            errln("UnicodeSet([:Alphabetic=%s:]).complement()!=UnicodeSet([:Alphabetic:])\n", falseValues[i]);
316        }
317    }
318    for(i=0; i<LENGTHOF(trueValues); ++i) {
319        UnicodeString pattern=UNICODE_STRING_SIMPLE("[:Alphabetic=:]");
320        pattern.insert(pattern.length()-2, UnicodeString(trueValues[i], -1, US_INV));
321        errorCode=U_ZERO_ERROR;
322        UnicodeSet set(pattern, errorCode);
323        if(U_FAILURE(errorCode)) {
324            errln("UnicodeSet([:Alphabetic=%s:]) failed - %s\n", trueValues[i], u_errorName(errorCode));
325            continue;
326        }
327        if(set!=alpha) {
328            errln("UnicodeSet([:Alphabetic=%s:])!=UnicodeSet([:Alphabetic:])\n", trueValues[i]);
329        }
330    }
331}
332
333void UnicodeTest::TestConsistency() {
334#if !UCONFIG_NO_NORMALIZATION
335    /*
336     * Test for an example that getCanonStartSet() delivers
337     * all characters that compose from the input one,
338     * even in multiple steps.
339     * For example, the set for "I" (0049) should contain both
340     * I-diaeresis (00CF) and I-diaeresis-acute (1E2E).
341     * In general, the set for the middle such character should be a subset
342     * of the set for the first.
343     */
344    IcuTestErrorCode errorCode(*this, "TestConsistency");
345    const Normalizer2 *nfd=Normalizer2::getNFDInstance(errorCode);
346    const Normalizer2Impl *nfcImpl=Normalizer2Factory::getNFCImpl(errorCode);
347    if(!nfcImpl->ensureCanonIterData(errorCode) || errorCode.isFailure()) {
348        dataerrln("Normalizer2::getInstance(NFD) or Normalizer2Factory::getNFCImpl() failed - %s\n",
349                  errorCode.errorName());
350        errorCode.reset();
351        return;
352    }
353
354    UnicodeSet set1, set2;
355    if (nfcImpl->getCanonStartSet(0x49, set1)) {
356        /* enumerate all characters that are plausible to be latin letters */
357        for(UChar start=0xa0; start<0x2000; ++start) {
358            UnicodeString decomp=nfd->normalize(UnicodeString(start), errorCode);
359            if(decomp.length()>1 && decomp[0]==0x49) {
360                set2.add(start);
361            }
362        }
363
364        if (set1!=set2) {
365            errln("[canon start set of 0049] != [all c with canon decomp with 0049]");
366        }
367        // This was available in cucdtst.c but the test had to move to intltest
368        // because the new internal normalization functions are in C++.
369        //compareUSets(set1, set2,
370        //             "[canon start set of 0049]", "[all c with canon decomp with 0049]",
371        //             TRUE);
372    } else {
373        errln("NFC.getCanonStartSet() returned FALSE");
374    }
375#endif
376}
377
378/**
379 * Test various implementations of Pattern_Syntax & Pattern_White_Space.
380 */
381void UnicodeTest::TestPatternProperties() {
382    IcuTestErrorCode errorCode(*this, "TestPatternProperties()");
383    UnicodeSet syn_pp;
384    UnicodeSet syn_prop(UNICODE_STRING_SIMPLE("[:Pattern_Syntax:]"), errorCode);
385    UnicodeSet syn_list(
386        "[!-/\\:-@\\[-\\^`\\{-~"
387        "\\u00A1-\\u00A7\\u00A9\\u00AB\\u00AC\\u00AE\\u00B0\\u00B1\\u00B6\\u00BB\\u00BF\\u00D7\\u00F7"
388        "\\u2010-\\u2027\\u2030-\\u203E\\u2041-\\u2053\\u2055-\\u205E\\u2190-\\u245F\\u2500-\\u2775"
389        "\\u2794-\\u2BFF\\u2E00-\\u2E7F\\u3001-\\u3003\\u3008-\\u3020\\u3030\\uFD3E\\uFD3F\\uFE45\\uFE46]", errorCode);
390    UnicodeSet ws_pp;
391    UnicodeSet ws_prop(UNICODE_STRING_SIMPLE("[:Pattern_White_Space:]"), errorCode);
392    UnicodeSet ws_list(UNICODE_STRING_SIMPLE("[\\u0009-\\u000D\\ \\u0085\\u200E\\u200F\\u2028\\u2029]"), errorCode);
393    UnicodeSet syn_ws_pp;
394    UnicodeSet syn_ws_prop(syn_prop);
395    syn_ws_prop.addAll(ws_prop);
396    for(UChar32 c=0; c<=0xffff; ++c) {
397        if(PatternProps::isSyntax(c)) {
398            syn_pp.add(c);
399        }
400        if(PatternProps::isWhiteSpace(c)) {
401            ws_pp.add(c);
402        }
403        if(PatternProps::isSyntaxOrWhiteSpace(c)) {
404            syn_ws_pp.add(c);
405        }
406    }
407    compareUSets(syn_pp, syn_prop,
408                 "PatternProps.isSyntax()", "[:Pattern_Syntax:]", TRUE);
409    compareUSets(syn_pp, syn_list,
410                 "PatternProps.isSyntax()", "[Pattern_Syntax ranges]", TRUE);
411    compareUSets(ws_pp, ws_prop,
412                 "PatternProps.isWhiteSpace()", "[:Pattern_White_Space:]", TRUE);
413    compareUSets(ws_pp, ws_list,
414                 "PatternProps.isWhiteSpace()", "[Pattern_White_Space ranges]", TRUE);
415    compareUSets(syn_ws_pp, syn_ws_prop,
416                 "PatternProps.isSyntaxOrWhiteSpace()",
417                 "[[:Pattern_Syntax:][:Pattern_White_Space:]]", TRUE);
418}
419
420// So far only minimal port of Java & cucdtst.c compareUSets().
421UBool
422UnicodeTest::compareUSets(const UnicodeSet &a, const UnicodeSet &b,
423                          const char *a_name, const char *b_name,
424                          UBool diffIsError) {
425    UBool same= a==b;
426    if(!same && diffIsError) {
427        errln("Sets are different: %s vs. %s\n", a_name, b_name);
428    }
429    return same;
430}
431
432namespace {
433
434/**
435 * Maps a special script code to the most common script of its encoded characters.
436 */
437UScriptCode getCharScript(UScriptCode script) {
438    switch(script) {
439    case USCRIPT_SIMPLIFIED_HAN:
440    case USCRIPT_TRADITIONAL_HAN:
441        return USCRIPT_HAN;
442    case USCRIPT_JAPANESE:
443        return USCRIPT_HIRAGANA;
444    case USCRIPT_KOREAN:
445        return USCRIPT_HANGUL;
446    default:
447        return script;
448    }
449}
450
451}  // namespace
452
453void UnicodeTest::TestScriptMetadata() {
454    IcuTestErrorCode errorCode(*this, "TestScriptMetadata()");
455    UnicodeSet rtl("[[:bc=R:][:bc=AL:]-[:Cn:]-[:sc=Common:]]", errorCode);
456    // So far, sample characters are uppercase.
457    // Georgian is special.
458    UnicodeSet cased("[[:Lu:]-[:sc=Common:]-[:sc=Geor:]]", errorCode);
459    for(int32_t sci = 0; sci < USCRIPT_CODE_LIMIT; ++sci) {
460        UScriptCode sc = (UScriptCode)sci;
461        // Run the test with -v to see which script has failures:
462        // .../intltest$ make && ./intltest utility/UnicodeTest/TestScriptMetadata -v | grep -C 3 FAIL
463        logln(uscript_getShortName(sc));
464        UScriptUsage usage = uscript_getUsage(sc);
465        UnicodeString sample = uscript_getSampleUnicodeString(sc);
466        UnicodeSet scriptSet;
467        scriptSet.applyIntPropertyValue(UCHAR_SCRIPT, sc, errorCode);
468        if(usage == USCRIPT_USAGE_NOT_ENCODED) {
469            assertTrue("not encoded, no sample", sample.isEmpty());
470            assertFalse("not encoded, not RTL", uscript_isRightToLeft(sc));
471            assertFalse("not encoded, not LB letters", uscript_breaksBetweenLetters(sc));
472            assertFalse("not encoded, not cased", uscript_isCased(sc));
473            assertTrue("not encoded, no characters", scriptSet.isEmpty());
474        } else {
475            assertFalse("encoded, has a sample character", sample.isEmpty());
476            UChar32 firstChar = sample.char32At(0);
477            UScriptCode charScript = getCharScript(sc);
478            assertEquals("script(sample(script))",
479                         charScript, uscript_getScript(firstChar, errorCode));
480            assertEquals("RTL vs. set", rtl.contains(firstChar), uscript_isRightToLeft(sc));
481            assertEquals("cased vs. set", cased.contains(firstChar), uscript_isCased(sc));
482            assertEquals("encoded, has characters", sc == charScript, !scriptSet.isEmpty());
483            if(uscript_isRightToLeft(sc)) {
484                rtl.removeAll(scriptSet);
485            }
486            if(uscript_isCased(sc)) {
487                cased.removeAll(scriptSet);
488            }
489        }
490    }
491    UnicodeString pattern;
492    assertEquals("no remaining RTL characters",
493                 UnicodeString("[]"), rtl.toPattern(pattern));
494    assertEquals("no remaining cased characters",
495                 UnicodeString("[]"), cased.toPattern(pattern));
496
497    assertTrue("Hani breaks between letters", uscript_breaksBetweenLetters(USCRIPT_HAN));
498    assertTrue("Thai breaks between letters", uscript_breaksBetweenLetters(USCRIPT_THAI));
499    assertFalse("Latn does not break between letters", uscript_breaksBetweenLetters(USCRIPT_LATIN));
500}
501