1/********************************************************************
2 * COPYRIGHT:
3 * Copyright (c) 1997-2013, International Business Machines Corporation and
4 * others. All Rights Reserved.
5 ********************************************************************/
6
7#include "unicode/ustring.h"
8#include "unicode/uchar.h"
9#include "unicode/uniset.h"
10#include "unicode/putil.h"
11#include "unicode/uscript.h"
12#include "cstring.h"
13#include "hash.h"
14#include "patternprops.h"
15#include "normalizer2impl.h"
16#include "uparse.h"
17#include "ucdtest.h"
18
19#define LENGTHOF(array) (int32_t)(sizeof(array)/sizeof(array[0]))
20
21static const char *ignorePropNames[]={
22    "FC_NFKC",
23    "NFD_QC",
24    "NFC_QC",
25    "NFKD_QC",
26    "NFKC_QC",
27    "Expands_On_NFD",
28    "Expands_On_NFC",
29    "Expands_On_NFKD",
30    "Expands_On_NFKC",
31    "NFKC_CF"
32};
33
34UnicodeTest::UnicodeTest()
35{
36    UErrorCode errorCode=U_ZERO_ERROR;
37    unknownPropertyNames=new U_NAMESPACE_QUALIFIER Hashtable(errorCode);
38    if(U_FAILURE(errorCode)) {
39        delete unknownPropertyNames;
40        unknownPropertyNames=NULL;
41    }
42    // Ignore some property names altogether.
43    for(int32_t i=0; i<LENGTHOF(ignorePropNames); ++i) {
44        unknownPropertyNames->puti(UnicodeString(ignorePropNames[i], -1, US_INV), 1, errorCode);
45    }
46}
47
48UnicodeTest::~UnicodeTest()
49{
50    delete unknownPropertyNames;
51}
52
53void UnicodeTest::runIndexedTest( int32_t index, UBool exec, const char* &name, char* /*par*/ )
54{
55    if(exec) {
56        logln("TestSuite UnicodeTest: ");
57    }
58    TESTCASE_AUTO_BEGIN;
59    TESTCASE_AUTO(TestAdditionalProperties);
60    TESTCASE_AUTO(TestBinaryValues);
61    TESTCASE_AUTO(TestConsistency);
62    TESTCASE_AUTO(TestPatternProperties);
63    TESTCASE_AUTO(TestScriptMetadata);
64    TESTCASE_AUTO(TestBidiPairedBracketType);
65    TESTCASE_AUTO_END;
66}
67
68//====================================================
69// private data used by the tests
70//====================================================
71
72// test DerivedCoreProperties.txt -------------------------------------------
73
74// copied from genprops.c
75static int32_t
76getTokenIndex(const char *const tokens[], int32_t countTokens, const char *s) {
77    const char *t, *z;
78    int32_t i, j;
79
80    s=u_skipWhitespace(s);
81    for(i=0; i<countTokens; ++i) {
82        t=tokens[i];
83        if(t!=NULL) {
84            for(j=0;; ++j) {
85                if(t[j]!=0) {
86                    if(s[j]!=t[j]) {
87                        break;
88                    }
89                } else {
90                    z=u_skipWhitespace(s+j);
91                    if(*z==';' || *z==0) {
92                        return i;
93                    } else {
94                        break;
95                    }
96                }
97            }
98        }
99    }
100    return -1;
101}
102
103static const char *const
104derivedPropsNames[]={
105    "Math",
106    "Alphabetic",
107    "Lowercase",
108    "Uppercase",
109    "ID_Start",
110    "ID_Continue",
111    "XID_Start",
112    "XID_Continue",
113    "Default_Ignorable_Code_Point",
114    "Full_Composition_Exclusion",
115    "Grapheme_Extend",
116    "Grapheme_Link", /* Unicode 5 moves this property here from PropList.txt */
117    "Grapheme_Base",
118    "Cased",
119    "Case_Ignorable",
120    "Changes_When_Lowercased",
121    "Changes_When_Uppercased",
122    "Changes_When_Titlecased",
123    "Changes_When_Casefolded",
124    "Changes_When_Casemapped",
125    "Changes_When_NFKC_Casefolded"
126};
127
128static const UProperty
129derivedPropsIndex[]={
130    UCHAR_MATH,
131    UCHAR_ALPHABETIC,
132    UCHAR_LOWERCASE,
133    UCHAR_UPPERCASE,
134    UCHAR_ID_START,
135    UCHAR_ID_CONTINUE,
136    UCHAR_XID_START,
137    UCHAR_XID_CONTINUE,
138    UCHAR_DEFAULT_IGNORABLE_CODE_POINT,
139    UCHAR_FULL_COMPOSITION_EXCLUSION,
140    UCHAR_GRAPHEME_EXTEND,
141    UCHAR_GRAPHEME_LINK,
142    UCHAR_GRAPHEME_BASE,
143    UCHAR_CASED,
144    UCHAR_CASE_IGNORABLE,
145    UCHAR_CHANGES_WHEN_LOWERCASED,
146    UCHAR_CHANGES_WHEN_UPPERCASED,
147    UCHAR_CHANGES_WHEN_TITLECASED,
148    UCHAR_CHANGES_WHEN_CASEFOLDED,
149    UCHAR_CHANGES_WHEN_CASEMAPPED,
150    UCHAR_CHANGES_WHEN_NFKC_CASEFOLDED
151};
152
153static int32_t numErrors[LENGTHOF(derivedPropsIndex)]={ 0 };
154
155enum { MAX_ERRORS=50 };
156
157U_CFUNC void U_CALLCONV
158derivedPropsLineFn(void *context,
159                   char *fields[][2], int32_t /* fieldCount */,
160                   UErrorCode *pErrorCode)
161{
162    UnicodeTest *me=(UnicodeTest *)context;
163    uint32_t start, end;
164    int32_t i;
165
166    u_parseCodePointRange(fields[0][0], &start, &end, pErrorCode);
167    if(U_FAILURE(*pErrorCode)) {
168        me->errln("UnicodeTest: syntax error in DerivedCoreProperties.txt or DerivedNormalizationProps.txt field 0 at %s\n", fields[0][0]);
169        return;
170    }
171
172    /* parse derived binary property name, ignore unknown names */
173    i=getTokenIndex(derivedPropsNames, LENGTHOF(derivedPropsNames), fields[1][0]);
174    if(i<0) {
175        UnicodeString propName(fields[1][0], (int32_t)(fields[1][1]-fields[1][0]));
176        propName.trim();
177        if(me->unknownPropertyNames->find(propName)==NULL) {
178            UErrorCode errorCode=U_ZERO_ERROR;
179            me->unknownPropertyNames->puti(propName, 1, errorCode);
180            me->errln("UnicodeTest warning: unknown property name '%s' in DerivedCoreProperties.txt or DerivedNormalizationProps.txt\n", fields[1][0]);
181        }
182        return;
183    }
184
185    me->derivedProps[i].add(start, end);
186}
187
188void UnicodeTest::TestAdditionalProperties() {
189#if !UCONFIG_NO_NORMALIZATION
190    // test DerivedCoreProperties.txt and DerivedNormalizationProps.txt
191    if(LENGTHOF(derivedProps)<LENGTHOF(derivedPropsNames)) {
192        errln("error: UnicodeTest::derivedProps[] too short, need at least %d UnicodeSets\n",
193              LENGTHOF(derivedPropsNames));
194        return;
195    }
196    if(LENGTHOF(derivedPropsIndex)!=LENGTHOF(derivedPropsNames)) {
197        errln("error in ucdtest.cpp: LENGTHOF(derivedPropsIndex)!=LENGTHOF(derivedPropsNames)\n");
198        return;
199    }
200
201    char newPath[256];
202    char backupPath[256];
203    char *fields[2][2];
204    UErrorCode errorCode=U_ZERO_ERROR;
205
206    /* Look inside ICU_DATA first */
207    strcpy(newPath, pathToDataDirectory());
208    strcat(newPath, "unidata" U_FILE_SEP_STRING "DerivedCoreProperties.txt");
209
210    // As a fallback, try to guess where the source data was located
211    // at the time ICU was built, and look there.
212#   ifdef U_TOPSRCDIR
213        strcpy(backupPath, U_TOPSRCDIR  U_FILE_SEP_STRING "data");
214#   else
215        strcpy(backupPath, loadTestData(errorCode));
216        strcat(backupPath, U_FILE_SEP_STRING ".." U_FILE_SEP_STRING ".." U_FILE_SEP_STRING ".." U_FILE_SEP_STRING ".." U_FILE_SEP_STRING "data");
217#   endif
218    strcat(backupPath, U_FILE_SEP_STRING);
219    strcat(backupPath, "unidata" U_FILE_SEP_STRING "DerivedCoreProperties.txt");
220
221    char *path=newPath;
222    u_parseDelimitedFile(newPath, ';', fields, 2, derivedPropsLineFn, this, &errorCode);
223
224    if(errorCode==U_FILE_ACCESS_ERROR) {
225        errorCode=U_ZERO_ERROR;
226        path=backupPath;
227        u_parseDelimitedFile(backupPath, ';', fields, 2, derivedPropsLineFn, this, &errorCode);
228    }
229    if(U_FAILURE(errorCode)) {
230        errln("error parsing DerivedCoreProperties.txt: %s\n", u_errorName(errorCode));
231        return;
232    }
233    char *basename=path+strlen(path)-strlen("DerivedCoreProperties.txt");
234    strcpy(basename, "DerivedNormalizationProps.txt");
235    u_parseDelimitedFile(path, ';', fields, 2, derivedPropsLineFn, this, &errorCode);
236    if(U_FAILURE(errorCode)) {
237        errln("error parsing DerivedNormalizationProps.txt: %s\n", u_errorName(errorCode));
238        return;
239    }
240
241    // now we have all derived core properties in the UnicodeSets
242    // run them all through the API
243    int32_t rangeCount, range;
244    uint32_t i;
245    UChar32 start, end;
246
247    // test all TRUE properties
248    for(i=0; i<LENGTHOF(derivedPropsNames); ++i) {
249        rangeCount=derivedProps[i].getRangeCount();
250        for(range=0; range<rangeCount && numErrors[i]<MAX_ERRORS; ++range) {
251            start=derivedProps[i].getRangeStart(range);
252            end=derivedProps[i].getRangeEnd(range);
253            for(; start<=end; ++start) {
254                if(!u_hasBinaryProperty(start, derivedPropsIndex[i])) {
255                    dataerrln("UnicodeTest error: u_hasBinaryProperty(U+%04lx, %s)==FALSE is wrong", start, derivedPropsNames[i]);
256                    if(++numErrors[i]>=MAX_ERRORS) {
257                      dataerrln("Too many errors, moving to the next test");
258                      break;
259                    }
260                }
261            }
262        }
263    }
264
265    // invert all properties
266    for(i=0; i<LENGTHOF(derivedPropsNames); ++i) {
267        derivedProps[i].complement();
268    }
269
270    // test all FALSE properties
271    for(i=0; i<LENGTHOF(derivedPropsNames); ++i) {
272        rangeCount=derivedProps[i].getRangeCount();
273        for(range=0; range<rangeCount && numErrors[i]<MAX_ERRORS; ++range) {
274            start=derivedProps[i].getRangeStart(range);
275            end=derivedProps[i].getRangeEnd(range);
276            for(; start<=end; ++start) {
277                if(u_hasBinaryProperty(start, derivedPropsIndex[i])) {
278                    errln("UnicodeTest error: u_hasBinaryProperty(U+%04lx, %s)==TRUE is wrong\n", start, derivedPropsNames[i]);
279                    if(++numErrors[i]>=MAX_ERRORS) {
280                      errln("Too many errors, moving to the next test");
281                      break;
282                    }
283                }
284            }
285        }
286    }
287#endif /* !UCONFIG_NO_NORMALIZATION */
288}
289
290void UnicodeTest::TestBinaryValues() {
291    /*
292     * Unicode 5.1 explicitly defines binary property value aliases.
293     * Verify that they are all recognized.
294     */
295    UErrorCode errorCode=U_ZERO_ERROR;
296    UnicodeSet alpha(UNICODE_STRING_SIMPLE("[:Alphabetic:]"), errorCode);
297    if(U_FAILURE(errorCode)) {
298        dataerrln("UnicodeSet([:Alphabetic:]) failed - %s", u_errorName(errorCode));
299        return;
300    }
301
302    static const char *const falseValues[]={ "N", "No", "F", "False" };
303    static const char *const trueValues[]={ "Y", "Yes", "T", "True" };
304    int32_t i;
305    for(i=0; i<LENGTHOF(falseValues); ++i) {
306        UnicodeString pattern=UNICODE_STRING_SIMPLE("[:Alphabetic=:]");
307        pattern.insert(pattern.length()-2, UnicodeString(falseValues[i], -1, US_INV));
308        errorCode=U_ZERO_ERROR;
309        UnicodeSet set(pattern, errorCode);
310        if(U_FAILURE(errorCode)) {
311            errln("UnicodeSet([:Alphabetic=%s:]) failed - %s\n", falseValues[i], u_errorName(errorCode));
312            continue;
313        }
314        set.complement();
315        if(set!=alpha) {
316            errln("UnicodeSet([:Alphabetic=%s:]).complement()!=UnicodeSet([:Alphabetic:])\n", falseValues[i]);
317        }
318    }
319    for(i=0; i<LENGTHOF(trueValues); ++i) {
320        UnicodeString pattern=UNICODE_STRING_SIMPLE("[:Alphabetic=:]");
321        pattern.insert(pattern.length()-2, UnicodeString(trueValues[i], -1, US_INV));
322        errorCode=U_ZERO_ERROR;
323        UnicodeSet set(pattern, errorCode);
324        if(U_FAILURE(errorCode)) {
325            errln("UnicodeSet([:Alphabetic=%s:]) failed - %s\n", trueValues[i], u_errorName(errorCode));
326            continue;
327        }
328        if(set!=alpha) {
329            errln("UnicodeSet([:Alphabetic=%s:])!=UnicodeSet([:Alphabetic:])\n", trueValues[i]);
330        }
331    }
332}
333
334void UnicodeTest::TestConsistency() {
335#if !UCONFIG_NO_NORMALIZATION
336    /*
337     * Test for an example that getCanonStartSet() delivers
338     * all characters that compose from the input one,
339     * even in multiple steps.
340     * For example, the set for "I" (0049) should contain both
341     * I-diaeresis (00CF) and I-diaeresis-acute (1E2E).
342     * In general, the set for the middle such character should be a subset
343     * of the set for the first.
344     */
345    IcuTestErrorCode errorCode(*this, "TestConsistency");
346    const Normalizer2 *nfd=Normalizer2::getNFDInstance(errorCode);
347    const Normalizer2Impl *nfcImpl=Normalizer2Factory::getNFCImpl(errorCode);
348    if(!nfcImpl->ensureCanonIterData(errorCode) || errorCode.isFailure()) {
349        dataerrln("Normalizer2::getInstance(NFD) or Normalizer2Factory::getNFCImpl() failed - %s\n",
350                  errorCode.errorName());
351        errorCode.reset();
352        return;
353    }
354
355    UnicodeSet set1, set2;
356    if (nfcImpl->getCanonStartSet(0x49, set1)) {
357        /* enumerate all characters that are plausible to be latin letters */
358        for(UChar start=0xa0; start<0x2000; ++start) {
359            UnicodeString decomp=nfd->normalize(UnicodeString(start), errorCode);
360            if(decomp.length()>1 && decomp[0]==0x49) {
361                set2.add(start);
362            }
363        }
364
365        if (set1!=set2) {
366            errln("[canon start set of 0049] != [all c with canon decomp with 0049]");
367        }
368        // This was available in cucdtst.c but the test had to move to intltest
369        // because the new internal normalization functions are in C++.
370        //compareUSets(set1, set2,
371        //             "[canon start set of 0049]", "[all c with canon decomp with 0049]",
372        //             TRUE);
373    } else {
374        errln("NFC.getCanonStartSet() returned FALSE");
375    }
376#endif
377}
378
379/**
380 * Test various implementations of Pattern_Syntax & Pattern_White_Space.
381 */
382void UnicodeTest::TestPatternProperties() {
383    IcuTestErrorCode errorCode(*this, "TestPatternProperties()");
384    UnicodeSet syn_pp;
385    UnicodeSet syn_prop(UNICODE_STRING_SIMPLE("[:Pattern_Syntax:]"), errorCode);
386    UnicodeSet syn_list(
387        "[!-/\\:-@\\[-\\^`\\{-~"
388        "\\u00A1-\\u00A7\\u00A9\\u00AB\\u00AC\\u00AE\\u00B0\\u00B1\\u00B6\\u00BB\\u00BF\\u00D7\\u00F7"
389        "\\u2010-\\u2027\\u2030-\\u203E\\u2041-\\u2053\\u2055-\\u205E\\u2190-\\u245F\\u2500-\\u2775"
390        "\\u2794-\\u2BFF\\u2E00-\\u2E7F\\u3001-\\u3003\\u3008-\\u3020\\u3030\\uFD3E\\uFD3F\\uFE45\\uFE46]", errorCode);
391    UnicodeSet ws_pp;
392    UnicodeSet ws_prop(UNICODE_STRING_SIMPLE("[:Pattern_White_Space:]"), errorCode);
393    UnicodeSet ws_list(UNICODE_STRING_SIMPLE("[\\u0009-\\u000D\\ \\u0085\\u200E\\u200F\\u2028\\u2029]"), errorCode);
394    UnicodeSet syn_ws_pp;
395    UnicodeSet syn_ws_prop(syn_prop);
396    syn_ws_prop.addAll(ws_prop);
397    for(UChar32 c=0; c<=0xffff; ++c) {
398        if(PatternProps::isSyntax(c)) {
399            syn_pp.add(c);
400        }
401        if(PatternProps::isWhiteSpace(c)) {
402            ws_pp.add(c);
403        }
404        if(PatternProps::isSyntaxOrWhiteSpace(c)) {
405            syn_ws_pp.add(c);
406        }
407    }
408    compareUSets(syn_pp, syn_prop,
409                 "PatternProps.isSyntax()", "[:Pattern_Syntax:]", TRUE);
410    compareUSets(syn_pp, syn_list,
411                 "PatternProps.isSyntax()", "[Pattern_Syntax ranges]", TRUE);
412    compareUSets(ws_pp, ws_prop,
413                 "PatternProps.isWhiteSpace()", "[:Pattern_White_Space:]", TRUE);
414    compareUSets(ws_pp, ws_list,
415                 "PatternProps.isWhiteSpace()", "[Pattern_White_Space ranges]", TRUE);
416    compareUSets(syn_ws_pp, syn_ws_prop,
417                 "PatternProps.isSyntaxOrWhiteSpace()",
418                 "[[:Pattern_Syntax:][:Pattern_White_Space:]]", TRUE);
419}
420
421// So far only minimal port of Java & cucdtst.c compareUSets().
422UBool
423UnicodeTest::compareUSets(const UnicodeSet &a, const UnicodeSet &b,
424                          const char *a_name, const char *b_name,
425                          UBool diffIsError) {
426    UBool same= a==b;
427    if(!same && diffIsError) {
428        errln("Sets are different: %s vs. %s\n", a_name, b_name);
429    }
430    return same;
431}
432
433namespace {
434
435/**
436 * Maps a special script code to the most common script of its encoded characters.
437 */
438UScriptCode getCharScript(UScriptCode script) {
439    switch(script) {
440    case USCRIPT_SIMPLIFIED_HAN:
441    case USCRIPT_TRADITIONAL_HAN:
442        return USCRIPT_HAN;
443    case USCRIPT_JAPANESE:
444        return USCRIPT_HIRAGANA;
445    case USCRIPT_KOREAN:
446        return USCRIPT_HANGUL;
447    default:
448        return script;
449    }
450}
451
452}  // namespace
453
454void UnicodeTest::TestScriptMetadata() {
455    IcuTestErrorCode errorCode(*this, "TestScriptMetadata()");
456    UnicodeSet rtl("[[:bc=R:][:bc=AL:]-[:Cn:]-[:sc=Common:]]", errorCode);
457    // So far, sample characters are uppercase.
458    // Georgian is special.
459    UnicodeSet cased("[[:Lu:]-[:sc=Common:]-[:sc=Geor:]]", errorCode);
460    for(int32_t sci = 0; sci < USCRIPT_CODE_LIMIT; ++sci) {
461        UScriptCode sc = (UScriptCode)sci;
462        // Run the test with -v to see which script has failures:
463        // .../intltest$ make && ./intltest utility/UnicodeTest/TestScriptMetadata -v | grep -C 3 FAIL
464        logln(uscript_getShortName(sc));
465        UScriptUsage usage = uscript_getUsage(sc);
466        UnicodeString sample = uscript_getSampleUnicodeString(sc);
467        UnicodeSet scriptSet;
468        scriptSet.applyIntPropertyValue(UCHAR_SCRIPT, sc, errorCode);
469        if(usage == USCRIPT_USAGE_NOT_ENCODED) {
470            assertTrue("not encoded, no sample", sample.isEmpty());
471            assertFalse("not encoded, not RTL", uscript_isRightToLeft(sc));
472            assertFalse("not encoded, not LB letters", uscript_breaksBetweenLetters(sc));
473            assertFalse("not encoded, not cased", uscript_isCased(sc));
474            assertTrue("not encoded, no characters", scriptSet.isEmpty());
475        } else {
476            assertFalse("encoded, has a sample character", sample.isEmpty());
477            UChar32 firstChar = sample.char32At(0);
478            UScriptCode charScript = getCharScript(sc);
479            assertEquals("script(sample(script))",
480                         (int32_t)charScript, (int32_t)uscript_getScript(firstChar, errorCode));
481            assertEquals("RTL vs. set", (UBool)rtl.contains(firstChar), (UBool)uscript_isRightToLeft(sc));
482            assertEquals("cased vs. set", (UBool)cased.contains(firstChar), (UBool)uscript_isCased(sc));
483            assertEquals("encoded, has characters", (UBool)(sc == charScript), (UBool)(!scriptSet.isEmpty()));
484            if(uscript_isRightToLeft(sc)) {
485                rtl.removeAll(scriptSet);
486            }
487            if(uscript_isCased(sc)) {
488                cased.removeAll(scriptSet);
489            }
490        }
491    }
492    UnicodeString pattern;
493    assertEquals("no remaining RTL characters",
494                 UnicodeString("[]"), rtl.toPattern(pattern));
495    assertEquals("no remaining cased characters",
496                 UnicodeString("[]"), cased.toPattern(pattern));
497
498    assertTrue("Hani breaks between letters", uscript_breaksBetweenLetters(USCRIPT_HAN));
499    assertTrue("Thai breaks between letters", uscript_breaksBetweenLetters(USCRIPT_THAI));
500    assertFalse("Latn does not break between letters", uscript_breaksBetweenLetters(USCRIPT_LATIN));
501}
502
503void UnicodeTest::TestBidiPairedBracketType() {
504    // BidiBrackets-6.3.0.txt says:
505    //
506    // The set of code points listed in this file was originally derived
507    // using the character properties General_Category (gc), Bidi_Class (bc),
508    // Bidi_Mirrored (Bidi_M), and Bidi_Mirroring_Glyph (bmg), as follows:
509    // two characters, A and B, form a pair if A has gc=Ps and B has gc=Pe,
510    // both have bc=ON and Bidi_M=Y, and bmg of A is B. Bidi_Paired_Bracket
511    // maps A to B and vice versa, and their Bidi_Paired_Bracket_Type
512    // property values are Open and Close, respectively.
513    IcuTestErrorCode errorCode(*this, "TestBidiPairedBracketType()");
514    UnicodeSet bpt("[:^bpt=n:]", errorCode);
515    assertTrue("bpt!=None is not empty", !bpt.isEmpty());
516    // The following should always be true.
517    UnicodeSet mirrored("[:Bidi_M:]", errorCode);
518    UnicodeSet other_neutral("[:bc=ON:]", errorCode);
519    assertTrue("bpt!=None is a subset of Bidi_M", mirrored.containsAll(bpt));
520    assertTrue("bpt!=None is a subset of bc=ON", other_neutral.containsAll(bpt));
521    // The following are true at least initially in Unicode 6.3.
522    UnicodeSet bpt_open("[:bpt=o:]", errorCode);
523    UnicodeSet bpt_close("[:bpt=c:]", errorCode);
524    UnicodeSet ps("[:Ps:]", errorCode);
525    UnicodeSet pe("[:Pe:]", errorCode);
526    assertTrue("bpt=Open is a subset of Ps", ps.containsAll(bpt_open));
527    assertTrue("bpt=Close is a subset of Pe", pe.containsAll(bpt_close));
528}
529