1/********************************************************************
2 * COPYRIGHT:
3 * Copyright (c) 1997-2014, International Business Machines Corporation and
4 * others. All Rights Reserved.
5 ********************************************************************/
6
7#include "unicode/ustring.h"
8#include "unicode/uchar.h"
9#include "unicode/uniset.h"
10#include "unicode/putil.h"
11#include "unicode/uscript.h"
12#include "cstring.h"
13#include "hash.h"
14#include "patternprops.h"
15#include "normalizer2impl.h"
16#include "uparse.h"
17#include "ucdtest.h"
18
19static const char *ignorePropNames[]={
20    "FC_NFKC",
21    "NFD_QC",
22    "NFC_QC",
23    "NFKD_QC",
24    "NFKC_QC",
25    "Expands_On_NFD",
26    "Expands_On_NFC",
27    "Expands_On_NFKD",
28    "Expands_On_NFKC",
29    "NFKC_CF"
30};
31
32UnicodeTest::UnicodeTest()
33{
34    UErrorCode errorCode=U_ZERO_ERROR;
35    unknownPropertyNames=new U_NAMESPACE_QUALIFIER Hashtable(errorCode);
36    if(U_FAILURE(errorCode)) {
37        delete unknownPropertyNames;
38        unknownPropertyNames=NULL;
39    }
40    // Ignore some property names altogether.
41    for(int32_t i=0; i<UPRV_LENGTHOF(ignorePropNames); ++i) {
42        unknownPropertyNames->puti(UnicodeString(ignorePropNames[i], -1, US_INV), 1, errorCode);
43    }
44}
45
46UnicodeTest::~UnicodeTest()
47{
48    delete unknownPropertyNames;
49}
50
51void UnicodeTest::runIndexedTest( int32_t index, UBool exec, const char* &name, char* /*par*/ )
52{
53    if(exec) {
54        logln("TestSuite UnicodeTest: ");
55    }
56    TESTCASE_AUTO_BEGIN;
57    TESTCASE_AUTO(TestAdditionalProperties);
58    TESTCASE_AUTO(TestBinaryValues);
59    TESTCASE_AUTO(TestConsistency);
60    TESTCASE_AUTO(TestPatternProperties);
61    TESTCASE_AUTO(TestScriptMetadata);
62    TESTCASE_AUTO(TestBidiPairedBracketType);
63    TESTCASE_AUTO_END;
64}
65
66//====================================================
67// private data used by the tests
68//====================================================
69
70// test DerivedCoreProperties.txt -------------------------------------------
71
72// copied from genprops.c
73static int32_t
74getTokenIndex(const char *const tokens[], int32_t countTokens, const char *s) {
75    const char *t, *z;
76    int32_t i, j;
77
78    s=u_skipWhitespace(s);
79    for(i=0; i<countTokens; ++i) {
80        t=tokens[i];
81        if(t!=NULL) {
82            for(j=0;; ++j) {
83                if(t[j]!=0) {
84                    if(s[j]!=t[j]) {
85                        break;
86                    }
87                } else {
88                    z=u_skipWhitespace(s+j);
89                    if(*z==';' || *z==0) {
90                        return i;
91                    } else {
92                        break;
93                    }
94                }
95            }
96        }
97    }
98    return -1;
99}
100
101static const char *const
102derivedPropsNames[]={
103    "Math",
104    "Alphabetic",
105    "Lowercase",
106    "Uppercase",
107    "ID_Start",
108    "ID_Continue",
109    "XID_Start",
110    "XID_Continue",
111    "Default_Ignorable_Code_Point",
112    "Full_Composition_Exclusion",
113    "Grapheme_Extend",
114    "Grapheme_Link", /* Unicode 5 moves this property here from PropList.txt */
115    "Grapheme_Base",
116    "Cased",
117    "Case_Ignorable",
118    "Changes_When_Lowercased",
119    "Changes_When_Uppercased",
120    "Changes_When_Titlecased",
121    "Changes_When_Casefolded",
122    "Changes_When_Casemapped",
123    "Changes_When_NFKC_Casefolded"
124};
125
126static const UProperty
127derivedPropsIndex[]={
128    UCHAR_MATH,
129    UCHAR_ALPHABETIC,
130    UCHAR_LOWERCASE,
131    UCHAR_UPPERCASE,
132    UCHAR_ID_START,
133    UCHAR_ID_CONTINUE,
134    UCHAR_XID_START,
135    UCHAR_XID_CONTINUE,
136    UCHAR_DEFAULT_IGNORABLE_CODE_POINT,
137    UCHAR_FULL_COMPOSITION_EXCLUSION,
138    UCHAR_GRAPHEME_EXTEND,
139    UCHAR_GRAPHEME_LINK,
140    UCHAR_GRAPHEME_BASE,
141    UCHAR_CASED,
142    UCHAR_CASE_IGNORABLE,
143    UCHAR_CHANGES_WHEN_LOWERCASED,
144    UCHAR_CHANGES_WHEN_UPPERCASED,
145    UCHAR_CHANGES_WHEN_TITLECASED,
146    UCHAR_CHANGES_WHEN_CASEFOLDED,
147    UCHAR_CHANGES_WHEN_CASEMAPPED,
148    UCHAR_CHANGES_WHEN_NFKC_CASEFOLDED
149};
150
151static int32_t numErrors[UPRV_LENGTHOF(derivedPropsIndex)]={ 0 };
152
153enum { MAX_ERRORS=50 };
154
155U_CFUNC void U_CALLCONV
156derivedPropsLineFn(void *context,
157                   char *fields[][2], int32_t /* fieldCount */,
158                   UErrorCode *pErrorCode)
159{
160    UnicodeTest *me=(UnicodeTest *)context;
161    uint32_t start, end;
162    int32_t i;
163
164    u_parseCodePointRange(fields[0][0], &start, &end, pErrorCode);
165    if(U_FAILURE(*pErrorCode)) {
166        me->errln("UnicodeTest: syntax error in DerivedCoreProperties.txt or DerivedNormalizationProps.txt field 0 at %s\n", fields[0][0]);
167        return;
168    }
169
170    /* parse derived binary property name, ignore unknown names */
171    i=getTokenIndex(derivedPropsNames, UPRV_LENGTHOF(derivedPropsNames), fields[1][0]);
172    if(i<0) {
173        UnicodeString propName(fields[1][0], (int32_t)(fields[1][1]-fields[1][0]));
174        propName.trim();
175        if(me->unknownPropertyNames->find(propName)==NULL) {
176            UErrorCode errorCode=U_ZERO_ERROR;
177            me->unknownPropertyNames->puti(propName, 1, errorCode);
178            me->errln("UnicodeTest warning: unknown property name '%s' in DerivedCoreProperties.txt or DerivedNormalizationProps.txt\n", fields[1][0]);
179        }
180        return;
181    }
182
183    me->derivedProps[i].add(start, end);
184}
185
186void UnicodeTest::TestAdditionalProperties() {
187#if !UCONFIG_NO_NORMALIZATION
188    // test DerivedCoreProperties.txt and DerivedNormalizationProps.txt
189    if(UPRV_LENGTHOF(derivedProps)<UPRV_LENGTHOF(derivedPropsNames)) {
190        errln("error: UnicodeTest::derivedProps[] too short, need at least %d UnicodeSets\n",
191              UPRV_LENGTHOF(derivedPropsNames));
192        return;
193    }
194    if(UPRV_LENGTHOF(derivedPropsIndex)!=UPRV_LENGTHOF(derivedPropsNames)) {
195        errln("error in ucdtest.cpp: UPRV_LENGTHOF(derivedPropsIndex)!=UPRV_LENGTHOF(derivedPropsNames)\n");
196        return;
197    }
198
199    char path[500];
200    if(getUnidataPath(path) == NULL) {
201        errln("unable to find path to source/data/unidata/");
202        return;
203    }
204    char *basename=strchr(path, 0);
205    strcpy(basename, "DerivedCoreProperties.txt");
206
207    char *fields[2][2];
208    UErrorCode errorCode=U_ZERO_ERROR;
209    u_parseDelimitedFile(path, ';', fields, 2, derivedPropsLineFn, this, &errorCode);
210    if(U_FAILURE(errorCode)) {
211        errln("error parsing DerivedCoreProperties.txt: %s\n", u_errorName(errorCode));
212        return;
213    }
214
215    strcpy(basename, "DerivedNormalizationProps.txt");
216    u_parseDelimitedFile(path, ';', fields, 2, derivedPropsLineFn, this, &errorCode);
217    if(U_FAILURE(errorCode)) {
218        errln("error parsing DerivedNormalizationProps.txt: %s\n", u_errorName(errorCode));
219        return;
220    }
221
222    // now we have all derived core properties in the UnicodeSets
223    // run them all through the API
224    int32_t rangeCount, range;
225    uint32_t i;
226    UChar32 start, end;
227
228    // test all TRUE properties
229    for(i=0; i<UPRV_LENGTHOF(derivedPropsNames); ++i) {
230        rangeCount=derivedProps[i].getRangeCount();
231        for(range=0; range<rangeCount && numErrors[i]<MAX_ERRORS; ++range) {
232            start=derivedProps[i].getRangeStart(range);
233            end=derivedProps[i].getRangeEnd(range);
234            for(; start<=end; ++start) {
235                if(!u_hasBinaryProperty(start, derivedPropsIndex[i])) {
236                    dataerrln("UnicodeTest error: u_hasBinaryProperty(U+%04lx, %s)==FALSE is wrong", start, derivedPropsNames[i]);
237                    if(++numErrors[i]>=MAX_ERRORS) {
238                      dataerrln("Too many errors, moving to the next test");
239                      break;
240                    }
241                }
242            }
243        }
244    }
245
246    // invert all properties
247    for(i=0; i<UPRV_LENGTHOF(derivedPropsNames); ++i) {
248        derivedProps[i].complement();
249    }
250
251    // test all FALSE properties
252    for(i=0; i<UPRV_LENGTHOF(derivedPropsNames); ++i) {
253        rangeCount=derivedProps[i].getRangeCount();
254        for(range=0; range<rangeCount && numErrors[i]<MAX_ERRORS; ++range) {
255            start=derivedProps[i].getRangeStart(range);
256            end=derivedProps[i].getRangeEnd(range);
257            for(; start<=end; ++start) {
258                if(u_hasBinaryProperty(start, derivedPropsIndex[i])) {
259                    errln("UnicodeTest error: u_hasBinaryProperty(U+%04lx, %s)==TRUE is wrong\n", start, derivedPropsNames[i]);
260                    if(++numErrors[i]>=MAX_ERRORS) {
261                      errln("Too many errors, moving to the next test");
262                      break;
263                    }
264                }
265            }
266        }
267    }
268#endif /* !UCONFIG_NO_NORMALIZATION */
269}
270
271void UnicodeTest::TestBinaryValues() {
272    /*
273     * Unicode 5.1 explicitly defines binary property value aliases.
274     * Verify that they are all recognized.
275     */
276    UErrorCode errorCode=U_ZERO_ERROR;
277    UnicodeSet alpha(UNICODE_STRING_SIMPLE("[:Alphabetic:]"), errorCode);
278    if(U_FAILURE(errorCode)) {
279        dataerrln("UnicodeSet([:Alphabetic:]) failed - %s", u_errorName(errorCode));
280        return;
281    }
282
283    static const char *const falseValues[]={ "N", "No", "F", "False" };
284    static const char *const trueValues[]={ "Y", "Yes", "T", "True" };
285    int32_t i;
286    for(i=0; i<UPRV_LENGTHOF(falseValues); ++i) {
287        UnicodeString pattern=UNICODE_STRING_SIMPLE("[:Alphabetic=:]");
288        pattern.insert(pattern.length()-2, UnicodeString(falseValues[i], -1, US_INV));
289        errorCode=U_ZERO_ERROR;
290        UnicodeSet set(pattern, errorCode);
291        if(U_FAILURE(errorCode)) {
292            errln("UnicodeSet([:Alphabetic=%s:]) failed - %s\n", falseValues[i], u_errorName(errorCode));
293            continue;
294        }
295        set.complement();
296        if(set!=alpha) {
297            errln("UnicodeSet([:Alphabetic=%s:]).complement()!=UnicodeSet([:Alphabetic:])\n", falseValues[i]);
298        }
299    }
300    for(i=0; i<UPRV_LENGTHOF(trueValues); ++i) {
301        UnicodeString pattern=UNICODE_STRING_SIMPLE("[:Alphabetic=:]");
302        pattern.insert(pattern.length()-2, UnicodeString(trueValues[i], -1, US_INV));
303        errorCode=U_ZERO_ERROR;
304        UnicodeSet set(pattern, errorCode);
305        if(U_FAILURE(errorCode)) {
306            errln("UnicodeSet([:Alphabetic=%s:]) failed - %s\n", trueValues[i], u_errorName(errorCode));
307            continue;
308        }
309        if(set!=alpha) {
310            errln("UnicodeSet([:Alphabetic=%s:])!=UnicodeSet([:Alphabetic:])\n", trueValues[i]);
311        }
312    }
313}
314
315void UnicodeTest::TestConsistency() {
316#if !UCONFIG_NO_NORMALIZATION
317    /*
318     * Test for an example that getCanonStartSet() delivers
319     * all characters that compose from the input one,
320     * even in multiple steps.
321     * For example, the set for "I" (0049) should contain both
322     * I-diaeresis (00CF) and I-diaeresis-acute (1E2E).
323     * In general, the set for the middle such character should be a subset
324     * of the set for the first.
325     */
326    IcuTestErrorCode errorCode(*this, "TestConsistency");
327    const Normalizer2 *nfd=Normalizer2::getNFDInstance(errorCode);
328    const Normalizer2Impl *nfcImpl=Normalizer2Factory::getNFCImpl(errorCode);
329    if(!nfcImpl->ensureCanonIterData(errorCode) || errorCode.isFailure()) {
330        dataerrln("Normalizer2::getInstance(NFD) or Normalizer2Factory::getNFCImpl() failed - %s\n",
331                  errorCode.errorName());
332        errorCode.reset();
333        return;
334    }
335
336    UnicodeSet set1, set2;
337    if (nfcImpl->getCanonStartSet(0x49, set1)) {
338        /* enumerate all characters that are plausible to be latin letters */
339        for(UChar start=0xa0; start<0x2000; ++start) {
340            UnicodeString decomp=nfd->normalize(UnicodeString(start), errorCode);
341            if(decomp.length()>1 && decomp[0]==0x49) {
342                set2.add(start);
343            }
344        }
345
346        if (set1!=set2) {
347            errln("[canon start set of 0049] != [all c with canon decomp with 0049]");
348        }
349        // This was available in cucdtst.c but the test had to move to intltest
350        // because the new internal normalization functions are in C++.
351        //compareUSets(set1, set2,
352        //             "[canon start set of 0049]", "[all c with canon decomp with 0049]",
353        //             TRUE);
354    } else {
355        errln("NFC.getCanonStartSet() returned FALSE");
356    }
357#endif
358}
359
360/**
361 * Test various implementations of Pattern_Syntax & Pattern_White_Space.
362 */
363void UnicodeTest::TestPatternProperties() {
364    IcuTestErrorCode errorCode(*this, "TestPatternProperties()");
365    UnicodeSet syn_pp;
366    UnicodeSet syn_prop(UNICODE_STRING_SIMPLE("[:Pattern_Syntax:]"), errorCode);
367    UnicodeSet syn_list(
368        "[!-/\\:-@\\[-\\^`\\{-~"
369        "\\u00A1-\\u00A7\\u00A9\\u00AB\\u00AC\\u00AE\\u00B0\\u00B1\\u00B6\\u00BB\\u00BF\\u00D7\\u00F7"
370        "\\u2010-\\u2027\\u2030-\\u203E\\u2041-\\u2053\\u2055-\\u205E\\u2190-\\u245F\\u2500-\\u2775"
371        "\\u2794-\\u2BFF\\u2E00-\\u2E7F\\u3001-\\u3003\\u3008-\\u3020\\u3030\\uFD3E\\uFD3F\\uFE45\\uFE46]", errorCode);
372    UnicodeSet ws_pp;
373    UnicodeSet ws_prop(UNICODE_STRING_SIMPLE("[:Pattern_White_Space:]"), errorCode);
374    UnicodeSet ws_list(UNICODE_STRING_SIMPLE("[\\u0009-\\u000D\\ \\u0085\\u200E\\u200F\\u2028\\u2029]"), errorCode);
375    UnicodeSet syn_ws_pp;
376    UnicodeSet syn_ws_prop(syn_prop);
377    syn_ws_prop.addAll(ws_prop);
378    for(UChar32 c=0; c<=0xffff; ++c) {
379        if(PatternProps::isSyntax(c)) {
380            syn_pp.add(c);
381        }
382        if(PatternProps::isWhiteSpace(c)) {
383            ws_pp.add(c);
384        }
385        if(PatternProps::isSyntaxOrWhiteSpace(c)) {
386            syn_ws_pp.add(c);
387        }
388    }
389    compareUSets(syn_pp, syn_prop,
390                 "PatternProps.isSyntax()", "[:Pattern_Syntax:]", TRUE);
391    compareUSets(syn_pp, syn_list,
392                 "PatternProps.isSyntax()", "[Pattern_Syntax ranges]", TRUE);
393    compareUSets(ws_pp, ws_prop,
394                 "PatternProps.isWhiteSpace()", "[:Pattern_White_Space:]", TRUE);
395    compareUSets(ws_pp, ws_list,
396                 "PatternProps.isWhiteSpace()", "[Pattern_White_Space ranges]", TRUE);
397    compareUSets(syn_ws_pp, syn_ws_prop,
398                 "PatternProps.isSyntaxOrWhiteSpace()",
399                 "[[:Pattern_Syntax:][:Pattern_White_Space:]]", TRUE);
400}
401
402// So far only minimal port of Java & cucdtst.c compareUSets().
403UBool
404UnicodeTest::compareUSets(const UnicodeSet &a, const UnicodeSet &b,
405                          const char *a_name, const char *b_name,
406                          UBool diffIsError) {
407    UBool same= a==b;
408    if(!same && diffIsError) {
409        errln("Sets are different: %s vs. %s\n", a_name, b_name);
410    }
411    return same;
412}
413
414namespace {
415
416/**
417 * Maps a special script code to the most common script of its encoded characters.
418 */
419UScriptCode getCharScript(UScriptCode script) {
420    switch(script) {
421    case USCRIPT_SIMPLIFIED_HAN:
422    case USCRIPT_TRADITIONAL_HAN:
423        return USCRIPT_HAN;
424    case USCRIPT_JAPANESE:
425        return USCRIPT_HIRAGANA;
426    case USCRIPT_KOREAN:
427        return USCRIPT_HANGUL;
428    default:
429        return script;
430    }
431}
432
433}  // namespace
434
435void UnicodeTest::TestScriptMetadata() {
436    IcuTestErrorCode errorCode(*this, "TestScriptMetadata()");
437    UnicodeSet rtl("[[:bc=R:][:bc=AL:]-[:Cn:]-[:sc=Common:]]", errorCode);
438    // So far, sample characters are uppercase.
439    // Georgian is special.
440    UnicodeSet cased("[[:Lu:]-[:sc=Common:]-[:sc=Geor:]]", errorCode);
441    for(int32_t sci = 0; sci < USCRIPT_CODE_LIMIT; ++sci) {
442        UScriptCode sc = (UScriptCode)sci;
443        // Run the test with -v to see which script has failures:
444        // .../intltest$ make && ./intltest utility/UnicodeTest/TestScriptMetadata -v | grep -C 3 FAIL
445        logln(uscript_getShortName(sc));
446        UScriptUsage usage = uscript_getUsage(sc);
447        UnicodeString sample = uscript_getSampleUnicodeString(sc);
448        UnicodeSet scriptSet;
449        scriptSet.applyIntPropertyValue(UCHAR_SCRIPT, sc, errorCode);
450        if(usage == USCRIPT_USAGE_NOT_ENCODED) {
451            assertTrue("not encoded, no sample", sample.isEmpty());
452            assertFalse("not encoded, not RTL", uscript_isRightToLeft(sc));
453            assertFalse("not encoded, not LB letters", uscript_breaksBetweenLetters(sc));
454            assertFalse("not encoded, not cased", uscript_isCased(sc));
455            assertTrue("not encoded, no characters", scriptSet.isEmpty());
456        } else {
457            assertFalse("encoded, has a sample character", sample.isEmpty());
458            UChar32 firstChar = sample.char32At(0);
459            UScriptCode charScript = getCharScript(sc);
460            assertEquals("script(sample(script))",
461                         (int32_t)charScript, (int32_t)uscript_getScript(firstChar, errorCode));
462            assertEquals("RTL vs. set", (UBool)rtl.contains(firstChar), (UBool)uscript_isRightToLeft(sc));
463            assertEquals("cased vs. set", (UBool)cased.contains(firstChar), (UBool)uscript_isCased(sc));
464            assertEquals("encoded, has characters", (UBool)(sc == charScript), (UBool)(!scriptSet.isEmpty()));
465            if(uscript_isRightToLeft(sc)) {
466                rtl.removeAll(scriptSet);
467            }
468            if(uscript_isCased(sc)) {
469                cased.removeAll(scriptSet);
470            }
471        }
472    }
473    UnicodeString pattern;
474    assertEquals("no remaining RTL characters",
475                 UnicodeString("[]"), rtl.toPattern(pattern));
476    assertEquals("no remaining cased characters",
477                 UnicodeString("[]"), cased.toPattern(pattern));
478
479    assertTrue("Hani breaks between letters", uscript_breaksBetweenLetters(USCRIPT_HAN));
480    assertTrue("Thai breaks between letters", uscript_breaksBetweenLetters(USCRIPT_THAI));
481    assertFalse("Latn does not break between letters", uscript_breaksBetweenLetters(USCRIPT_LATIN));
482}
483
484void UnicodeTest::TestBidiPairedBracketType() {
485    // BidiBrackets-6.3.0.txt says:
486    //
487    // The set of code points listed in this file was originally derived
488    // using the character properties General_Category (gc), Bidi_Class (bc),
489    // Bidi_Mirrored (Bidi_M), and Bidi_Mirroring_Glyph (bmg), as follows:
490    // two characters, A and B, form a pair if A has gc=Ps and B has gc=Pe,
491    // both have bc=ON and Bidi_M=Y, and bmg of A is B. Bidi_Paired_Bracket
492    // maps A to B and vice versa, and their Bidi_Paired_Bracket_Type
493    // property values are Open and Close, respectively.
494    IcuTestErrorCode errorCode(*this, "TestBidiPairedBracketType()");
495    UnicodeSet bpt("[:^bpt=n:]", errorCode);
496    assertTrue("bpt!=None is not empty", !bpt.isEmpty());
497    // The following should always be true.
498    UnicodeSet mirrored("[:Bidi_M:]", errorCode);
499    UnicodeSet other_neutral("[:bc=ON:]", errorCode);
500    assertTrue("bpt!=None is a subset of Bidi_M", mirrored.containsAll(bpt));
501    assertTrue("bpt!=None is a subset of bc=ON", other_neutral.containsAll(bpt));
502    // The following are true at least initially in Unicode 6.3.
503    UnicodeSet bpt_open("[:bpt=o:]", errorCode);
504    UnicodeSet bpt_close("[:bpt=c:]", errorCode);
505    UnicodeSet ps("[:Ps:]", errorCode);
506    UnicodeSet pe("[:Pe:]", errorCode);
507    assertTrue("bpt=Open is a subset of Ps", ps.containsAll(bpt_open));
508    assertTrue("bpt=Close is a subset of Pe", pe.containsAll(bpt_close));
509}
510