1/********************************************************************
2 * COPYRIGHT:
3 * Copyright (c) 1997-2011, International Business Machines Corporation and
4 * others. All Rights Reserved.
5 ********************************************************************/
6
7#include "unicode/ustring.h"
8#include "unicode/uchar.h"
9#include "unicode/uniset.h"
10#include "unicode/putil.h"
11#include "cstring.h"
12#include "hash.h"
13#include "patternprops.h"
14#include "normalizer2impl.h"
15#include "uparse.h"
16#include "ucdtest.h"
17
18#define LENGTHOF(array) (int32_t)(sizeof(array)/sizeof(array[0]))
19
20static const char *ignorePropNames[]={
21    "FC_NFKC",
22    "NFD_QC",
23    "NFC_QC",
24    "NFKD_QC",
25    "NFKC_QC",
26    "Expands_On_NFD",
27    "Expands_On_NFC",
28    "Expands_On_NFKD",
29    "Expands_On_NFKC",
30    "NFKC_CF"
31};
32
33UnicodeTest::UnicodeTest()
34{
35    UErrorCode errorCode=U_ZERO_ERROR;
36    unknownPropertyNames=new U_NAMESPACE_QUALIFIER Hashtable(errorCode);
37    if(U_FAILURE(errorCode)) {
38        delete unknownPropertyNames;
39        unknownPropertyNames=NULL;
40    }
41    // Ignore some property names altogether.
42    for(int32_t i=0; i<LENGTHOF(ignorePropNames); ++i) {
43        unknownPropertyNames->puti(UnicodeString(ignorePropNames[i], -1, US_INV), 1, errorCode);
44    }
45}
46
47UnicodeTest::~UnicodeTest()
48{
49    delete unknownPropertyNames;
50}
51
52void UnicodeTest::runIndexedTest( int32_t index, UBool exec, const char* &name, char* /*par*/ )
53{
54    if(exec) {
55        logln("TestSuite UnicodeTest: ");
56    }
57    TESTCASE_AUTO_BEGIN;
58    TESTCASE_AUTO(TestAdditionalProperties);
59    TESTCASE_AUTO(TestBinaryValues);
60    TESTCASE_AUTO(TestConsistency);
61    TESTCASE_AUTO(TestPatternProperties);
62    TESTCASE_AUTO_END;
63}
64
65//====================================================
66// private data used by the tests
67//====================================================
68
69// test DerivedCoreProperties.txt -------------------------------------------
70
71// copied from genprops.c
72static int32_t
73getTokenIndex(const char *const tokens[], int32_t countTokens, const char *s) {
74    const char *t, *z;
75    int32_t i, j;
76
77    s=u_skipWhitespace(s);
78    for(i=0; i<countTokens; ++i) {
79        t=tokens[i];
80        if(t!=NULL) {
81            for(j=0;; ++j) {
82                if(t[j]!=0) {
83                    if(s[j]!=t[j]) {
84                        break;
85                    }
86                } else {
87                    z=u_skipWhitespace(s+j);
88                    if(*z==';' || *z==0) {
89                        return i;
90                    } else {
91                        break;
92                    }
93                }
94            }
95        }
96    }
97    return -1;
98}
99
100static const char *const
101derivedPropsNames[]={
102    "Math",
103    "Alphabetic",
104    "Lowercase",
105    "Uppercase",
106    "ID_Start",
107    "ID_Continue",
108    "XID_Start",
109    "XID_Continue",
110    "Default_Ignorable_Code_Point",
111    "Full_Composition_Exclusion",
112    "Grapheme_Extend",
113    "Grapheme_Link", /* Unicode 5 moves this property here from PropList.txt */
114    "Grapheme_Base",
115    "Cased",
116    "Case_Ignorable",
117    "Changes_When_Lowercased",
118    "Changes_When_Uppercased",
119    "Changes_When_Titlecased",
120    "Changes_When_Casefolded",
121    "Changes_When_Casemapped",
122    "Changes_When_NFKC_Casefolded"
123};
124
125static const UProperty
126derivedPropsIndex[]={
127    UCHAR_MATH,
128    UCHAR_ALPHABETIC,
129    UCHAR_LOWERCASE,
130    UCHAR_UPPERCASE,
131    UCHAR_ID_START,
132    UCHAR_ID_CONTINUE,
133    UCHAR_XID_START,
134    UCHAR_XID_CONTINUE,
135    UCHAR_DEFAULT_IGNORABLE_CODE_POINT,
136    UCHAR_FULL_COMPOSITION_EXCLUSION,
137    UCHAR_GRAPHEME_EXTEND,
138    UCHAR_GRAPHEME_LINK,
139    UCHAR_GRAPHEME_BASE,
140    UCHAR_CASED,
141    UCHAR_CASE_IGNORABLE,
142    UCHAR_CHANGES_WHEN_LOWERCASED,
143    UCHAR_CHANGES_WHEN_UPPERCASED,
144    UCHAR_CHANGES_WHEN_TITLECASED,
145    UCHAR_CHANGES_WHEN_CASEFOLDED,
146    UCHAR_CHANGES_WHEN_CASEMAPPED,
147    UCHAR_CHANGES_WHEN_NFKC_CASEFOLDED
148};
149
150static int32_t numErrors[LENGTHOF(derivedPropsIndex)]={ 0 };
151
152enum { MAX_ERRORS=50 };
153
154U_CFUNC void U_CALLCONV
155derivedPropsLineFn(void *context,
156                   char *fields[][2], int32_t /* fieldCount */,
157                   UErrorCode *pErrorCode)
158{
159    UnicodeTest *me=(UnicodeTest *)context;
160    uint32_t start, end;
161    int32_t i;
162
163    u_parseCodePointRange(fields[0][0], &start, &end, pErrorCode);
164    if(U_FAILURE(*pErrorCode)) {
165        me->errln("UnicodeTest: syntax error in DerivedCoreProperties.txt or DerivedNormalizationProps.txt field 0 at %s\n", fields[0][0]);
166        return;
167    }
168
169    /* parse derived binary property name, ignore unknown names */
170    i=getTokenIndex(derivedPropsNames, LENGTHOF(derivedPropsNames), fields[1][0]);
171    if(i<0) {
172        UnicodeString propName(fields[1][0], (int32_t)(fields[1][1]-fields[1][0]));
173        propName.trim();
174        if(me->unknownPropertyNames->find(propName)==NULL) {
175            UErrorCode errorCode=U_ZERO_ERROR;
176            me->unknownPropertyNames->puti(propName, 1, errorCode);
177            me->errln("UnicodeTest warning: unknown property name '%s' in DerivedCoreProperties.txt or DerivedNormalizationProps.txt\n", fields[1][0]);
178        }
179        return;
180    }
181
182    me->derivedProps[i].add(start, end);
183}
184
185void UnicodeTest::TestAdditionalProperties() {
186#if !UCONFIG_NO_NORMALIZATION
187    // test DerivedCoreProperties.txt and DerivedNormalizationProps.txt
188    if(LENGTHOF(derivedProps)<LENGTHOF(derivedPropsNames)) {
189        errln("error: UnicodeTest::derivedProps[] too short, need at least %d UnicodeSets\n",
190              LENGTHOF(derivedPropsNames));
191        return;
192    }
193    if(LENGTHOF(derivedPropsIndex)!=LENGTHOF(derivedPropsNames)) {
194        errln("error in ucdtest.cpp: LENGTHOF(derivedPropsIndex)!=LENGTHOF(derivedPropsNames)\n");
195        return;
196    }
197
198    char newPath[256];
199    char backupPath[256];
200    char *fields[2][2];
201    UErrorCode errorCode=U_ZERO_ERROR;
202
203    /* Look inside ICU_DATA first */
204    strcpy(newPath, pathToDataDirectory());
205    strcat(newPath, "unidata" U_FILE_SEP_STRING "DerivedCoreProperties.txt");
206
207    // As a fallback, try to guess where the source data was located
208    // at the time ICU was built, and look there.
209#   ifdef U_TOPSRCDIR
210        strcpy(backupPath, U_TOPSRCDIR  U_FILE_SEP_STRING "data");
211#   else
212        strcpy(backupPath, loadTestData(errorCode));
213        strcat(backupPath, U_FILE_SEP_STRING ".." U_FILE_SEP_STRING ".." U_FILE_SEP_STRING ".." U_FILE_SEP_STRING ".." U_FILE_SEP_STRING "data");
214#   endif
215    strcat(backupPath, U_FILE_SEP_STRING);
216    strcat(backupPath, "unidata" U_FILE_SEP_STRING "DerivedCoreProperties.txt");
217
218    char *path=newPath;
219    u_parseDelimitedFile(newPath, ';', fields, 2, derivedPropsLineFn, this, &errorCode);
220
221    if(errorCode==U_FILE_ACCESS_ERROR) {
222        errorCode=U_ZERO_ERROR;
223        path=backupPath;
224        u_parseDelimitedFile(backupPath, ';', fields, 2, derivedPropsLineFn, this, &errorCode);
225    }
226    if(U_FAILURE(errorCode)) {
227        errln("error parsing DerivedCoreProperties.txt: %s\n", u_errorName(errorCode));
228        return;
229    }
230    char *basename=path+strlen(path)-strlen("DerivedCoreProperties.txt");
231    strcpy(basename, "DerivedNormalizationProps.txt");
232    u_parseDelimitedFile(path, ';', fields, 2, derivedPropsLineFn, this, &errorCode);
233    if(U_FAILURE(errorCode)) {
234        errln("error parsing DerivedNormalizationProps.txt: %s\n", u_errorName(errorCode));
235        return;
236    }
237
238    // now we have all derived core properties in the UnicodeSets
239    // run them all through the API
240    int32_t rangeCount, range;
241    uint32_t i;
242    UChar32 start, end;
243
244    // test all TRUE properties
245    for(i=0; i<LENGTHOF(derivedPropsNames); ++i) {
246        rangeCount=derivedProps[i].getRangeCount();
247        for(range=0; range<rangeCount && numErrors[i]<MAX_ERRORS; ++range) {
248            start=derivedProps[i].getRangeStart(range);
249            end=derivedProps[i].getRangeEnd(range);
250            for(; start<=end; ++start) {
251                if(!u_hasBinaryProperty(start, derivedPropsIndex[i])) {
252                    dataerrln("UnicodeTest error: u_hasBinaryProperty(U+%04lx, %s)==FALSE is wrong", start, derivedPropsNames[i]);
253                    if(++numErrors[i]>=MAX_ERRORS) {
254                      dataerrln("Too many errors, moving to the next test");
255                      break;
256                    }
257                }
258            }
259        }
260    }
261
262    // invert all properties
263    for(i=0; i<LENGTHOF(derivedPropsNames); ++i) {
264        derivedProps[i].complement();
265    }
266
267    // test all FALSE properties
268    for(i=0; i<LENGTHOF(derivedPropsNames); ++i) {
269        rangeCount=derivedProps[i].getRangeCount();
270        for(range=0; range<rangeCount && numErrors[i]<MAX_ERRORS; ++range) {
271            start=derivedProps[i].getRangeStart(range);
272            end=derivedProps[i].getRangeEnd(range);
273            for(; start<=end; ++start) {
274                if(u_hasBinaryProperty(start, derivedPropsIndex[i])) {
275                    errln("UnicodeTest error: u_hasBinaryProperty(U+%04lx, %s)==TRUE is wrong\n", start, derivedPropsNames[i]);
276                    if(++numErrors[i]>=MAX_ERRORS) {
277                      errln("Too many errors, moving to the next test");
278                      break;
279                    }
280                }
281            }
282        }
283    }
284#endif /* !UCONFIG_NO_NORMALIZATION */
285}
286
287void UnicodeTest::TestBinaryValues() {
288    /*
289     * Unicode 5.1 explicitly defines binary property value aliases.
290     * Verify that they are all recognized.
291     */
292    UErrorCode errorCode=U_ZERO_ERROR;
293    UnicodeSet alpha(UNICODE_STRING_SIMPLE("[:Alphabetic:]"), errorCode);
294    if(U_FAILURE(errorCode)) {
295        dataerrln("UnicodeSet([:Alphabetic:]) failed - %s", u_errorName(errorCode));
296        return;
297    }
298
299    static const char *const falseValues[]={ "N", "No", "F", "False" };
300    static const char *const trueValues[]={ "Y", "Yes", "T", "True" };
301    int32_t i;
302    for(i=0; i<LENGTHOF(falseValues); ++i) {
303        UnicodeString pattern=UNICODE_STRING_SIMPLE("[:Alphabetic=:]");
304        pattern.insert(pattern.length()-2, UnicodeString(falseValues[i], -1, US_INV));
305        errorCode=U_ZERO_ERROR;
306        UnicodeSet set(pattern, errorCode);
307        if(U_FAILURE(errorCode)) {
308            errln("UnicodeSet([:Alphabetic=%s:]) failed - %s\n", falseValues[i], u_errorName(errorCode));
309            continue;
310        }
311        set.complement();
312        if(set!=alpha) {
313            errln("UnicodeSet([:Alphabetic=%s:]).complement()!=UnicodeSet([:Alphabetic:])\n", falseValues[i]);
314        }
315    }
316    for(i=0; i<LENGTHOF(trueValues); ++i) {
317        UnicodeString pattern=UNICODE_STRING_SIMPLE("[:Alphabetic=:]");
318        pattern.insert(pattern.length()-2, UnicodeString(trueValues[i], -1, US_INV));
319        errorCode=U_ZERO_ERROR;
320        UnicodeSet set(pattern, errorCode);
321        if(U_FAILURE(errorCode)) {
322            errln("UnicodeSet([:Alphabetic=%s:]) failed - %s\n", trueValues[i], u_errorName(errorCode));
323            continue;
324        }
325        if(set!=alpha) {
326            errln("UnicodeSet([:Alphabetic=%s:])!=UnicodeSet([:Alphabetic:])\n", trueValues[i]);
327        }
328    }
329}
330
331void UnicodeTest::TestConsistency() {
332#if !UCONFIG_NO_NORMALIZATION
333    /*
334     * Test for an example that getCanonStartSet() delivers
335     * all characters that compose from the input one,
336     * even in multiple steps.
337     * For example, the set for "I" (0049) should contain both
338     * I-diaeresis (00CF) and I-diaeresis-acute (1E2E).
339     * In general, the set for the middle such character should be a subset
340     * of the set for the first.
341     */
342    IcuTestErrorCode errorCode(*this, "TestConsistency");
343    const Normalizer2 *nfd=Normalizer2::getInstance(NULL, "nfc", UNORM2_DECOMPOSE, errorCode);
344    const Normalizer2Impl *nfcImpl=Normalizer2Factory::getNFCImpl(errorCode);
345    if(!nfcImpl->ensureCanonIterData(errorCode) || errorCode.isFailure()) {
346        dataerrln("Normalizer2::getInstance(NFD) or Normalizer2Factory::getNFCImpl() failed - %s\n",
347                  errorCode.errorName());
348        errorCode.reset();
349        return;
350    }
351
352    UnicodeSet set1, set2;
353    if (nfcImpl->getCanonStartSet(0x49, set1)) {
354        /* enumerate all characters that are plausible to be latin letters */
355        for(UChar start=0xa0; start<0x2000; ++start) {
356            UnicodeString decomp=nfd->normalize(UnicodeString(start), errorCode);
357            if(decomp.length()>1 && decomp[0]==0x49) {
358                set2.add(start);
359            }
360        }
361
362        if (set1!=set2) {
363            errln("[canon start set of 0049] != [all c with canon decomp with 0049]");
364        }
365        // This was available in cucdtst.c but the test had to move to intltest
366        // because the new internal normalization functions are in C++.
367        //compareUSets(set1, set2,
368        //             "[canon start set of 0049]", "[all c with canon decomp with 0049]",
369        //             TRUE);
370    } else {
371        errln("NFC.getCanonStartSet() returned FALSE");
372    }
373#endif
374}
375
376/**
377 * Test various implementations of Pattern_Syntax & Pattern_White_Space.
378 */
379void UnicodeTest::TestPatternProperties() {
380    IcuTestErrorCode errorCode(*this, "TestPatternProperties()");
381    UnicodeSet syn_pp;
382    UnicodeSet syn_prop(UNICODE_STRING_SIMPLE("[:Pattern_Syntax:]"), errorCode);
383    UnicodeSet syn_list(
384        "[!-/\\:-@\\[-\\^`\\{-~"
385        "\\u00A1-\\u00A7\\u00A9\\u00AB\\u00AC\\u00AE\\u00B0\\u00B1\\u00B6\\u00BB\\u00BF\\u00D7\\u00F7"
386        "\\u2010-\\u2027\\u2030-\\u203E\\u2041-\\u2053\\u2055-\\u205E\\u2190-\\u245F\\u2500-\\u2775"
387        "\\u2794-\\u2BFF\\u2E00-\\u2E7F\\u3001-\\u3003\\u3008-\\u3020\\u3030\\uFD3E\\uFD3F\\uFE45\\uFE46]", errorCode);
388    UnicodeSet ws_pp;
389    UnicodeSet ws_prop(UNICODE_STRING_SIMPLE("[:Pattern_White_Space:]"), errorCode);
390    UnicodeSet ws_list(UNICODE_STRING_SIMPLE("[\\u0009-\\u000D\\ \\u0085\\u200E\\u200F\\u2028\\u2029]"), errorCode);
391    UnicodeSet syn_ws_pp;
392    UnicodeSet syn_ws_prop(syn_prop);
393    syn_ws_prop.addAll(ws_prop);
394    for(UChar32 c=0; c<=0xffff; ++c) {
395        if(PatternProps::isSyntax(c)) {
396            syn_pp.add(c);
397        }
398        if(PatternProps::isWhiteSpace(c)) {
399            ws_pp.add(c);
400        }
401        if(PatternProps::isSyntaxOrWhiteSpace(c)) {
402            syn_ws_pp.add(c);
403        }
404    }
405    compareUSets(syn_pp, syn_prop,
406                 "PatternProps.isSyntax()", "[:Pattern_Syntax:]", TRUE);
407    compareUSets(syn_pp, syn_list,
408                 "PatternProps.isSyntax()", "[Pattern_Syntax ranges]", TRUE);
409    compareUSets(ws_pp, ws_prop,
410                 "PatternProps.isWhiteSpace()", "[:Pattern_White_Space:]", TRUE);
411    compareUSets(ws_pp, ws_list,
412                 "PatternProps.isWhiteSpace()", "[Pattern_White_Space ranges]", TRUE);
413    compareUSets(syn_ws_pp, syn_ws_prop,
414                 "PatternProps.isSyntaxOrWhiteSpace()",
415                 "[[:Pattern_Syntax:][:Pattern_White_Space:]]", TRUE);
416}
417
418// So far only minimal port of Java & cucdtst.c compareUSets().
419UBool
420UnicodeTest::compareUSets(const UnicodeSet &a, const UnicodeSet &b,
421                          const char *a_name, const char *b_name,
422                          UBool diffIsError) {
423    UBool same= a==b;
424    if(!same && diffIsError) {
425        errln("Sets are different: %s vs. %s\n", a_name, b_name);
426    }
427    return same;
428}
429