1/*
2**********************************************************************
3*   Copyright (C) 1999-2014, International Business Machines
4*   Corporation and others.  All Rights Reserved.
5**********************************************************************
6*   Date        Name        Description
7*   12/09/99    aliu        Ported from Java.
8**********************************************************************
9*/
10
11#include "unicode/utypes.h"
12
13#if !UCONFIG_NO_COLLATION
14
15#include "thcoll.h"
16#include "unicode/utypes.h"
17#include "unicode/coll.h"
18#include "unicode/localpointer.h"
19#include "unicode/sortkey.h"
20#include "unicode/tblcoll.h"
21#include "unicode/ustring.h"
22#include "cstring.h"
23#include "filestrm.h"
24#include "textfile.h"
25
26/**
27 * The TestDictionary test expects a file of this name, with this
28 * encoding, to be present in the directory $ICU/source/test/testdata.
29 */
30//#define TEST_FILE           "th18057.txt"
31
32/**
33 * This is the most failures we show in TestDictionary.  If this number
34 * is < 0, we show all failures.
35 */
36#define MAX_FAILURES_TO_SHOW -1
37
38CollationThaiTest::CollationThaiTest() {
39    UErrorCode status = U_ZERO_ERROR;
40    coll = Collator::createInstance(Locale("th", "TH", ""), status);
41    if (coll && U_SUCCESS(status)) {
42        //coll->setStrength(Collator::TERTIARY);
43    } else {
44        delete coll;
45        coll = 0;
46    }
47}
48
49CollationThaiTest::~CollationThaiTest() {
50    delete coll;
51}
52
53void CollationThaiTest::runIndexedTest(int32_t index, UBool exec, const char* &name,
54                                       char* /*par*/) {
55
56    if((!coll) && exec) {
57      dataerrln(__FILE__ " cannot test - failed to create collator.");
58      name = "some test";
59      return;
60    }
61
62    switch (index) {
63        TESTCASE(0,TestDictionary);
64        TESTCASE(1,TestCornerCases);
65        TESTCASE(2,TestNamesList);
66        TESTCASE(3,TestInvalidThai);
67        TESTCASE(4,TestReordering);
68        default: name = ""; break;
69    }
70}
71
72/**
73 * Read the external names list, and confirms that the collator
74 * gets the same results when comparing lines one to another
75 * using regular and iterative comparison.
76 */
77void CollationThaiTest::TestNamesList(void) {
78    if (coll == 0) {
79        errln("Error: could not construct Thai collator");
80        return;
81    }
82
83    UErrorCode ec = U_ZERO_ERROR;
84    TextFile names("TestNames_Thai.txt", "UTF16LE", ec);
85    if (U_FAILURE(ec)) {
86        logln("Can't open TestNames_Thai.txt: %s; skipping test",
87              u_errorName(ec));
88        return;
89    }
90
91    //
92    // Loop through each word in the dictionary and compare it to the previous
93    // word.  They should be in sorted order.
94    //
95    UnicodeString lastWord, word;
96    //int32_t failed = 0;
97    int32_t wordCount = 0;
98    while (names.readLineSkippingComments(word, ec, FALSE) && U_SUCCESS(ec)) {
99
100        // Show the first 8 words being compared, so we can see what's happening
101        ++wordCount;
102        if (wordCount <= 8) {
103            UnicodeString str;
104            logln((UnicodeString)"Word " + wordCount + ": " + IntlTest::prettify(word, str));
105        }
106
107        if (lastWord.length() > 0) {
108            Collator::EComparisonResult result = coll->compare(lastWord, word);
109            doTest(coll, lastWord, word, result);
110        }
111        lastWord = word;
112    }
113
114    assertSuccess("readLine", ec);
115
116    logln((UnicodeString)"Words checked: " + wordCount);
117}
118
119/**
120 * Read the external dictionary file, which is already in proper
121 * sorted order, and confirm that the collator compares each line as
122 * preceding the following line.
123 */
124void CollationThaiTest::TestDictionary(void) {
125    if (coll == 0) {
126        errln("Error: could not construct Thai collator");
127        return;
128    }
129
130    UErrorCode ec = U_ZERO_ERROR;
131    TextFile riwords("riwords.txt", "UTF8", ec);
132    if (U_FAILURE(ec)) {
133        logln("Can't open riwords.txt: %s; skipping test",
134              u_errorName(ec));
135        return;
136    }
137
138    //
139    // Loop through each word in the dictionary and compare it to the previous
140    // word.  They should be in sorted order.
141    //
142    UnicodeString lastWord, word;
143    int32_t failed = 0;
144    int32_t wordCount = 0;
145    while (riwords.readLineSkippingComments(word, ec, FALSE) && U_SUCCESS(ec)) {
146
147        // Show the first 8 words being compared, so we can see what's happening
148        ++wordCount;
149        if (wordCount <= 8) {
150            UnicodeString str;
151            logln((UnicodeString)"Word " + wordCount + ": " + IntlTest::prettify(word, str));
152        }
153
154        if (lastWord.length() > 0) {
155            int32_t result = coll->compare(lastWord, word);
156
157            if (result > 0) {
158                failed++;
159                if (MAX_FAILURES_TO_SHOW < 0 || failed <= MAX_FAILURES_TO_SHOW) {
160                    UnicodeString str;
161                    UnicodeString msg =
162                        UnicodeString("--------------------------------------------\n")
163                        + riwords.getLineNumber()
164                        + " compare(" + IntlTest::prettify(lastWord, str);
165                    msg += UnicodeString(", ")
166                        + IntlTest::prettify(word, str) + ") returned " + result
167                        + ", expected -1\n";
168                    UErrorCode status = U_ZERO_ERROR;
169                    CollationKey k1, k2;
170                    coll->getCollationKey(lastWord, k1, status);
171                    coll->getCollationKey(word, k2, status);
172                    if (U_FAILURE(status)) {
173                        errln((UnicodeString)"Fail: getCollationKey returned " + u_errorName(status));
174                        return;
175                    }
176                    msg.append("key1: ").append(prettify(k1, str)).append("\n");
177                    msg.append("key2: ").append(prettify(k2, str));
178                    errln(msg);
179                }
180            }
181        }
182        lastWord = word;
183    }
184
185    assertSuccess("readLine", ec);
186
187    if (failed != 0) {
188        if (failed > MAX_FAILURES_TO_SHOW) {
189            errln((UnicodeString)"Too many failures; only the first " +
190                  MAX_FAILURES_TO_SHOW + " failures were shown");
191        }
192        errln((UnicodeString)"Summary: " + failed + " of " + (riwords.getLineNumber() - 1) +
193              " comparisons failed");
194    }
195
196    logln((UnicodeString)"Words checked: " + wordCount);
197}
198
199/**
200 * Odd corner conditions taken from "How to Sort Thai Without Rewriting Sort",
201 * by Doug Cooper, http://seasrc.th.net/paper/thaisort.zip
202 */
203void CollationThaiTest::TestCornerCases(void) {
204    const char* TESTS[] = {
205        // Shorter words precede longer
206        "\\u0e01",                               "<",    "\\u0e01\\u0e01",
207
208        // Tone marks are considered after letters (i.e. are primary ignorable)
209        "\\u0e01\\u0e32",                        "<",    "\\u0e01\\u0e49\\u0e32",
210
211        // ditto for other over-marks
212        "\\u0e01\\u0e32",                        "<",    "\\u0e01\\u0e32\\u0e4c",
213
214        // commonly used mark-in-context order.
215        // In effect, marks are sorted after each syllable.
216        "\\u0e01\\u0e32\\u0e01\\u0e49\\u0e32",   "<",    "\\u0e01\\u0e48\\u0e32\\u0e01\\u0e49\\u0e32",
217
218        // Hyphens and other punctuation follow whitespace but come before letters
219        "\\u0e01\\u0e32",                        "=",    "\\u0e01\\u0e32-",
220        "\\u0e01\\u0e32-",                       "<",    "\\u0e01\\u0e32\\u0e01\\u0e32",
221
222        // Doubler follows an indentical word without the doubler
223        "\\u0e01\\u0e32",                        "=",    "\\u0e01\\u0e32\\u0e46",
224        "\\u0e01\\u0e32\\u0e46",                 "<",    "\\u0e01\\u0e32\\u0e01\\u0e32",
225
226
227        // \\u0e45 after either \\u0e24 or \\u0e26 is treated as a single
228        // combining character, similar to "c < ch" in traditional spanish.
229        // TODO: beef up this case
230        "\\u0e24\\u0e29\\u0e35",                 "<",    "\\u0e24\\u0e45\\u0e29\\u0e35",
231        "\\u0e26\\u0e29\\u0e35",                 "<",    "\\u0e26\\u0e45\\u0e29\\u0e35",
232
233        // Vowels reorder, should compare \\u0e2d and \\u0e34
234        "\\u0e40\\u0e01\\u0e2d",                 "<",    "\\u0e40\\u0e01\\u0e34",
235
236        // Tones are compared after the rest of the word (e.g. primary ignorable)
237        "\\u0e01\\u0e32\\u0e01\\u0e48\\u0e32",   "<",    "\\u0e01\\u0e49\\u0e32\\u0e01\\u0e32",
238
239        // Periods are ignored entirely
240        "\\u0e01.\\u0e01.",                      "<",    "\\u0e01\\u0e32",
241    };
242    const int32_t TESTS_length = (int32_t)(sizeof(TESTS)/sizeof(TESTS[0]));
243
244    if (coll == 0) {
245        errln("Error: could not construct Thai collator");
246        return;
247    }
248    compareArray(*coll, TESTS, TESTS_length);
249}
250
251//------------------------------------------------------------------------
252// Internal utilities
253//------------------------------------------------------------------------
254
255void CollationThaiTest::compareArray(Collator& c, const char* tests[],
256                                     int32_t testsLength) {
257    for (int32_t i = 0; i < testsLength; i += 3) {
258
259        Collator::EComparisonResult expect;
260        if (tests[i+1][0] == '<') {
261          expect = Collator::LESS;
262        } else if (tests[i+1][0] == '>') {
263          expect = Collator::GREATER;
264        } else if (tests[i+1][0] == '=') {
265          expect = Collator::EQUAL;
266        } else {
267            // expect = Integer.decode(tests[i+1]).intValue();
268            errln((UnicodeString)"Error: unknown operator " + tests[i+1]);
269            return;
270        }
271
272        UnicodeString s1, s2;
273        parseChars(s1, tests[i]);
274        parseChars(s2, tests[i+2]);
275
276        doTest(&c, s1, s2, expect);
277#if 0
278        UErrorCode status = U_ZERO_ERROR;
279        int32_t result = c.compare(s1, s2);
280        if (sign(result) != sign(expect))
281        {
282            UnicodeString t1, t2;
283            errln(UnicodeString("") +
284                  i/3 + ": compare(" + IntlTest::prettify(s1, t1)
285                  + " , " + IntlTest::prettify(s2, t2)
286                  + ") got " + result + "; expected " + expect);
287
288            CollationKey k1, k2;
289            c.getCollationKey(s1, k1, status);
290            c.getCollationKey(s2, k2, status);
291            if (U_FAILURE(status)) {
292                errln((UnicodeString)"Fail: getCollationKey returned " + u_errorName(status));
293                return;
294            }
295            errln((UnicodeString)"  key1: " + prettify(k1, t1) );
296            errln((UnicodeString)"  key2: " + prettify(k2, t2) );
297        }
298        else
299        {
300            // Collator.compare worked OK; now try the collation keys
301            CollationKey k1, k2;
302            c.getCollationKey(s1, k1, status);
303            c.getCollationKey(s2, k2, status);
304            if (U_FAILURE(status)) {
305                errln((UnicodeString)"Fail: getCollationKey returned " + u_errorName(status));
306                return;
307            }
308
309            result = k1.compareTo(k2);
310            if (sign(result) != sign(expect)) {
311                UnicodeString t1, t2;
312                errln(UnicodeString("") +
313                      i/3 + ": key(" + IntlTest::prettify(s1, t1)
314                      + ").compareTo(key(" + IntlTest::prettify(s2, t2)
315                      + ")) got " + result + "; expected " + expect);
316
317                errln((UnicodeString)"  " + prettify(k1, t1) + " vs. " + prettify(k2, t2));
318            }
319        }
320#endif
321    }
322}
323
324int8_t CollationThaiTest::sign(int32_t i) {
325    if (i < 0) return -1;
326    if (i > 0) return 1;
327    return 0;
328}
329
330/**
331 * Set a UnicodeString corresponding to the given string.  Use
332 * UnicodeString and the default converter, unless we see the sequence
333 * "\\u", in which case we interpret the subsequent escape.
334 */
335UnicodeString& CollationThaiTest::parseChars(UnicodeString& result,
336                                             const char* chars) {
337    return result = CharsToUnicodeString(chars);
338}
339
340UCollator *thaiColl = NULL;
341
342U_CDECL_BEGIN
343static int U_CALLCONV
344StrCmp(const void *p1, const void *p2) {
345  return ucol_strcoll(thaiColl, *(UChar **) p1, -1,  *(UChar **)p2, -1);
346}
347U_CDECL_END
348
349
350#define LINES 6
351
352void CollationThaiTest::TestInvalidThai(void) {
353  const char *tests[LINES] = {
354    "\\u0E44\\u0E01\\u0E44\\u0E01",
355    "\\u0E44\\u0E01\\u0E01\\u0E44",
356    "\\u0E01\\u0E44\\u0E01\\u0E44",
357    "\\u0E01\\u0E01\\u0E44\\u0E44",
358    "\\u0E44\\u0E44\\u0E01\\u0E01",
359    "\\u0E01\\u0E44\\u0E44\\u0E01",
360  };
361
362  UChar strings[LINES][20];
363
364  UChar *toSort[LINES];
365
366  int32_t i = 0, j = 0, len = 0;
367
368  UErrorCode coll_status = U_ZERO_ERROR;
369  UnicodeString iteratorText;
370
371  thaiColl = ucol_open ("th_TH", &coll_status);
372  if (U_FAILURE(coll_status)) {
373    errln("Error opening Thai collator: %s", u_errorName(coll_status));
374    return;
375  }
376
377  CollationElementIterator* c = ((RuleBasedCollator *)coll)->createCollationElementIterator( iteratorText );
378
379  for(i = 0; i < (int32_t)(sizeof(tests)/sizeof(tests[0])); i++) {
380    len = u_unescape(tests[i], strings[i], 20);
381    strings[i][len] = 0;
382    toSort[i] = strings[i];
383  }
384
385  qsort (toSort, LINES, sizeof (UChar *), StrCmp);
386
387  for (i=0; i < LINES; i++)
388  {
389    logln("%i", i);
390      for (j=i+1; j < LINES; j++) {
391          if (ucol_strcoll (thaiColl, toSort[i], -1, toSort[j], -1) == UCOL_GREATER)
392          {
393              // inconsistency ordering found!
394            errln("Inconsistent ordering between strings %i and %i", i, j);
395          }
396      }
397      iteratorText.setTo(toSort[i]);
398      c->setText(iteratorText, coll_status);
399      backAndForth(*c);
400  }
401
402
403  ucol_close(thaiColl);
404  delete c;
405}
406
407void CollationThaiTest::TestReordering(void) {
408  // Until UCA 4.1, the collation code swapped Thai/Lao prevowels with the following consonants,
409  // resulting in consonant+prevowel == prevowel+consonant.
410  // From UCA 5.0 on, there are order-reversing contractions for prevowel+consonant.
411  // From UCA 5.0 until UCA 6.1, there was a tertiary difference between
412  // consonant+prevowel and prevowel+consonant.
413  // In UCA 6.2, they compare equal again.
414  // The test was modified to using a collator with strength=secondary,
415  // ignoring possible tertiary differences.
416  const char *tests[] = {
417    "\\u0E41c\\u0301",       "=", "\\u0E41\\u0107", // composition
418    "\\u0E41\\U0001D7CE",    "<", "\\u0E41\\U0001D7CF", // supplementaries
419    "\\u0E41\\U0001D15F",    "=", "\\u0E41\\U0001D158\\U0001D165", // supplementary composition decomps to supplementary
420    "\\u0E41\\U0002F802",    "=", "\\u0E41\\u4E41", // supplementary composition decomps to BMP
421    "\\u0E41\\u0301",        "=", "\\u0E41\\u0301", // unsafe (just checking backwards iteration)
422    "\\u0E41\\u0301\\u0316", "=", "\\u0E41\\u0316\\u0301",
423
424    "\\u0e24\\u0e41",        "=", "\\u0e41\\u0e24", // exiting contraction bug
425    "\\u0e3f\\u0e3f\\u0e24\\u0e41", "=", "\\u0e3f\\u0e3f\\u0e41\\u0e24",
426
427    "abc\\u0E41c\\u0301",       "=", "abc\\u0E41\\u0107", // composition
428    "abc\\u0E41\\U0001D000",    "<", "abc\\u0E41\\U0001D001", // supplementaries
429    "abc\\u0E41\\U0001D15F",    "=", "abc\\u0E41\\U0001D158\\U0001D165", // supplementary composition decomps to supplementary
430    "abc\\u0E41\\U0002F802",    "=", "abc\\u0E41\\u4E41", // supplementary composition decomps to BMP
431    "abc\\u0E41\\u0301",        "=", "abc\\u0E41\\u0301", // unsafe (just checking backwards iteration)
432    "abc\\u0E41\\u0301\\u0316", "=", "abc\\u0E41\\u0316\\u0301",
433
434    "\\u0E41c\\u0301abc",       "=", "\\u0E41\\u0107abc", // composition
435    "\\u0E41\\U0001D000abc",    "<", "\\u0E41\\U0001D001abc", // supplementaries
436    "\\u0E41\\U0001D15Fabc",    "=", "\\u0E41\\U0001D158\\U0001D165abc", // supplementary composition decomps to supplementary
437    "\\u0E41\\U0002F802abc",    "=", "\\u0E41\\u4E41abc", // supplementary composition decomps to BMP
438    "\\u0E41\\u0301abc",        "=", "\\u0E41\\u0301abc", // unsafe (just checking backwards iteration)
439    "\\u0E41\\u0301\\u0316abc", "=", "\\u0E41\\u0316\\u0301abc",
440
441    "abc\\u0E41c\\u0301abc",       "=", "abc\\u0E41\\u0107abc", // composition
442    "abc\\u0E41\\U0001D000abc",    "<", "abc\\u0E41\\U0001D001abc", // supplementaries
443    "abc\\u0E41\\U0001D15Fabc",    "=", "abc\\u0E41\\U0001D158\\U0001D165abc", // supplementary composition decomps to supplementary
444    "abc\\u0E41\\U0002F802abc",    "=", "abc\\u0E41\\u4E41abc", // supplementary composition decomps to BMP
445    "abc\\u0E41\\u0301abc",        "=", "abc\\u0E41\\u0301abc", // unsafe (just checking backwards iteration)
446    "abc\\u0E41\\u0301\\u0316abc", "=", "abc\\u0E41\\u0316\\u0301abc",
447  };
448
449  LocalPointer<Collator> coll2(coll->clone());
450  UErrorCode status = U_ZERO_ERROR;
451  coll2->setAttribute(UCOL_STRENGTH, UCOL_SECONDARY, status);
452  if(U_FAILURE(status)) {
453    errln("Unable to set the Thai collator clone to secondary strength");
454    return;
455  }
456  compareArray(*coll2, tests, sizeof(tests)/sizeof(tests[0]));
457
458  const char *rule = "& c < ab";
459  const char *testcontraction[] = { "\\u0E41ab", ">", "\\u0E41c"}; // After UCA 4.1 Thai are normal so won't break a contraction
460  UnicodeString rules;
461  parseChars(rules, rule);
462  LocalPointer<RuleBasedCollator> rcoll(new RuleBasedCollator(rules, status), status);
463  if(U_SUCCESS(status)) {
464    compareArray(*rcoll, testcontraction, 3);
465  } else {
466    errln("Couldn't instantiate collator from rules");
467  }
468
469}
470
471
472#endif /* #if !UCONFIG_NO_COLLATION */
473