1/********************************************************************
2 * COPYRIGHT:
3 * Copyright (c) 2012-2014, International Business Machines Corporation
4 * and others. All Rights Reserved.
5 ********************************************************************/
6//
7//   file:  alphaindextst.cpp
8//          Alphabetic Index Tests.
9//
10//   Note: please... no character literals cast to UChars.. use (UChar)0xZZZZ
11
12#include <stdio.h>  // for sprintf
13
14#include "intltest.h"
15#include "alphaindextst.h"
16
17#include "unicode/alphaindex.h"
18#include "unicode/coll.h"
19#include "unicode/localpointer.h"
20#include "unicode/tblcoll.h"
21#include "unicode/uniset.h"
22
23#if !UCONFIG_NO_COLLATION && !UCONFIG_NO_NORMALIZATION
24
25// #include <string>
26// #include <iostream>
27
28#define LENGTHOF(array) (int32_t)(sizeof(array)/sizeof((array)[0]))
29
30namespace {
31
32UnicodeString joinLabelsAndAppend(AlphabeticIndex::ImmutableIndex &index, UnicodeString &dest) {
33    int32_t oldLength = dest.length();
34    const AlphabeticIndex::Bucket *bucket;
35    for (int32_t i = 0; (bucket = index.getBucket(i)) != NULL; ++i) {
36        if (dest.length() > oldLength) {
37            dest.append((UChar)0x3A);  // ':'
38        }
39        dest.append(bucket->getLabel());
40    }
41    return dest;
42}
43
44}  // namespace
45
46AlphabeticIndexTest::AlphabeticIndexTest() {
47}
48
49AlphabeticIndexTest::~AlphabeticIndexTest() {
50}
51
52void AlphabeticIndexTest::runIndexedTest( int32_t index, UBool exec, const char* &name, char* /*par*/ )
53{
54    if (exec) logln("TestSuite AlphabeticIndex: ");
55    TESTCASE_AUTO_BEGIN;
56    TESTCASE_AUTO(APITest);
57    TESTCASE_AUTO(ManyLocalesTest);
58    TESTCASE_AUTO(HackPinyinTest);
59    TESTCASE_AUTO(TestBug9009);
60    TESTCASE_AUTO(TestIndexCharactersList);
61    TESTCASE_AUTO(TestHaniFirst);
62    TESTCASE_AUTO(TestPinyinFirst);
63    TESTCASE_AUTO(TestSchSt);
64    TESTCASE_AUTO(TestNoLabels);
65    TESTCASE_AUTO(TestChineseZhuyin);
66    TESTCASE_AUTO(TestJapaneseKanji);
67    TESTCASE_AUTO_END;
68}
69
70#define TEST_CHECK_STATUS {if (U_FAILURE(status)) {dataerrln("%s:%d: Test failure.  status=%s", \
71                                                              __FILE__, __LINE__, u_errorName(status)); return;}}
72
73#define TEST_ASSERT(expr) {if ((expr)==FALSE) {errln("%s:%d: Test failure \n", __FILE__, __LINE__);};}
74
75//
76//  APITest.   Invoke every function at least once, and check that it does something.
77//             Does not attempt to check complete functionality.
78//
79void AlphabeticIndexTest::APITest() {
80    //
81    //  Simple constructor and destructor,  getBucketCount()
82    //
83    UErrorCode status = U_ZERO_ERROR;
84    int32_t lc = 0;
85    int32_t i  = 0;
86    AlphabeticIndex *index = new AlphabeticIndex(Locale::getEnglish(), status);
87    TEST_CHECK_STATUS;
88    lc = index->getBucketCount(status);
89    TEST_CHECK_STATUS;
90    TEST_ASSERT(28 == lc);    // 26 letters plus two under/overflow labels.
91    //printf("getBucketCount() == %d\n", lc);
92    delete index;
93
94    // Constructor from a Collator
95    //
96    status = U_ZERO_ERROR;
97    RuleBasedCollator *coll = dynamic_cast<RuleBasedCollator *>(
98        Collator::createInstance(Locale::getGerman(), status));
99    TEST_CHECK_STATUS;
100    TEST_ASSERT(coll != NULL);
101    index = new AlphabeticIndex(coll, status);
102    TEST_CHECK_STATUS;
103    TEST_ASSERT(coll == &index->getCollator());
104    assertEquals("only the underflow label in an index built from a collator",
105                 1, index->getBucketCount(status));
106    TEST_CHECK_STATUS;
107    delete index;
108
109
110    // addLabels()
111
112    status = U_ZERO_ERROR;
113    index = new AlphabeticIndex(Locale::getEnglish(), status);
114    TEST_CHECK_STATUS;
115    UnicodeSet additions;
116    additions.add((UChar32)0x410).add((UChar32)0x415);   // A couple of Cyrillic letters
117    index->addLabels(additions, status);
118    TEST_CHECK_STATUS;
119    lc = index->getBucketCount(status);
120    TEST_CHECK_STATUS;
121    assertEquals("underflow, A-Z, inflow, 2 Cyrillic, overflow",
122                 31, index->getBucketCount(status));
123    // std::cout << lc << std::endl;
124    delete index;
125
126
127    // addLabels(Locale)
128
129    status = U_ZERO_ERROR;
130    index = new AlphabeticIndex(Locale::getEnglish(), status);
131    TEST_CHECK_STATUS;
132    AlphabeticIndex &aip = index->addLabels(Locale::getJapanese(), status);
133    TEST_ASSERT(&aip == index);
134    TEST_CHECK_STATUS;
135    lc = index->getBucketCount(status);
136    TEST_CHECK_STATUS;
137    TEST_ASSERT(35 < lc);  // Japanese should add a bunch.  Don't rely on the exact value.
138    delete index;
139
140    // GetCollator(),  Get under/in/over flow labels
141
142    status = U_ZERO_ERROR;
143    index = new AlphabeticIndex(Locale::getGerman(), status);
144    TEST_CHECK_STATUS;
145    Collator *germanCol = Collator::createInstance(Locale::getGerman(), status);
146    TEST_CHECK_STATUS;
147    const RuleBasedCollator &indexCol = index->getCollator();
148    TEST_ASSERT(*germanCol == indexCol);
149    delete germanCol;
150
151    UnicodeString ELLIPSIS;  ELLIPSIS.append((UChar32)0x2026);
152    UnicodeString s = index->getUnderflowLabel();
153    TEST_ASSERT(ELLIPSIS == s);
154    s = index->getOverflowLabel();
155    TEST_ASSERT(ELLIPSIS == s);
156    s = index->getInflowLabel();
157    TEST_ASSERT(ELLIPSIS == s);
158    index->setOverflowLabel(UNICODE_STRING_SIMPLE("O"), status);
159    index->setUnderflowLabel(UNICODE_STRING_SIMPLE("U"), status).setInflowLabel(UNICODE_STRING_SIMPLE("I"), status);
160    s = index->getUnderflowLabel();
161    TEST_ASSERT(UNICODE_STRING_SIMPLE("U") == s);
162    s = index->getOverflowLabel();
163    TEST_ASSERT(UNICODE_STRING_SIMPLE("O") == s);
164    s = index->getInflowLabel();
165    TEST_ASSERT(UNICODE_STRING_SIMPLE("I") == s);
166
167
168
169
170    delete index;
171
172
173
174    const UnicodeString adam = UNICODE_STRING_SIMPLE("Adam");
175    const UnicodeString baker = UNICODE_STRING_SIMPLE("Baker");
176    const UnicodeString charlie = UNICODE_STRING_SIMPLE("Charlie");
177    const UnicodeString chad = UNICODE_STRING_SIMPLE("Chad");
178    const UnicodeString zed  = UNICODE_STRING_SIMPLE("Zed");
179    const UnicodeString Cyrillic = UNICODE_STRING_SIMPLE("\\u0410\\u0443\\u0435").unescape();
180
181    // addRecord(), verify that it comes back out.
182    //
183    status = U_ZERO_ERROR;
184    index = new AlphabeticIndex(Locale::getEnglish(), status);
185    TEST_CHECK_STATUS;
186    index->addRecord(UnicodeString("Adam"), this, status);
187    UBool   b;
188    TEST_CHECK_STATUS;
189    index->resetBucketIterator(status);
190    TEST_CHECK_STATUS;
191    index->nextBucket(status);  // Move to underflow label
192    index->nextBucket(status);  // Move to "A"
193    TEST_CHECK_STATUS;
194    const UnicodeString &label2 = index->getBucketLabel();
195    UnicodeString A_STR = UNICODE_STRING_SIMPLE("A");
196    TEST_ASSERT(A_STR == label2);
197
198    b = index->nextRecord(status);
199    TEST_CHECK_STATUS;
200    TEST_ASSERT(b);
201    const UnicodeString &itemName = index->getRecordName();
202    TEST_ASSERT(adam == itemName);
203
204    const void *itemContext = index->getRecordData();
205    TEST_ASSERT(itemContext == this);
206
207    delete index;
208
209    // clearRecords, addRecord(), Iteration
210
211    status = U_ZERO_ERROR;
212    index = new AlphabeticIndex(Locale::getEnglish(), status);
213    TEST_CHECK_STATUS;
214    while (index->nextBucket(status)) {
215        TEST_CHECK_STATUS;
216        while (index->nextRecord(status)) {
217            TEST_CHECK_STATUS;
218            TEST_ASSERT(FALSE);   // No items have been added.
219        }
220        TEST_CHECK_STATUS;
221    }
222
223    index->addRecord(adam, NULL, status);
224    index->addRecord(baker, NULL, status);
225    index->addRecord(charlie, NULL, status);
226    index->addRecord(chad, NULL, status);
227    TEST_CHECK_STATUS;
228    int itemCount = 0;
229    index->resetBucketIterator(status);
230    while (index->nextBucket(status)) {
231        TEST_CHECK_STATUS;
232        while (index->nextRecord(status)) {
233            TEST_CHECK_STATUS;
234            ++itemCount;
235        }
236    }
237    TEST_CHECK_STATUS;
238    TEST_ASSERT(itemCount == 4);
239
240    TEST_ASSERT(index->nextBucket(status) == FALSE);
241    index->resetBucketIterator(status);
242    TEST_CHECK_STATUS;
243    TEST_ASSERT(index->nextBucket(status) == TRUE);
244
245    index->clearRecords(status);
246    TEST_CHECK_STATUS;
247    index->resetBucketIterator(status);
248    while (index->nextBucket(status)) {
249        TEST_CHECK_STATUS;
250        while (index->nextRecord(status)) {
251            TEST_ASSERT(FALSE);   // No items have been added.
252        }
253    }
254    TEST_CHECK_STATUS;
255    delete index;
256
257    // getBucketLabel(), getBucketType()
258
259    status = U_ZERO_ERROR;
260    index = new AlphabeticIndex(Locale::getEnglish(), status);
261    TEST_CHECK_STATUS;
262    index->setUnderflowLabel(adam, status).setOverflowLabel(charlie, status);
263    TEST_CHECK_STATUS;
264    for (i=0; index->nextBucket(status); i++) {
265        TEST_CHECK_STATUS;
266        UnicodeString label = index->getBucketLabel();
267        UAlphabeticIndexLabelType type = index->getBucketLabelType();
268        if (i == 0) {
269            TEST_ASSERT(type == U_ALPHAINDEX_UNDERFLOW);
270            TEST_ASSERT(label == adam);
271        } else if (i <= 26) {
272            // Labels A - Z for English locale
273            TEST_ASSERT(type == U_ALPHAINDEX_NORMAL);
274            UnicodeString expectedLabel((UChar)(0x40 + i));
275            TEST_ASSERT(expectedLabel == label);
276        } else if (i == 27) {
277            TEST_ASSERT(type == U_ALPHAINDEX_OVERFLOW);
278            TEST_ASSERT(label == charlie);
279        } else {
280            TEST_ASSERT(FALSE);
281        }
282    }
283    TEST_ASSERT(i==28);
284    delete index;
285
286    // getBucketIndex()
287
288    status = U_ZERO_ERROR;
289    index = new AlphabeticIndex(Locale::getEnglish(), status);
290    TEST_CHECK_STATUS;
291    int32_t n = index->getBucketIndex(adam, status);
292    TEST_CHECK_STATUS;
293    TEST_ASSERT(n == 1);    /*  Label #0 is underflow, 1 is A, etc. */
294    n = index->getBucketIndex(baker, status);
295    TEST_ASSERT(n == 2);
296    n = index->getBucketIndex(Cyrillic, status);
297    TEST_ASSERT(n == 27);   // Overflow label
298    n = index->getBucketIndex(zed, status);
299    TEST_ASSERT(n == 26);
300
301    for (i=0; index->nextBucket(status); i++) {
302        n = index->getBucketIndex();
303        TEST_ASSERT(n == i);
304        UnicodeString label = index->getBucketLabel();
305        TEST_ASSERT(n == i);
306    }
307    TEST_ASSERT(i == 28);
308
309    delete index;
310    index = new AlphabeticIndex(Locale::createFromName("ru"), status);
311    TEST_CHECK_STATUS;
312    assertEquals("Russian index.getBucketCount()", 32, index->getBucketCount(status));
313    // Latin-script names should go into the underflow label (0)
314    // if the Russian collation does not use script reordering,
315    // but into the overflow label (getBucketCount()-1)
316    // if Russian sorts Cyrillic first.
317    int32_t reorderCodes[20];
318    int32_t expectedLatinIndex = 0;
319    if (index->getCollator().getReorderCodes(reorderCodes, LENGTHOF(reorderCodes), status) > 0) {
320        expectedLatinIndex = index->getBucketCount(status) - 1;
321    }
322    n = index->getBucketIndex(adam, status);
323    TEST_CHECK_STATUS;
324    assertEquals("Russian index.getBucketIndex(adam)", expectedLatinIndex, n);
325    n = index->getBucketIndex(baker, status);
326    assertEquals("Russian index.getBucketIndex(baker)", expectedLatinIndex, n);
327    n = index->getBucketIndex(Cyrillic, status);
328    assertEquals("Russian index.getBucketIndex(Cyrillic)", 1, n);
329    n = index->getBucketIndex(zed, status);
330    assertEquals("Russian index.getBucketIndex(zed)", expectedLatinIndex, n);
331
332    delete index;
333
334}
335
336
337static const char * KEY_LOCALES[] = {
338            "en", "es", "de", "fr", "ja", "it", "tr", "pt", "zh", "nl",
339            "pl", "ar", "ru", "zh_Hant", "ko", "th", "sv", "fi", "da",
340            "he", "nb", "el", "hr", "bg", "sk", "lt", "vi", "lv", "sr",
341            "pt_PT", "ro", "hu", "cs", "id", "sl", "fil", "fa", "uk",
342            "ca", "hi", "et", "eu", "is", "sw", "ms", "bn", "am", "ta",
343            "te", "mr", "ur", "ml", "kn", "gu", "or", ""};
344
345
346void AlphabeticIndexTest::ManyLocalesTest() {
347    UErrorCode status = U_ZERO_ERROR;
348    int32_t  lc = 0;
349
350    for (int i=0; ; ++i) {
351        status = U_ZERO_ERROR;
352        const char *localeName = KEY_LOCALES[i];
353        if (localeName[0] == 0) {
354            break;
355        }
356        // std::cout <<  localeName << "  ";
357        Locale loc = Locale::createFromName(localeName);
358        AlphabeticIndex index(loc, status);
359        TEST_CHECK_STATUS;
360        lc = index.getBucketCount(status);
361        TEST_CHECK_STATUS;
362        // std::cout << "getBucketCount() == " << lc << std::endl;
363
364        LocalPointer<AlphabeticIndex::ImmutableIndex> immIndex(index.buildImmutableIndex(status));
365        TEST_CHECK_STATUS;
366        TEST_ASSERT(lc == immIndex->getBucketCount());
367
368        assertEquals("initial bucket index", -1, index.getBucketIndex());
369        int32_t bucketIndex = 0;
370        while (index.nextBucket(status)) {
371            TEST_CHECK_STATUS;
372            assertEquals("bucket index", bucketIndex, index.getBucketIndex());
373            const UnicodeString &label = index.getBucketLabel();
374            TEST_ASSERT(label.length()>0);
375            // std::string ss;
376            // std::cout << ":" << label.toUTF8String(ss);
377            const AlphabeticIndex::Bucket *bucket = immIndex->getBucket(bucketIndex);
378            TEST_ASSERT(bucket != NULL);
379            assertEquals("bucket label vs. immutable: locale=" + UnicodeString(localeName) +
380                         " index=" + bucketIndex,
381                         label, bucket->getLabel());
382            TEST_ASSERT(&label != &bucket->getLabel());  // not the same pointers
383            UAlphabeticIndexLabelType labelType = index.getBucketLabelType();
384            TEST_ASSERT(labelType == bucket->getLabelType());
385            ++bucketIndex;
386        }
387        // std::cout << ":" << std::endl;
388
389        TEST_ASSERT(immIndex->getBucketCount() == bucketIndex);
390        TEST_ASSERT(immIndex->getBucket(-1) == NULL);
391        TEST_ASSERT(immIndex->getBucket(bucketIndex) == NULL);
392    }
393}
394
395
396// Test data for Pinyin based indexes.
397//  The Chinese characters should be distributed under latin labels in
398//  an index.
399
400static const char *pinyinTestData[] = {
401        "\\u0101", "\\u5416", "\\u58ba", //
402        "b", "\\u516b", "\\u62d4", "\\u8500", //
403        "c", "\\u5693", "\\u7938", "\\u9e7e", //
404        "d", "\\u5491", "\\u8fcf", "\\u964a", //
405        "\\u0113","\\u59b8", "\\u92e8", "\\u834b", //
406        "f", "\\u53d1", "\\u9197", "\\u99a5", //
407        "g", "\\u7324", "\\u91d3", "\\u8142", //
408        "h", "\\u598e", "\\u927f", "\\u593b", //
409        "j", "\\u4e0c", "\\u6785", "\\u9d58", //
410        "k", "\\u5494", "\\u958b", "\\u7a52", //
411        "l", "\\u5783", "\\u62c9", "\\u9ba5", //
412        "m", "\\u5638", "\\u9ebb", "\\u65c0", //
413        "n", "\\u62ff", "\\u80ad", "\\u685b", //
414        "\\u014D", "\\u5662", "\\u6bee", "\\u8bb4", //
415        "p", "\\u5991", "\\u8019", "\\u8c31", //
416        "q", "\\u4e03", "\\u6053", "\\u7f56", //
417        "r", "\\u5465", "\\u72aa", "\\u6e03", //
418        "s", "\\u4ee8", "\\u9491", "\\u93c1", //
419        "t", "\\u4ed6", "\\u9248", "\\u67dd", //
420        "w", "\\u5c72", "\\u5558", "\\u5a7a", //
421        "x", "\\u5915", "\\u5438", "\\u6bbe", //
422        "y", "\\u4e2b", "\\u82bd", "\\u8574", //
423        "z", "\\u5e00", "\\u707d", "\\u5c0a",
424        NULL
425    };
426
427void AlphabeticIndexTest::HackPinyinTest() {
428    UErrorCode status = U_ZERO_ERROR;
429    AlphabeticIndex aindex(Locale::createFromName("zh"), status);
430    TEST_CHECK_STATUS;
431
432    UnicodeString names[sizeof(pinyinTestData) / sizeof(pinyinTestData[0])];
433    int32_t  nameCount;
434    for (nameCount=0; pinyinTestData[nameCount] != NULL; nameCount++) {
435        names[nameCount] = UnicodeString(pinyinTestData[nameCount], -1, UnicodeString::kInvariant).unescape();
436        aindex.addRecord(names[nameCount], &names[nameCount], status);
437        TEST_CHECK_STATUS;
438        if (U_FAILURE(status)) {
439            return;
440        }
441    }
442    TEST_ASSERT(nameCount == aindex.getRecordCount(status));
443
444    // Weak checking:  make sure that none of the Chinese names landed in the overflow bucket
445    //   of the index, and that the names are distributed among several buckets.
446    //   (Exact expected data would be subject to change with evolution of the collation rules.)
447
448    int32_t bucketCount = 0;
449    int32_t filledBucketCount = 0;
450    while (aindex.nextBucket(status)) {
451        bucketCount++;
452        UnicodeString label = aindex.getBucketLabel();
453        // std::string s;
454        // std::cout << label.toUTF8String(s) << ":  ";
455
456        UBool  bucketHasContents = FALSE;
457        while (aindex.nextRecord(status)) {
458            bucketHasContents = TRUE;
459            UnicodeString name = aindex.getRecordName();
460            if (aindex.getBucketLabelType() != U_ALPHAINDEX_NORMAL) {
461                errln("File %s, Line %d, Name \"\\u%x\" is in an under or overflow bucket.",
462                    __FILE__, __LINE__, name.char32At(0));
463            }
464            // s.clear();
465            // std::cout << aindex.getRecordName().toUTF8String(s) << " ";
466        }
467        if (bucketHasContents) {
468            filledBucketCount++;
469        }
470        // std::cout << std::endl;
471    }
472    TEST_ASSERT(bucketCount > 25);
473    TEST_ASSERT(filledBucketCount > 15);
474}
475
476
477void AlphabeticIndexTest::TestBug9009() {
478    UErrorCode status = U_ZERO_ERROR;
479    Locale loc("root");
480    AlphabeticIndex aindex(loc, status);
481    TEST_CHECK_STATUS;
482    aindex.nextBucket(status);  // Crash here before bug was fixed.
483    TEST_CHECK_STATUS;
484}
485
486static const char *localeAndIndexCharactersLists[][2] = {
487    /* Arabic*/ {"ar", "\\u0627:\\u0628:\\u062A:\\u062B:\\u062C:\\u062D:\\u062E:\\u062F:\\u0630:\\u0631:\\u0632:\\u0633:\\u0634:\\u0635:\\u0636:\\u0637:\\u0638:\\u0639:\\u063A:\\u0641:\\u0642:\\u0643:\\u0644:\\u0645:\\u0646:\\u0647:\\u0648:\\u064A"},
488    /* Bulgarian*/  {"bg", "\\u0410:\\u0411:\\u0412:\\u0413:\\u0414:\\u0415:\\u0416:\\u0417:\\u0418:\\u0419:\\u041A:\\u041B:\\u041C:\\u041D:\\u041E:\\u041F:\\u0420:\\u0421:\\u0422:\\u0423:\\u0424:\\u0425:\\u0426:\\u0427:\\u0428:\\u0429:\\u042E:\\u042F"},
489    /* Catalan*/    {"ca", "A:B:C:D:E:F:G:H:I:J:K:L:M:N:O:P:Q:R:S:T:U:V:W:X:Y:Z"},
490    /* Czech*/  {"cs", "A:B:C:\\u010C:D:E:F:G:H:CH:I:J:K:L:M:N:O:P:Q:R:\\u0158:S:\\u0160:T:U:V:W:X:Y:Z:\\u017D"},
491    /* Danish*/ {"da", "A:B:C:D:E:F:G:H:I:J:K:L:M:N:O:P:Q:R:S:T:U:V:W:X:Y:Z:\\u00C6:\\u00D8:\\u00C5"},
492    /* German*/ {"de", "A:B:C:D:E:F:G:H:I:J:K:L:M:N:O:P:Q:R:S:T:U:V:W:X:Y:Z"},
493    /* Greek*/  {"el", "\\u0391:\\u0392:\\u0393:\\u0394:\\u0395:\\u0396:\\u0397:\\u0398:\\u0399:\\u039A:\\u039B:\\u039C:\\u039D:\\u039E:\\u039F:\\u03A0:\\u03A1:\\u03A3:\\u03A4:\\u03A5:\\u03A6:\\u03A7:\\u03A8:\\u03A9"},
494    /* English*/    {"en", "A:B:C:D:E:F:G:H:I:J:K:L:M:N:O:P:Q:R:S:T:U:V:W:X:Y:Z"},
495    /* Spanish*/    {"es", "A:B:C:D:E:F:G:H:I:J:K:L:M:N:\\u00D1:O:P:Q:R:S:T:U:V:W:X:Y:Z"},
496    /* Estonian*/   {"et", "A:B:C:D:E:F:G:H:I:J:K:L:M:N:O:P:Q:R:S:\\u0160:Z:\\u017D:T:U:V:\\u00D5:\\u00C4:\\u00D6:\\u00DC:X:Y"},
497    /* Finnish*/    {"fi", "A:B:C:D:E:F:G:H:I:J:K:L:M:N:O:P:Q:R:S:T:U:V:W:X:Y:Z:\\u00C5:\\u00C4:\\u00D6"},
498    /* Filipino*/   {"fil", "A:B:C:D:E:F:G:H:I:J:K:L:M:N:O:P:Q:R:S:T:U:V:W:X:Y:Z"},
499    /* French*/ {"fr", "A:B:C:D:E:F:G:H:I:J:K:L:M:N:O:P:Q:R:S:T:U:V:W:X:Y:Z"},
500    /* Hebrew*/ {"he", "\\u05D0:\\u05D1:\\u05D2:\\u05D3:\\u05D4:\\u05D5:\\u05D6:\\u05D7:\\u05D8:\\u05D9:\\u05DB:\\u05DC:\\u05DE:\\u05E0:\\u05E1:\\u05E2:\\u05E4:\\u05E6:\\u05E7:\\u05E8:\\u05E9:\\u05EA"},
501    /* Icelandic*/  {"is", "A:\\u00C1:B:C:D:\\u00D0:E:\\u00C9:F:G:H:I:\\u00CD:J:K:L:M:N:O:\\u00D3:P:Q:R:S:T:U:\\u00DA:V:W:X:Y:\\u00DD:Z:\\u00DE:\\u00C6:\\u00D6"},
502    /* Italian*/    {"it", "A:B:C:D:E:F:G:H:I:J:K:L:M:N:O:P:Q:R:S:T:U:V:W:X:Y:Z"},
503    /* Japanese*/   {"ja", "\\u3042:\\u304B:\\u3055:\\u305F:\\u306A:\\u306F:\\u307E:\\u3084:\\u3089:\\u308F"},
504    /* Korean*/ {"ko", "\\u3131:\\u3134:\\u3137:\\u3139:\\u3141:\\u3142:\\u3145:\\u3147:\\u3148:\\u314A:\\u314B:\\u314C:\\u314D:\\u314E"},
505    /* Lithuanian*/ {"lt", "A:B:C:\\u010C:D:E:F:G:H:I:J:K:L:M:N:O:P:R:S:\\u0160:T:U:V:Z:\\u017D"},
506    /* Latvian*/    {"lv", "A:B:C:\\u010C:D:E:F:G:\\u0122:H:I:J:K:\\u0136:L:\\u013B:M:N:\\u0145:O:P:Q:R:S:\\u0160:T:U:V:W:X:Z:\\u017D"},
507    /* Norwegian Bokm\\u00E5l*/  {"nb", "A:B:C:D:E:F:G:H:I:J:K:L:M:N:O:P:Q:R:S:T:U:V:W:X:Y:Z:\\u00C6:\\u00D8:\\u00C5"},
508    /* Dutch*/  {"nl", "A:B:C:D:E:F:G:H:I:J:K:L:M:N:O:P:Q:R:S:T:U:V:W:X:Y:Z"},
509    /* Polish*/ {"pl", "A:\\u0104:B:C:\\u0106:D:E:\\u0118:F:G:H:I:J:K:L:\\u0141:M:N:\\u0143:O:\\u00D3:P:Q:R:S:\\u015A:T:U:V:W:X:Y:Z:\\u0179:\\u017B"},
510    /* Portuguese*/ {"pt", "A:B:C:D:E:F:G:H:I:J:K:L:M:N:O:P:Q:R:S:T:U:V:W:X:Y:Z"},
511    /* Romanian*/   {"ro", "A:\\u0102:\\u00C2:B:C:D:E:F:G:H:I:\\u00CE:J:K:L:M:N:O:P:Q:R:S:\\u0218:T:\\u021A:U:V:W:X:Y:Z"},
512    /* Russian*/    {"ru", "\\u0410:\\u0411:\\u0412:\\u0413:\\u0414:\\u0415:\\u0416:\\u0417:\\u0418:\\u0419:\\u041A:\\u041B:\\u041C:\\u041D:\\u041E:\\u041F:\\u0420:\\u0421:\\u0422:\\u0423:\\u0424:\\u0425:\\u0426:\\u0427:\\u0428:\\u0429:\\u042B:\\u042D:\\u042E:\\u042F"},
513    /* Slovak*/ {"sk", "A:\\u00C4:B:C:\\u010C:D:E:F:G:H:CH:I:J:K:L:M:N:O:\\u00D4:P:Q:R:S:\\u0160:T:U:V:W:X:Y:Z:\\u017D"},
514    /* Slovenian*/  {"sl", "A:B:C:\\u010C:\\u0106:D:\\u0110:E:F:G:H:I:J:K:L:M:N:O:P:Q:R:S:\\u0160:T:U:V:W:X:Y:Z:\\u017D"},
515    /* Serbian*/    {"sr", "\\u0410:\\u0411:\\u0412:\\u0413:\\u0414:\\u0402:\\u0415:\\u0416:\\u0417:\\u0418:\\u0408:\\u041A:\\u041B:\\u0409:\\u041C:\\u041D:\\u040A:\\u041E:\\u041F:\\u0420:\\u0421:\\u0422:\\u040B:\\u0423:\\u0424:\\u0425:\\u0426:\\u0427:\\u040F:\\u0428"},
516    /* Swedish*/    {"sv", "A:B:C:D:E:F:G:H:I:J:K:L:M:N:O:P:Q:R:S:T:U:V:W:X:Y:Z:\\u00C5:\\u00C4:\\u00D6"},
517    /* Turkish*/    {"tr", "A:B:C:\\u00C7:D:E:F:G:H:I:\\u0130:J:K:L:M:N:O:\\u00D6:P:Q:R:S:\\u015E:T:U:\\u00DC:V:W:X:Y:Z"},
518    /* Ukrainian*/  {"uk", "\\u0410:\\u0411:\\u0412:\\u0413:\\u0490:\\u0414:\\u0415:\\u0404:\\u0416:\\u0417:\\u0418:\\u0406:\\u0407:\\u0419:\\u041A:\\u041B:\\u041C:\\u041D:\\u041E:\\u041F:\\u0420:\\u0421:\\u0422:\\u0423:\\u0424:\\u0425:\\u0426:\\u0427:\\u0428:\\u0429:\\u042E:\\u042F"},
519    /* Vietnamese*/ {"vi", "A:\\u0102:\\u00C2:B:C:D:\\u0110:E:\\u00CA:F:G:H:I:J:K:L:M:N:O:\\u00D4:\\u01A0:P:Q:R:S:T:U:\\u01AF:V:W:X:Y:Z"},
520    /* Chinese*/    {"zh", "A:B:C:D:E:F:G:H:I:J:K:L:M:N:O:P:Q:R:S:T:U:V:W:X:Y:Z"},
521    /* Chinese (Traditional Han)*/  {"zh_Hant", "1\\u5283:2\\u5283:3\\u5283:4\\u5283:5\\u5283:6\\u5283:7\\u5283:8\\u5283:9\\u5283:10\\u5283:11\\u5283:12\\u5283:13\\u5283:14\\u5283:15\\u5283:16\\u5283:17\\u5283:18\\u5283:19\\u5283:20\\u5283:21\\u5283:22\\u5283:23\\u5283:24\\u5283:25\\u5283:26\\u5283:27\\u5283:28\\u5283:29\\u5283:30\\u5283:31\\u5283:32\\u5283:33\\u5283:35\\u5283:36\\u5283:39\\u5283:48\\u5283"},
522
523    // As of ICU 52, ICU does not have collation data for the following language.
524    // Therefore, constructing an AlphabeticIndex for it
525    // ends up with a collator for the default locale
526    // which makes the test unreliable. (see ticket #10277)
527    // It exposes a bigger problem in that it may not be desirable for collation
528    // to fall back to the default locale.
529
530    // /* Basque*/ {"eu", "A:B:C:D:E:F:G:H:I:J:K:L:M:N:O:P:Q:R:S:T:U:V:W:X:Y:Z"},
531};
532
533void AlphabeticIndexTest::TestIndexCharactersList() {
534    UErrorCode status = U_ZERO_ERROR;
535    for (int32_t i = 0; i < LENGTHOF(localeAndIndexCharactersLists); ++i) {
536        const char *(&localeAndIndexCharacters)[2] = localeAndIndexCharactersLists[i];
537        const char *locale = localeAndIndexCharacters[0];
538        UnicodeString expectedIndexCharacters
539            = (UnicodeString("\\u2026:") + localeAndIndexCharacters[1] + ":\\u2026").unescape();
540        AlphabeticIndex index(locale, status);
541        TEST_CHECK_STATUS;
542        LocalPointer<AlphabeticIndex::ImmutableIndex> immIndex(index.buildImmutableIndex(status));
543        TEST_CHECK_STATUS;
544
545        // Join the elements of the list to a string with delimiter ":"
546        UnicodeString actualIndexCharacters;
547        assertEquals(locale,
548                     expectedIndexCharacters,
549                     joinLabelsAndAppend(*immIndex, actualIndexCharacters));
550        logln(locale + UnicodeString(": ") + actualIndexCharacters);
551    }
552}
553
554void AlphabeticIndexTest::TestHaniFirst() {
555    UErrorCode status = U_ZERO_ERROR;
556    LocalPointer<RuleBasedCollator> coll(
557        static_cast<RuleBasedCollator *>(Collator::createInstance(Locale::getRoot(), status)));
558
559    if (U_FAILURE(status)) {
560        dataerrln("Failed Collator::createInstance call - %s", u_errorName(status));
561        return;
562    }
563    int32_t reorderCodes[] = { USCRIPT_HAN };
564    coll->setReorderCodes(reorderCodes, LENGTHOF(reorderCodes), status);
565    TEST_CHECK_STATUS;
566    AlphabeticIndex index(coll.orphan(), status);
567    TEST_CHECK_STATUS;
568    assertEquals("getBucketCount()", 1, index.getBucketCount(status));   // ... (underflow only)
569    index.addLabels(Locale::getEnglish(), status);
570    assertEquals("getBucketCount()", 28, index.getBucketCount(status));  // ... A-Z ...
571    int32_t bucketIndex = index.getBucketIndex(UnicodeString((UChar)0x897f), status);
572    assertEquals("getBucketIndex(U+897F)", 0, bucketIndex);  // underflow bucket
573    bucketIndex = index.getBucketIndex("i", status);
574    assertEquals("getBucketIndex(i)", 9, bucketIndex);
575    bucketIndex = index.getBucketIndex(UnicodeString((UChar)0x03B1), status);
576    assertEquals("getBucketIndex(Greek alpha)", 27, bucketIndex);
577    // U+50005 is an unassigned code point which sorts at the end, independent of the Hani group.
578    bucketIndex = index.getBucketIndex(UnicodeString(0x50005), status);
579    assertEquals("getBucketIndex(U+50005)", 27, bucketIndex);
580    bucketIndex = index.getBucketIndex(UnicodeString((UChar)0xFFFF), status);
581    assertEquals("getBucketIndex(U+FFFF)", 27, bucketIndex);
582}
583
584void AlphabeticIndexTest::TestPinyinFirst() {
585    UErrorCode status = U_ZERO_ERROR;
586    LocalPointer<RuleBasedCollator> coll(
587        static_cast<RuleBasedCollator *>(Collator::createInstance(Locale::getChinese(), status)));
588    if (U_FAILURE(status)) {
589        dataerrln("Failed Collator::createInstance call - %s", u_errorName(status));
590        return;
591    }
592    int32_t reorderCodes[] = { USCRIPT_HAN };
593    coll->setReorderCodes(reorderCodes, LENGTHOF(reorderCodes), status);
594    TEST_CHECK_STATUS;
595    AlphabeticIndex index(coll.orphan(), status);
596    TEST_CHECK_STATUS;
597    assertEquals("getBucketCount()", 28, index.getBucketCount(status));   // ... A-Z ...
598    index.addLabels(Locale::getChinese(), status);
599    assertEquals("getBucketCount()", 28, index.getBucketCount(status));  // ... A-Z ...
600    int32_t bucketIndex = index.getBucketIndex(UnicodeString((UChar)0x897f), status);
601    assertEquals("getBucketIndex(U+897F)", (int32_t)((UChar)0x0058/*X*/ - (UChar)0x0041/*A*/ + 1), (int32_t)bucketIndex);
602    bucketIndex = index.getBucketIndex("i", status);
603    assertEquals("getBucketIndex(i)", 9, bucketIndex);
604    bucketIndex = index.getBucketIndex(UnicodeString((UChar)0x03B1), status);
605    assertEquals("getBucketIndex(Greek alpha)", (int32_t)27, bucketIndex);
606    // U+50005 is an unassigned code point which sorts at the end, independent of the Hani group.
607    bucketIndex = index.getBucketIndex(UnicodeString(0x50005), status);
608    assertEquals("getBucketIndex(U+50005)", 27, bucketIndex);
609    bucketIndex = index.getBucketIndex(UnicodeString((UChar)0xFFFF), status);
610    assertEquals("getBucketIndex(U+FFFF)", 27, bucketIndex);
611}
612
613void AlphabeticIndexTest::TestSchSt() {
614    UErrorCode status = U_ZERO_ERROR;
615    AlphabeticIndex index(Locale::getGerman(), status);
616    index.addLabels(UnicodeSet("[\\u00C6{Sch*}{St*}]", status), status);
617    TEST_CHECK_STATUS;
618    // ... A AE-ligature B-R S Sch St T-Z ...
619    LocalPointer<AlphabeticIndex::ImmutableIndex> immIndex(index.buildImmutableIndex(status));
620    TEST_CHECK_STATUS;
621    assertEquals("getBucketCount()", 31, index.getBucketCount(status));
622    assertEquals("immutable getBucketCount()", 31, immIndex->getBucketCount());
623    static const struct TestCase {
624        const char *name;
625        int32_t bucketIndex;
626        const char *bucketLabel;
627    } testCases[] = {
628        // name, bucket index, bucket label
629        { "Adelbert", 1, "A" },
630        { "Afrika", 1, "A" },
631        { "\\u00C6sculap", 2, "\\u00C6" },
632        { "Aesthet", 2, "\\u00C6" },
633        { "Berlin", 3, "B" },
634        { "Rilke", 19, "R" },
635        { "Sacher", 20, "S" },
636        { "Seiler", 20, "S" },
637        { "Sultan", 20, "S" },
638        { "Schiller", 21, "Sch" },
639        { "Steiff", 22, "St" },
640        { "Thomas", 23, "T" }
641    };
642    for (int32_t i = 0; i < LENGTHOF(testCases); ++i) {
643        const TestCase &testCase = testCases[i];
644        UnicodeString name = UnicodeString(testCase.name).unescape();
645        UnicodeString label = UnicodeString(testCase.bucketLabel).unescape();
646        char msg[100];
647        sprintf(msg, "getBucketIndex(%s)", testCase.name);
648        assertEquals(msg, testCase.bucketIndex, index.getBucketIndex(name, status));
649        sprintf(msg, "immutable getBucketIndex(%s)", testCase.name);
650        assertEquals(msg, testCase.bucketIndex, immIndex->getBucketIndex(name, status));
651        sprintf(msg, "immutable bucket label (%s)", testCase.name);
652        assertEquals(msg, label, immIndex->getBucket(testCase.bucketIndex)->getLabel());
653    }
654}
655
656void AlphabeticIndexTest::TestNoLabels() {
657    UErrorCode status = U_ZERO_ERROR;
658    LocalPointer<RuleBasedCollator> coll(
659        static_cast<RuleBasedCollator *>(Collator::createInstance(Locale::getRoot(), status)));
660    TEST_CHECK_STATUS;
661    AlphabeticIndex index(coll.orphan(), status);
662    TEST_CHECK_STATUS;
663    index.addRecord(UnicodeString((UChar)0x897f), NULL, status);
664    index.addRecord("i", NULL, status);
665    index.addRecord(UnicodeString((UChar)0x03B1), NULL, status);
666    assertEquals("getBucketCount()", 1, index.getBucketCount(status));  // ...
667    TEST_ASSERT(index.nextBucket(status));
668    assertEquals("underflow label type", (int32_t)U_ALPHAINDEX_UNDERFLOW, index.getBucketLabelType());
669    assertEquals("all records in the underflow bucket", (int32_t)3, index.getBucketRecordCount());
670}
671
672void AlphabeticIndexTest::TestChineseZhuyin() {
673    UErrorCode status = U_ZERO_ERROR;
674    char loc[100];
675    uloc_forLanguageTag("zh-u-co-zhuyin", loc, LENGTHOF(loc), NULL, &status);
676    AlphabeticIndex index(loc, status);
677    LocalPointer<AlphabeticIndex::ImmutableIndex> immIndex(index.buildImmutableIndex(status));
678    TEST_CHECK_STATUS;
679    assertEquals("getBucketCount()", 38, immIndex->getBucketCount());
680    assertEquals("label 1", UnicodeString((UChar)0x3105), immIndex->getBucket(1)->getLabel());
681    assertEquals("label 2", UnicodeString((UChar)0x3106), immIndex->getBucket(2)->getLabel());
682    assertEquals("label 3", UnicodeString((UChar)0x3107), immIndex->getBucket(3)->getLabel());
683    assertEquals("label 4", UnicodeString((UChar)0x3108), immIndex->getBucket(4)->getLabel());
684    assertEquals("label 5", UnicodeString((UChar)0x3109), immIndex->getBucket(5)->getLabel());
685}
686
687void AlphabeticIndexTest::TestJapaneseKanji() {
688    UErrorCode status = U_ZERO_ERROR;
689    AlphabeticIndex index(Locale::getJapanese(), status);
690    LocalPointer<AlphabeticIndex::ImmutableIndex> immIndex(index.buildImmutableIndex(status));
691    TEST_CHECK_STATUS;
692    // There are no index characters for Kanji in the Japanese standard collator.
693    // They should all go into the overflow bucket.
694    static const UChar32 kanji[] = { 0x4E9C, 0x95C7, 0x4E00, 0x58F1 };
695    int32_t overflowIndex = immIndex->getBucketCount() - 1;
696    for(int32_t i = 0; i < LENGTHOF(kanji); ++i) {
697        char msg[40];
698        sprintf(msg, "kanji[%d]=U+%04lX in overflow bucket", (int)i, (long)kanji[i]);
699        assertEquals(msg, overflowIndex, immIndex->getBucketIndex(UnicodeString(kanji[i]), status));
700        TEST_CHECK_STATUS;
701    }
702}
703
704#endif
705